per-pixel transparent window using Windows Composition engine in C++

I want to display a raw BGRA data (so I have an unsigned int *, with width and height) on a window so that the alpha component is taken into account to have per-pixel transparency. The final purpose is to integrate this in a graphic toolkit.

after reading Kenny Kerr's article (https://docs.microsoft.com/en-us/archive/msdn-magazine/2014/june/windows-with-c-high-performance-window-layering-using-the-windows-composition-engine) I think that it can solve my problem.

I have already written a complete code that does what the article described (transparent window using Windows Composition engine in c++) : transparent window with a translucent disk

So, basically, what I am doing now is :

• get the buffer (a surface) from the swapchain
• get a bitmap with CreateBitmapFromDxgiSurface
• I allocate a BGRA test data
• use CopyFromMemory to copy my BGRA data onto the bitmap
• between BeginDraw and EndDraw, I call DrawBitmap

But whatever flag I use for the properties' bitmapOptions passed to CreateBitmapFromDxgiSurface, the app crashes and the D2D debug layer mentions an error like : "warning: D2D DEBUG ERROR - The bitmap options [0x1] must be a subset of the flags associated with the DXGI surface."

the complete code is here :

#include <iostream>

#ifdef _WIN32_WINNT
# undef _WIN32_WINNT
#endif
#define _WIN32_WINNT 0x0A00

#include <guiddef.h>
#include <d2d1_1.h>
#include <d3d11.h>
#include <dxgi1_3.h>
#include <dcomp.h>

static HINSTANCE instance = NULL;
static HWND win = NULL;

LRESULT CALLBACK _window_procedure(HWND   window,
                                   UINT   message,
                                   WPARAM window_param,
                                   LPARAM data_param);

typedef struct
{
    ID3D11Device *d3d_device;
    IDXGIDevice *dxgi_device;
    IDXGIFactory2 *dxgi_factory;
    ID2D1Factory1 *d2d_factory;
    ID2D1Device *d2d_device;
    ID2D1DeviceContext *d2d_device_ctx;
    IDCompositionDevice *dcomp_device;
    IDCompositionVisual *dcomp_visual;
    IDCompositionTarget *dcomp_target;
    IDXGISwapChain1 *dxgi_swapchain;
} D2d;

int d2d_init(HWND window, D2d *d2d)
{
    const D3D_FEATURE_LEVEL levels[] =
    {
        D3D_FEATURE_LEVEL_12_1,
        D3D_FEATURE_LEVEL_12_0,
        D3D_FEATURE_LEVEL_11_1,
        D3D_FEATURE_LEVEL_11_0,
        D3D_FEATURE_LEVEL_10_1,
        D3D_FEATURE_LEVEL_10_0
    };
    D2D1_FACTORY_OPTIONS opt;
    DXGI_SWAP_CHAIN_DESC1 desc;
    RECT r;
    HRESULT res;

    /* direct3d device */
    res = D3D11CreateDevice(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL,
                            D3D11_CREATE_DEVICE_BGRA_SUPPORT |
                            D3D11_CREATE_DEVICE_DEBUG,
                            levels, sizeof(levels) / sizeof(D3D_FEATURE_LEVEL),
                            D3D11_SDK_VERSION, &d2d->d3d_device, NULL, NULL);
    if (FAILED(res))
        return 0;

    /* dxgi device */
    res = d2d->d3d_device->QueryInterface(&d2d->dxgi_device);
    if (FAILED(res))
        goto release_d3d_device;

    /* dxgi factory */
    res = CreateDXGIFactory2(DXGI_CREATE_FACTORY_DEBUG,
                             __uuidof(d2d->dxgi_factory),
                             (void **)&d2d->dxgi_factory);
    if (FAILED(res))
        goto release_dxgi_device;

    /* d2d factory */
    opt.debugLevel = D2D1_DEBUG_LEVEL_INFORMATION;
    res = D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED,
                            __uuidof(d2d->d2d_factory), &opt,
                            (void **)&d2d->d2d_factory);
    if (FAILED(res))
        goto release_dxgi_factory;

    /* d2d device */

    res = d2d->d2d_factory->CreateDevice(d2d->dxgi_device, &d2d->d2d_device);
    if (FAILED(res))
        goto release_d2d_factory;

    /* d2d device context */

    // FIXME : D2D1_DEVICE_CONTEXT_OPTIONS_ENABLE_MULTITHREADED_OPTIMIZATIONS
    res = d2d->d2d_device->CreateDeviceContext(D2D1_DEVICE_CONTEXT_OPTIONS_NONE,
                                               &d2d->d2d_device_ctx);
    if (FAILED(res))
        goto release_d2d_device;

    /* dcomp device */
    res = DCompositionCreateDevice(d2d->dxgi_device,
                                   __uuidof(d2d->dcomp_device),
                                   (void **)&d2d->dcomp_device);
    if (FAILED(res))
        goto release_d2d_device_ctx;

    /* dcomp visual */
    res = d2d->dcomp_device->CreateVisual(&d2d->dcomp_visual);
    if (FAILED(res))
        goto release_dcomp_device;

    /* dcomp target */
    res = d2d->dcomp_device->CreateTargetForHwnd(window, TRUE,
                                                 &d2d->dcomp_target);
    if (FAILED(res))
        goto release_dcomp_visual;

    /* dxgi swapchain */
    if (!GetClientRect(window, &r))
        goto release_dcomp_target;

    desc.Width = r.right - r.left; /* width of client area */
    desc.Height = r.bottom - r.top; /* height of client area */
    desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
    desc.Stereo = FALSE;
    desc.SampleDesc.Count = 1;
    desc.SampleDesc.Quality = 0;
    desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
    desc.BufferCount = 2;
    desc.Scaling = DXGI_SCALING_STRETCH;
    desc.SwapEffect= DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
    desc.AlphaMode = DXGI_ALPHA_MODE_PREMULTIPLIED;
    desc.Flags = 0;

    res = d2d->dxgi_factory->CreateSwapChainForComposition(d2d->dxgi_device,
                                                           &desc,
                                                           NULL,
                                                           &d2d->dxgi_swapchain);
    if (FAILED(res))
        goto release_dcomp_target;

    return 1;

  release_dcomp_target:
    d2d->dcomp_target->Release();
  release_dcomp_visual:
    d2d->dcomp_visual->Release();
  release_dcomp_device:
    d2d->dcomp_device->Release();
  release_d2d_device_ctx:
    d2d->d2d_device_ctx->Release();
  release_d2d_device:
    d2d->d2d_device->Release();
  release_d2d_factory:
    d2d->d2d_factory->Release();
  release_dxgi_factory:
    d2d->dxgi_factory->Release();
  release_dxgi_device:
    d2d->dxgi_device->Release();
  release_d3d_device:
    d2d->d3d_device->Release();

    return 0;
}

void d2d_shutdown(D2d *d2d)
{
    d2d->dxgi_swapchain->Release();
    d2d->dcomp_target->Release();
    d2d->dcomp_visual->Release();
    d2d->dcomp_device->Release();
    d2d->d2d_device_ctx->Release();
    d2d->d2d_device->Release();
    d2d->d2d_factory->Release();
    d2d->dxgi_factory->Release();
    d2d->dxgi_device->Release();
    d2d->d3d_device->Release();
}

void render(D2d *d2d)
{
    IDXGISurface *surface;
    ID2D1Bitmap1 *bitmap;
    D2D1_BITMAP_PROPERTIES1 properties;
    D2D1_COLOR_F c;
    HRESULT res;

    std::cout << "render" << std::endl;

    /* swapchain buffer in a IDXGISurface */
    res = d2d->dxgi_swapchain->GetBuffer(0, __uuidof(surface),
                                         (void **)&surface);
    if (FAILED(res))
        return;
    std::cout << "render 1" << std::endl;

    properties.pixelFormat.alphaMode = D2D1_ALPHA_MODE_PREMULTIPLIED;
    properties.pixelFormat.format = DXGI_FORMAT_B8G8R8A8_UNORM;
    properties.dpiX = 96;
    properties.dpiY = 96;
    // which value to set here ??
    properties.bitmapOptions = D2D1_BITMAP_OPTIONS_TARGET;
    properties.colorContext = NULL;

    res = d2d->d2d_device_ctx->CreateBitmapFromDxgiSurface(surface,
                                                           &properties,
                                                           &bitmap);
    if (FAILED(res))
        goto release_surface;
    std::cout << "render 2" << std::endl;

    d2d->d2d_device_ctx->SetTarget(bitmap);
    std::cout << "render 3" << std::endl;

    unsigned int *data, *iter;
    unsigned int i, j;
    D2D1_SIZE_U s;
    D2D1_RECT_U r;

    // BGRA test data
    s = bitmap->GetPixelSize();
    data = (unsigned int *)malloc(s.width * s.height * sizeof(unsigned int));
    if (!data)
    {
        std::cout << "malloc fails" << std::endl;
        goto bitmap_release;
    }
    iter = data;
    for (i = 0; i < s.height; i++)
    {
        for (j = 0; j < s.width; j++, iter++)
        {
            if (i >= 50 && j>= 50)
                *iter = 0xff0000ff;
            else
                *iter = 0xffff0000;
        }
    }

    r.left = 0;
    r.top = 0;
    r.right = s.width;
    r.bottom = s.height;
    std::cout << "render 4" << std::endl;
    res = bitmap->CopyFromMemory(&r, data, 4 * s.width);
    if (FAILED(res))
    {
        std::cout << "copy failed" << std::endl;
        goto data_free;
    }
    std::cout << "render 5" << std::endl;

    d2d->d2d_device_ctx->BeginDraw();

    // c.r = 0.18f;
    // c.g = 0.55f;
    // c.b = 0.34f;
    // c.a = 0.75f;

    c.r = 0.0f;
    c.g = 0.0f;
    c.b = 0.0f;
    c.a = 0.0f;
    d2d->d2d_device_ctx->Clear(&c);
    std::cout << "render 6" << std::endl;

    D2D1_RECT_F rect;
    rect.left = 0.0f;
    rect.top = 0.0f;
    rect.right = (float)s.width;
    rect.bottom = (float)s.height;
    d2d->d2d_device_ctx->DrawBitmap(bitmap, rect, 1.0f, D2D1_BITMAP_INTERPOLATION_MODE_LINEAR, rect);
    std::cout << "render 7" << std::endl;

    d2d->d2d_device_ctx->EndDraw(NULL, NULL);

    d2d->dxgi_swapchain->Present(1, 0);
    d2d->dcomp_visual->SetContent(d2d->dxgi_swapchain);
    d2d->dcomp_target->SetRoot(d2d->dcomp_visual);
    d2d->dcomp_device->Commit();

  data_free:
    free(data);
  bitmap_release:
    bitmap->Release();
  release_surface:
    surface->Release();
}


LRESULT CALLBACK
_window_procedure(HWND   window,
                  UINT   message,
                  WPARAM window_param,
                  LPARAM data_param)
{
  switch (message)
    {
    case WM_CLOSE:
      PostQuitMessage(0);
      return 0;
    case WM_KEYUP:
      if (window_param == 'Q')
        {
          PostQuitMessage(0);
        }
      if (window_param == 'T')
        {
            std::cout << "transp" << std::endl;
        }
      return 0;
      /* GDI notifications */
    case WM_CREATE:
        return 0;
    case WM_PAINT:
      {
        RECT rect;
        D2d *d2d;

        d2d = (D2d *)GetWindowLongPtr(window, GWLP_USERDATA);

        std::cout << "paint" << std::endl;

        if (GetUpdateRect(window, &rect, FALSE))
          {
            PAINTSTRUCT ps;
            BeginPaint(window, &ps);

            render(d2d);

            EndPaint(window, &ps);
          }
        return 0;
      }
    default:
      return DefWindowProc(window, message, window_param, data_param);
    }
}

int main()
{
    /* class */
    WNDCLASS wc;

    instance = GetModuleHandle(NULL);
    if (!instance)
        return 1;

    memset (&wc, 0, sizeof (WNDCLASS));
    wc.style = CS_HREDRAW | CS_VREDRAW;
    wc.lpfnWndProc = _window_procedure;
    wc.cbClsExtra = 0;
    wc.cbWndExtra = 0;
    wc.hInstance = instance;
    wc.hIcon = LoadIcon (NULL, IDI_APPLICATION);
    wc.hCursor = LoadCursor (NULL, IDC_ARROW);
    wc.hbrBackground = (HBRUSH)(1 + COLOR_BTNFACE);
    wc.lpszMenuName =  NULL;
    wc.lpszClassName = "D2D";

    if(!RegisterClass(&wc))
        goto free_library;

    /* Window */
    int w;
    int h;
    RECT r;
    DWORD style;
    DWORD exstyle;

    w = 640;
    h = 480;

    style = WS_OVERLAPPEDWINDOW | WS_SIZEBOX;
    exstyle = WS_EX_NOREDIRECTIONBITMAP;

    r.left = 0;
    r.top = 0;
    r.right = w;
    r.bottom = h;
    if (!AdjustWindowRectEx(&r, style, FALSE, exstyle))
        goto unregister_class;

    win = CreateWindowEx(exstyle,
                         "D2D", "Test",
                         style,
                         100, 100,
                         r.right - r.left,
                         r.bottom - r.top,
                         NULL,
                         NULL, instance, NULL);
    if (!win)
        goto unregister_class;

    /* d2d init */
    D2d d2d;

    if (!d2d_init(win, &d2d))
    {
        std::cout << "d2d init failed" << std::endl;
        goto destroy_win;
    }

    SetWindowLongPtr(win, GWLP_USERDATA, (LONG_PTR)&d2d);
    ShowWindow(win, SW_SHOWNORMAL);
    UpdateWindow(win);

    /* msg loop */
    while(1)
    {
        MSG msg;
        BOOL ret;

        ret = PeekMessage(&msg, NULL, 0, 0, PM_REMOVE);
        if (ret)
        {
            do
            {
                if (msg.message == WM_QUIT)
                  goto beach;
                TranslateMessage(&msg);
                DispatchMessageW(&msg);
            } while (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE));
        }
    }

 beach:
    d2d_shutdown(&d2d);
    DestroyWindow(win);
    UnregisterClass("D2D", instance);
    FreeLibrary(instance);

    std::cout << "exiting..." << std::endl;

    return 0;

  destroy_win:
    DestroyWindow(win);
  unregister_class:
    UnregisterClass("D2D", instance);
  free_library:
    FreeLibrary(instance);

    std::cout << "exiting 2..." << std::endl;

    return 1;
}

after 1 week of search on internet, especially on msdn, i couldn't found a solution. I have tried several combination of flags, but always a crash of this program.

Does someone know what is wrong ?

thank you

Bitmaps (D2D1 or others in GPU context) are "just" GPU resources that can serve very different purposes.

Your error comes from the fact you're confusing the target bitmap for the device context with a bitmap you want to write on the device context.

• The first bitmap is just here to express the target for the device context is in fact the DXGI surface.
• The second bitmap is used to write some pixels on the device context (which in turn will land on the DXGI surface).

So you need to change your code. First, just connect the device context (=render target) to the DXGI surface, like this:

void setBackBuffer(D2d* d2d)
{
    IDXGISurface* surface;
    ID2D1Bitmap1* bitmap;
    HRESULT res = d2d->dxgi_swapchain->GetBuffer(0, __uuidof(surface), (void**)&surface);
    if (FAILED(res))
        return;

    res = d2d->d2d_device_ctx->CreateBitmapFromDxgiSurface(surface, NULL, &bitmap);
    if (FAILED(res))
        goto release_surface;

    d2d->d2d_device_ctx->SetTarget(bitmap);
    bitmap->Release();
release_surface:
    surface->Release();
}

And add that just after your call to CreateSwapChainForComposition.

After that, modify your render method like this for example:

void render(D2d* d2d)
{
    ID2D1Bitmap1* bitmap;
    D2D1_BITMAP_PROPERTIES1 properties;
    DXGI_SWAP_CHAIN_DESC1 desc;
    D2D1_COLOR_F c;
    HRESULT res;

    std::cout << "render" << std::endl;

    // get size from swapchain (for example)
    d2d->dxgi_swapchain->GetDesc1(&desc);
    D2D1_SIZE_U s = { desc.Width, desc.Height };

    unsigned int* data, * iter;
    unsigned int i, j;

    // BGRA test data
    data = (unsigned int*)malloc(s.width * s.height * sizeof(unsigned int));
    if (!data)
    {
        std::cout << "malloc fails" << std::endl;
        return;
    }
    iter = data;
    for (i = 0; i < s.height; i++)
    {
        for (j = 0; j < s.width; j++, iter++)
        {
            if (i >= 50 && j >= 50)
                *iter = 0xff0000ff;
            else
                *iter = 0xffff0000;
        }
    }

    std::cout << "render 4" << std::endl;

    // create a bitmap from properties & pixel buffer
    // hint: in general for most structures, it's much easier to use ZeroMemory or memset(0) so by default values are automatically set
    ZeroMemory(&properties, sizeof(properties));
    properties.pixelFormat.format = desc.Format;
    properties.pixelFormat.alphaMode = D2D1_ALPHA_MODE_PREMULTIPLIED;
    res = d2d->d2d_device_ctx->CreateBitmap(s, data, 4 * s.width, properties, &bitmap);
    if (FAILED(res))
    {
        std::cout << "create failed" << std::endl;
        goto data_free;
    }
    std::cout << "render 5" << std::endl;

    d2d->d2d_device_ctx->BeginDraw();

    c.r = 0.0f;
    c.g = 0.0f;
    c.b = 0.0f;
    c.a = 0.0f;
    d2d->d2d_device_ctx->Clear(&c);
    std::cout << "render 6" << std::endl;

    D2D1_RECT_F rect;
    rect.left = 0.0f;
    rect.top = 0.0f;
    rect.right = (float)s.width;
    rect.bottom = (float)s.height;
    d2d->d2d_device_ctx->DrawBitmap(bitmap, rect, 1.0f, D2D1_BITMAP_INTERPOLATION_MODE_LINEAR, rect);
    std::cout << "render 7" << std::endl;

    d2d->d2d_device_ctx->EndDraw(NULL, NULL);

    d2d->dxgi_swapchain->Present(1, 0);
    d2d->dcomp_visual->SetContent(d2d->dxgi_swapchain);
    d2d->dcomp_target->SetRoot(d2d->dcomp_visual);
    d2d->dcomp_device->Commit();

    bitmap->Release();
data_free:
    free(data);
}