TrackMouseEvent Problems

I have a subclassed button that I am trying to highlight when the mouse cursor is over it. However, I cannot seem to get the TrackMouseEvent() function to work properly. Here is the code that creates the subclass:

hBtn = CreateWindow(L"button", L"", WS_CHILD | WS_VISIBLE | BS_OWNERDRAW, 0, 0, 0, 0, hWnd, HMENU(400), hInst, NULL);
SetWindowSubclass(hBtn[0], subSIproc, 400, 0);

Here is the subclass procedure:

LRESULT CALLBACK subSIproc(HWND hButton, UINT iMsg, WPARAM wParam, LPARAM lParam, UINT_PTR uId, DWORD_PTR dwRefData){
     int     HIflag=0;

     switch(iMsg)
      {
        case WM_MOUSEMOVE:
         {
            TRACKMOUSEEVENT me{};
            me.cbSize = sizeof(TRACKMOUSEEVENT);
            me.dwFlags = TME_HOVER | TME_LEAVE;
            me.hwndTrack = hButton;
            me.dwHoverTime = HOVER_DEFAULT;
            TrackMouseEvent(&me);
            HIflag = 1;
            RedrawWindow(hButton, NULL, NULL, RDW_INVALIDATE);
         }
         
        case WM_MOUSEHOVER:
         {
            HIflag = 2;
            RedrawWindow(hButton, NULL, NULL, RDW_INVALIDATE);
         }
         
        case WM_MOUSELEAVE:
         {
            TRACKMOUSEEVENT me{};
            me.cbSize = sizeof(TRACKMOUSEEVENT);
            me.dwFlags = TME_HOVER | TME_LEAVE | TME_CANCEL;
            me.hwndTrack = hButton;
            me.dwHoverTime = HOVER_DEFAULT;
            TrackMouseEvent(&me);
            HIflag = 3;
            RedrawWindow(hButton, NULL, NULL, RDW_INVALIDATE);
         }

        case WM_PAINT:
         {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hButton, &ps);
            if(HIflag==0)  FillRect(hdc, &ps.rcPaint, CreateSolidBrush(0x007F7F7F));
            if(HIflag==1)  FillRect(hdc, &ps.rcPaint, CreateSolidBrush(0x00FF0000));
            if(HIflag==2)  FillRect(hdc, &ps.rcPaint, CreateSolidBrush(0x000000FF));
            if(HIflag==3)  FillRect(hdc, &ps.rcPaint, CreateSolidBrush(0x0000FF00));
            EndPaint(hButton, &ps);
         }
      }
     return DefSubclassProc(hButton, iMsg, wParam, lParam);
  }

The variable "HIflag" has four possible values, to determine which messages are being received, and when---'0' (gray) for no messages yet received; '1' (blue) for WM_MOUSEMOVE received; '2' (red) for WM_MOUSEHOVER received; and '3' for WM_MOUSELEAVE received.

Here is what is happening: When I initially run the program, the button is gray (no mouse messages received). The button remains gray, until I move the cursor onto the button. At that point, it turns green (indicating "WM_MOUSELEAVE"). It should turn red (for "WM_MOUSEHOVER") and not turn green until I move the cursor away from the button. The button now remains green, no matter where I move the cursor.

Does anyone know what I am doing wrong?

Your case blocks are missing breaks, so when WM_MOUSEMOVE is received then the code will fall through to the code for WM_MOUSEHOVER, which will then fall through to the code for WM_MOUSELEAVE, which will then fall through to the code for WM_PAINT, etc. So HIflag will always be 3 when any fall through to WM_PAINT occurs.

You are also leaking the HBRUSH that CreateSolidBrush() returns. You need to DeleteObject() the brush when you are done using it.

But most importantly, HIflag is a local variable inside of subSIproc(), so it will always be 0 when WM_PAINT is issued by the OS itself, rather than when it is reached as the result of fall through.

You need to store HIflag outside of subSIproc() on a per-button basis, such as inside the HWND itself using (Get|Set)WindowLongPtr(GWL_USERDATA), (Get|Set)Prop(), etc.

Try something more like this instead:

hBtn = CreateWindow(L"button", L"", WS_CHILD | WS_VISIBLE | BS_OWNERDRAW, 0, 0, 0, 0, hWnd, HMENU(400), hInst, NULL);
SetProp(hBtn, TEXT("HIflag"), (HANDLE)0);
SetWindowSubclass(hBtn, subSIproc, 400, 0);

...

LRESULT CALLBACK subSIproc(HWND hButton, UINT iMsg, WPARAM wParam, LPARAM lParam, UINT_PTR uId, DWORD_PTR dwRefData)
{
    switch (iMsg)
    {
        case WM_NCDESTROY:
        {
            RemoveProp(hButton, TEXT("HIflag"));
            break;
        }

        case WM_MOUSEMOVE:
        {
            TRACKMOUSEEVENT me{};
            me.cbSize = sizeof(TRACKMOUSEEVENT);
            me.dwFlags = TME_HOVER | TME_LEAVE;
            me.hwndTrack = hButton;
            me.dwHoverTime = HOVER_DEFAULT;
            TrackMouseEvent(&me);
            SetProp(hButton, TEXT("HIflag"), (HANDLE)1);
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }
         
        case WM_MOUSEHOVER:
        {
            SetProp(hButton, TEXT("HIflag"), (HANDLE)2);
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }
         
        case WM_MOUSELEAVE:
        {
            // tracking is cancelled automatically when this message is generated!
            SetProp(hButton, TEXT("HIflag"), (HANDLE)3);
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }

        case WM_PAINT:
        {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hButton, &ps);

            COLORREF color;
            switch ( (int) GetProp(hButton, TEXT("HIflag")) ) {
                case 1:  color = RGB(0x00, 0x00, 0xFF); break;
                case 2:  color = RGB(0xFF, 0x00, 0x00); break;
                case 3:  color = RGB(0x00, 0xFF, 0x00); break;
                default: color = RGB(0x7F, 0x7F, 0x7F); break;
            }

            HBRUSH hBrush = CreateSolidBrush(color);
            FillRect(hdc, &ps.rcPaint, hBrush);
            DeleteObject(hBrush);

            EndPaint(hButton, &ps);
            return 0;
        }
    }

    return DefSubclassProc(hButton, iMsg, wParam, lParam);
}

With that said, another way to color an owner-drawn button's background is to have the parent window handle the WM_CTLCOLORBTN message, or at least the WM_DRAWITEM message. This way, the parent window can store the button's HWND and its associated HIFlag together in an array somewhere, and not have to track the flag inside the HWND itself.

For example:

Buttons[index].hWnd = CreateWindow(L"button", L"", WS_CHILD | WS_VISIBLE | BS_OWNERDRAW, 0, 0, 0, 0, hWnd, HMENU(index+1), hInst, NULL);
Buttons[index].HIflag = 0;
SetWindowSubclass(hBtn, subSIproc, index+1, 0);

...

LRESULT CALLBACK ParentWndProc(HWND hWnd, UINT uiMsg, WPARAM wParam, LPARAM lParam)
{
    switch (uiMsg)
    {
        case WM_CTLCOLORBTN:
        {
            HWND hBtn = (HWND) lParam;

            for (int index = 0; index < NumberOfButtons; ++index)
            {
                if (Buttons[index].hWnd == hBtn)
                {
                    COLORREF color;
                    switch ( Buttons[index].HIflag ) {
                        case 1:  color = RGB(0x00, 0x00, 0xFF); break;
                        case 2:  color = RGB(0xFF, 0x00, 0x00); break;
                        case 3:  color = RGB(0x00, 0xFF, 0x00); break;
                        default: color = RGB(0x7F, 0x7F, 0x7F); break;
                    }
                    return CreateSolidBrush(color);
                }
            }

            break;
        }

        // or:

        case WM_DRAWITEM:
        {
            DRAWITEMSTRUCT *dis = (DRAWITEMSTRUCT*) lParam;

            COLORREF color;
            switch ( Buttons[dis->CtlID-1].HIflag ) {
                case 1:  color = RGB(0x00, 0x00, 0xFF); break;
                case 2:  color = RGB(0xFF, 0x00, 0x00); break;
                case 3:  color = RGB(0x00, 0xFF, 0x00); break;
                default: color = RGB(0x7F, 0x7F, 0x7F); break;
            }

            HBRUSH hBrush = CreateSolidBrush(color);
            FillRect(dis->hDC, &(dis->rcItem), hBrush);
            DeleteObject(hBrush);

            return TRUE;
        }
    }

    return DefWindowProc(hWnd, uiMsg, wParam, lParam);
}

LRESULT CALLBACK subSIproc(HWND hButton, UINT uiMsg, WPARAM wParam, LPARAM lParam, UINT_PTR uId, DWORD_PTR dwRefData)
{
    switch (uiMsg)
    {
        case WM_MOUSEMOVE:
        {
            TRACKMOUSEEVENT me{};
            me.cbSize = sizeof(TRACKMOUSEEVENT);
            me.dwFlags = TME_HOVER | TME_LEAVE;
            me.hwndTrack = hButton;
            me.dwHoverTime = HOVER_DEFAULT;
            TrackMouseEvent(&me);
            Buttons[uId-1].HIflag = 1;
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }
         
        case WM_MOUSEHOVER:
        {
            Buttons[uId-1].HIflag = 2;
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }
         
        case WM_MOUSELEAVE:
        {
            // tracking is cancelled automatically when this message is generated!
            Buttons[uId-1].HIflag = 3;
            InvalidateRect(hButton, NULL, TRUE);
            break;
        }
    }

    return DefSubclassProc(hButton, uiMsg, wParam, lParam);
}