Capturing video when screen resolution is above 1920x1080

I am trying to write a Win32 c++ program which uses Media Foundation, for capturing mp4 videos of the desktop. I found that on a PC with screen resolution of over 1920x1080, capturing fails. The failure is following a call to SetInputMediaType() which returns error 0xc00d36b4 (The data specified for the media type is invalid, inconsistent, or not supported by this object)

Any solution?

HRESULT InitializeDirect3D9(IDirect3DDevice9** ppDevice, IDirect3DSurface9** ppSurface, UINT32& uiWidth, UINT32& uiHeight) 
{

IDirect3D9* d3d = NULL;

d3d = Direct3DCreate9(D3D_SDK_VERSION);

if (d3d == NULL)
    return E_POINTER;

D3DDISPLAYMODE mode;
HRESULT hr = d3d->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &mode);

if (FAILED(hr))
{
    SafeRelease(&d3d);
    return hr;
}

D3DPRESENT_PARAMETERS parameters = { 0 };

parameters.Windowed = TRUE;
parameters.BackBufferCount = 1;
uiHeight = parameters.BackBufferHeight = mode.Height;
uiWidth = parameters.BackBufferWidth = mode.Width;
parameters.SwapEffect = D3DSWAPEFFECT_DISCARD;
parameters.hDeviceWindow = NULL;

hr = d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, NULL, D3DCREATE_SOFTWARE_VERTEXPROCESSING, &parameters, ppDevice);

if (FAILED(hr))
{
    SafeRelease(&d3d);
    return hr;
}

hr = (*ppDevice)->CreateOffscreenPlainSurface(mode.Width, mode.Height, D3DFMT_A8R8G8B8, D3DPOOL_SYSTEMMEM, ppSurface, nullptr);

SafeRelease(&d3d);

return hr;
}

HRESULT SGCaptureDecktopVideo::InitializeSinkWriter(std::wstring VideoPath, IMFSinkWriter **ppWriter, DWORD *pStreamIndex, const UINT32 uiWidth, const UINT32 uiHeight)
{
*ppWriter = NULL;
*pStreamIndex = NULL;

IMFSinkWriter   *pSinkWriter = NULL;
IMFMediaType    *pMediaTypeOut = NULL;
IMFMediaType    *pMediaTypeIn = NULL;
DWORD           streamIndex;

HRESULT hr = MFCreateSinkWriterFromURL(VideoPath.c_str(), NULL, NULL, &pSinkWriter);

// Set the output media type.
if (SUCCEEDED(hr))
{
    hr = MFCreateMediaType(&pMediaTypeOut);
}
if (SUCCEEDED(hr))
{
    hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
}
if (SUCCEEDED(hr))
{
    hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE, myVideoRecorder.CaptureParams.VIDEO_ENCODING_FORMAT);
}
�
void vidThread(vidThreadArgs *args)
{

WriteLogFile(L"Thread started");
HRESULT hr = CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);

if (SUCCEEDED(hr))
{
    WriteLogFile(L"CoInitializeEx - success");

    hr = MFStartup(MF_VERSION);

    if (SUCCEEDED(hr))
    {
        WriteLogFile(L"MFStartup - success");

        UINT32 uiWidth = 0;
        UINT32 uiHeight = 0;

        IDirect3DDevice9* pDevice = NULL;
        IDirect3DSurface9* pSurface = NULL;


        hr = myVideoRecorder.InitializeDirect3D9(&pDevice, &pSurface, uiWidth, uiHeight);

        if (SUCCEEDED(hr))
        {
            WriteLogFile(L"InitializeDirect3D9 - success");

            IMFSinkWriter *pSinkWriter = NULL;
            DWORD stream;

            hr = myVideoRecorder.InitializeSinkWriter(myVideoRecorder.CaptureParams.VideoPath, &pSinkWriter, &stream, uiWidth, uiHeight);

            if (SUCCEEDED(hr))
            {

                WriteLogFile(L"InitializeSinkWriter - success");

                LONGLONG rtStart = 0;
                unsigned long long int frameCounter = 0;

                while (true)
                {

                    hr = myVideoRecorder.WriteFrame(pDevice, pSurface, pSinkWriter, stream, rtStart, uiWidth, uiHeight);
                    frameCounter++;

                    if (FAILED(hr))
                    {

                        WriteLogFile(L"Internal error %d Frame %d\n",
                            GetLastError(), frameCounter);

                        break;
                    }

                    rtStart += myVideoRecorder.CaptureParams.VIDEO_FRAME_DURATION;
                    if (bStopRecording)
                    {
                        WriteLogFile(L"Finalizing recording of file %s. Frame %llu",
                            myVideoRecorder.CaptureParams.VideoPath.c_str(),
                            frameCounter);

                        break;
                    }
                }


            }
            else
            {

                DWORD error = GetLastError();
                WriteLogFile(L"failed here. Error %d\n", error);
            }

            if (SUCCEEDED(hr))
            {
                hr = pSinkWriter->Finalize();
            }

            SafeRelease(&pSinkWriter);
        }
        else
        {
            WriteLogFile(L"InitializeDirect3D9 - failed Error %d",GetLastError());
            result = STATUS_ERROR_INIT_DIRECT3D9;
        }
        SafeRelease(&pDevice);
        SafeRelease(&pSurface);
        MFShutdown();
        WriteLogFile(L"MFShutDown");
    }
    else
    {
        result = STATUS_ERROR_MFSSTARTUP;
        WriteLogFile(L"MFStartup failed");

    }
    CoUninitialize();
}
else
{
    result = STATUS_ERROR_COINITILIZE;
    WriteLogFile(L"CoInitilize - failed Error %d", GetLastError());
}



HRESULT WriteFrame(IDirect3DDevice9* pDevice, IDirect3DSurface9*
pSurface, IMFSinkWriter* pWriter, DWORD streamIndex, const
LONGLONG& rtStart, const UINT32 uiWidth, const UINT32 uiHeight){

    HRESULT hr = pDevice->GetFrontBufferData(0, pSurface);

    if(FAILED(hr)){
        return hr;
    }

    D3DLOCKED_RECT rc;
    hr = pSurface->LockRect(&rc, NULL, 0);

    if(FAILED(hr)){
        return hr;
    }

    IMFSample *pSample = NULL;
    IMFMediaBuffer *pBuffer = NULL;

    const LONG cbWidth = 4 * uiWidth;
    const DWORD cbBuffer = cbWidth * uiHeight;

    BYTE *pData = NULL;

    // Create a new memory buffer.
    hr = MFCreateMemoryBuffer(cbBuffer, &pBuffer);

    // Lock the buffer and copy the video frame to the buffer.
    if(SUCCEEDED(hr)){
        hr = pBuffer->Lock(&pData, NULL, NULL);
    }

    if(SUCCEEDED(hr)){

#ifdef REVERSE_IMAGE
        for(int i = 0, j = uiHeight - 1; i < uiHeight; i++, j--)
            for(int k = 0; k < cbWidth; k++)
                    pData[(i * cbWidth) + k] = ((BYTE*)rc.pBits)[(j *
cbWidth) + k];
#else
        hr = MFCopyImage(pData, cbWidth, (BYTE*)rc.pBits, rc.Pitch,
cbWidth, uiHeight);
#endif
    }

    if(pBuffer){
        pBuffer->Unlock();
    }

    // Set the data length of the buffer.
    if(SUCCEEDED(hr)){
        hr = pBuffer->SetCurrentLength(cbBuffer);
    }

    // Create a media sample and add the buffer to the sample.
    if(SUCCEEDED(hr)){
        hr = MFCreateSample(&pSample);
    }

    if(SUCCEEDED(hr)){
        hr = pSample->AddBuffer(pBuffer);
    }

    // Set the time stamp and the duration.
    if(SUCCEEDED(hr)){
        hr = pSample->SetSampleTime(rtStart);
    }

    if(SUCCEEDED(hr)){
        hr = pSample->SetSampleDuration(VIDEO_FRAME_DURATION);
    }

    // Send the sample to the Sink Writer.
    if(SUCCEEDED(hr)){
        hr = pWriter->WriteSample(streamIndex, pSample);
    }

    hr = pSurface->UnlockRect();

    SafeRelease(&pSample);
    SafeRelease(&pBuffer);
    return hr;
}

Code was tested on 5 different machines, all with Windows 10 x64, and found out that if I store the values of uiWidth and uiHeight in a log file, just before the following line:

hr = MFSetAttributeSize(pMediaTypeOut, MF_MT_FRAME_SIZE, uiWidth, uiHeight);

The code works perfectly when their values are bellow 1220 and 1080, and they get 0 sized video files when either (or both) of these values exceed 1220x1080.

Update: I am also publishing InitializeSinkWriter(). At Step 17, SetInputMediaType() fails with error c00d36b4.

HRESULT SGCaptureDecktopVideo::InitializeSinkWriter(std::wstring VideoPath, IMFSinkWriter **ppWriter, DWORD *pStreamIndex, const UINT32 uiWidth, const UINT32 uiHeight)
{
    *ppWriter = NULL;
    *pStreamIndex = NULL;

    IMFSinkWriter   *pSinkWriter = NULL;
    IMFMediaType    *pMediaTypeOut = NULL;
    IMFMediaType    *pMediaTypeIn = NULL;
    DWORD           streamIndex;

    // Step 1 - MFCreateSinkWriterFromURL
    HRESULT hr = MFCreateSinkWriterFromURL(VideoPath.c_str(), NULL, NULL, &pSinkWriter);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 1 - MFCreateSinkWriterFromURL() failed. %x", hr);
        goto cleanup;
    }

    // Step 2 - MFCreateMediaType
    hr = MFCreateMediaType(&pMediaTypeOut);
    if(SUCCEEDED(hr)==0)
    {
        WriteLogFile(L"Step 2 - MFCreateMediaType() failed. %x", hr);
        goto cleanup;
    }

    // Step 3 - SetGUID (MF_MT_MAJOR_TYPE)
    hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
    if(SUCCEEDED(hr)==0)
    {
        WriteLogFile(L"Step 3 - SetGUID (MF_MT_MAJOR_TYPE) failed. %x", hr);
        goto cleanup;
    }

    // Step 4 - SetGUID (MF_MT_SUBTYPE)
    hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE, myVideoRecorder.CaptureParams.VIDEO_ENCODING_FORMAT);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 4 - SetGUID (MF_MT_SUBTYPE) failed. %x", hr);
        goto cleanup;
    }

    // Step 5 - SetUINT32 (MF_MT_AVG_BITRATE)
    hr = pMediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE, myVideoRecorder.CaptureParams.VIDEO_BIT_RATE);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 5 - SetUINT32 (MF_MT_AVG_BITRATE) failed. %x", hr);
        goto cleanup;
    }

    // Step 6 - SetUINT32(MF_MT_INTERLACE_MODE)
    hr = pMediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 6 - SetUINT32(MF_MT_INTERLACE_MODE) failed. %x", hr);
        goto cleanup;
    }

    // Step 7 - MFSetAttributeSize
    WriteLogFile(L"w %d h %d",uiWidth, uiHeight);
    hr = MFSetAttributeSize(pMediaTypeOut, MF_MT_FRAME_SIZE, uiWidth, uiHeight);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 7 - MFSetAttributeSize() failed. %x", hr);
        goto cleanup;
    }

    // Step 8 - MFSetAttributeRatio - MF_MT_FRAME_RATE
    hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_FRAME_RATE, myVideoRecorder.CaptureParams.VIDEO_FPS, 1);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 8 - MFSetAttributeRatio(MF_MT_FRAME_RATE) failed. %x", hr);
        goto cleanup;
    }

    // Step 9 - MFSetAttributeRatio - MF_MT_PIXEL_ASPECT_RATIO
    hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 9 - MFSetAttributeRatio(MF_MT_PIXEL_ASPECT_RATIO) failed. %x", hr);
        goto cleanup;
    }

    // Step 10 - AddStream
    hr = pSinkWriter->AddStream(pMediaTypeOut, &streamIndex);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 10 - AddStream() failed. %x", hr);
        goto cleanup;
    }

    // Step 11 - MFCreateMediaType
    hr = MFCreateMediaType(&pMediaTypeIn);
    if (SUCCEEDED(hr) == 0 || pMediaTypeIn==NULL)
    {
        WriteLogFile(L"Step 11 - MFCreateMediaType() failed. %x", hr);
        goto cleanup;
    }

    // Step 12 - SetGUID (MF_MT_MAJOR_TYPE)
    hr = pMediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 12 - SetGUID (MF_MT_MAJOR_TYPE) failed. %x", hr);
        goto cleanup;
    }

    // Step 13 - SetGUID (MF_MT_SUBTYPE)
    hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, myVideoRecorder.CaptureParams.VIDEO_INPUT_FORMAT);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 13 - SetGUID (MF_MT_SUBTYPE) failed. %x", hr);
        goto cleanup;
    }

    // Step 13 - SetUINT32 (MF_MT_INTERLACE_MODE)
    hr = pMediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"SetUINT32 (MF_MT_INTERLACE_MODE). %x", hr);
        goto cleanup;
    }

    // Step 14 - SMFSetAttributeSize
    hr = MFSetAttributeSize(pMediaTypeIn, MF_MT_FRAME_SIZE, uiWidth, uiHeight);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 14 - MFSetAttributeSize() failed. %x", hr);
        goto cleanup;
    }

    // Step 15 - MFSetAttributeRatio(MF_MT_FRAME_RATE)
    hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_FRAME_RATE, myVideoRecorder.CaptureParams.VIDEO_FPS, 1);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 15 - MFSetAttributeRatio(MF_MT_FRAME_RATE) failed. %x", hr);
        goto cleanup;
    }

    // Step 16 - MFSetAttributeRatio(MF_MT_PIXEL_ASPECT_RATIO)
    hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 16 - MFSetAttributeRatio(MF_MT_PIXEL_ASPECT_RATIO) failed. %x", hr);
        goto cleanup;
    }
    // Step 17 - SetInputMediaType
    hr = pSinkWriter->SetInputMediaType(streamIndex, pMediaTypeIn, NULL);
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 17 - SetInputMediaType() failed. %x", hr);
        goto cleanup;
    }

    // Step 18 - BeginWriting() Tell the sink writer to start accepting data.
    hr = pSinkWriter->BeginWriting();
    if (SUCCEEDED(hr) == 0)
    {
        WriteLogFile(L"Step 18 - BeginWriting() failed. %x", hr);
        goto cleanup;
    }
    // Return the pointer to the caller.
    WriteLogFile(L"Success");

    *ppWriter = pSinkWriter;
    (*ppWriter)->AddRef();
    *pStreamIndex = streamIndex;

cleanup:;
    SafeRelease(&pSinkWriter);
    SafeRelease(&pMediaTypeOut);
    SafeRelease(&pMediaTypeIn);
    return hr;
}

I still get an error in this part on machines with a higher resolution. CreateDevice() fails with error 0x7E.

D3DDISPLAYMODE mode;

HRESULT hr = d3d->GetAdapterDisplayMode(D3DADAPTER_DEFAULT, &mode);

if (FAILED(hr))
{
    WriteLogFile(L"GetAdapterDisplayMode() failed. Error =%x", GetLastError());
    SafeRelease(&d3d);
    return hr;
}
WriteLogFile(L"Fetched mode. Width=%d Height=%d", mode.Width, mode.Height);
D3DPRESENT_PARAMETERS parameters = { 0 };

parameters.Windowed = TRUE;
parameters.BackBufferCount = 1;
uiHeight = parameters.BackBufferHeight = mode.Height;
uiWidth = parameters.BackBufferWidth = mode.Width;
parameters.SwapEffect = D3DSWAPEFFECT_DISCARD;
parameters.hDeviceWindow = NULL;

hr = d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, NULL, D3DCREATE_SOFTWARE_VERTEXPROCESSING, &parameters, ppDevice);

if (FAILED(hr))
{
    WriteLogFile(L"CreateDevice failed. Error =%x",GetLastError());
    SafeRelease(&d3d);
    return hr;
}

First of all, the error handling in this code is problematic. If will simply skip operations if anything fails, but never actually reports any failures. Almost certainly one of the API calls is failing, and you need to actually report the error at that point and display the hex value (easiest to read and look up) of hr, and bail. So you need to fix up the error reporting and tell us which API is failing, and what the result is.

As to why it could be failing, codecs like MP4 have encoding profiles, and they prefer to work with certain resolutions or bitrates. It's possible that the higher resolution exceeded the limit for that profile, or it is a non-standard resolution that MP4 doesn't support (or have a profile for). Or that the profile is bitrate-limited, and the higher resolution requested exceeds the bitrate available.