MediaFoundation SinkWriter: Enabling MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS results in WriteSample failing with E_FAIL error

I'm trying to encode RGB/NV12 samples to h264 and stream the encoded video over WebSocket through SinkWriter by implementing IMFByteStream. For this experiment, I have converted the RGB32 samples to NV12 samples using pixel shaders. The output format is H264 with FMPEG4 container.

I have also tried directly feeding RGB samples. It works fine for both RGB and NV12 samples through the software approach, but WriteSample fails with E_FAIL error when the below-mentioned line is uncommented. I'm properly setting MF_SINK_WRITER_D3D_MANAGER though.

COM_CHECK(attribs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE));

MFtrace log:

59980,EB60 10:18:27.65002 ### Exiting: traced process has exited
CMFPlatExportDetours::MFStartup @ Version=0x00020070, dwFlags=0x00000000
59980,EB60 10:18:27.65448 COle32ExportDetours::CoCreateInstance @ Created {60F9F51E-4613-4B35-AE88-332542B567B8} MF Fragmented MPEG4 Sink Class Factory (C:\WINDOWS\System32\mfmp4srcsnk.dll) @04B80148 - traced interfaces:
59980,EB60 10:18:27.65700 COle32ExportDetours::CoCreateInstance @ Created {9A02E012-6303-4E1E-B9A1-630F802592C5} Packed Property Storage Object (C:\WINDOWS\system32\propsys.dll) @012D5FEC - traced interfaces:
59980,EB60 10:18:27.65931 COle32ExportDetours::CoCreateInstance @ Created {48E2ED0F-98C2-4A37-BED5-166312DDD83F} MFReadWrite Class Factory (C:\WINDOWS\System32\mfreadwrite.dll) @02D13F00 - traced interfaces: IMFReadWriteClassFactory @02D13F00,
59980,EB60 10:18:27.65935 CMFReadWriteClassFactoryDetours::CreateInstanceFromObject @02D13F00 Object @04B859F0, MF_TRANSCODE_CONTAINERTYPE={9BA876F1-419F-4B77-A1E0-35959D9D4004};MF_SINK_WRITER_ASYNC_CALLBACK=@05ECCD98;{9C27891A-ED7A-40E1-88E8-B22727A024EE}=1;MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS=1;MF_SOURCE_READER_D3D_MANAGER=@04B82770
59980,EB60 10:18:27.65940 CMFAttributesDetours::GetUINT32 @02D17618 attribute not found guidKey = MF_SINK_WRITER_DISABLE_THROTTLING
59980,EB60 10:18:27.65941 CMFAttributesDetours::GetUnknown @02D17618 attribute not found guidKey = {2ACF1917-3743-41DF-A564-E727A80EA33E}
59980,EB60 10:18:27.65942 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = MF_SINK_WRITER_DISABLE_THROTTLING
59980,EB60 10:18:27.65942 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = MF_READWRITE_DISABLE_CONVERTERS
59980,EB60 10:18:27.65943 CMFAttributesDetours::GetUINT32 @02D17618 attribute not found guidKey = {BDAD7BCA-0E5F-4B10-AB16-26DE381B6293}
59980,EB60 10:18:27.65943 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = {39384300-D0EB-40B1-87A0-3318871B5A53}
59980,EB60 10:18:27.65944 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = {430847DA-0890-4B0E-938C-054332C547E1}
59980,EB60 10:18:27.65944 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = {43AD19CE-F33F-4BA9-A580-E4CD12F2D144}
59980,EB60 10:18:27.65945 CMFAttributesDetours::GetItemType @02D17618 attribute not found guidKey = {273DB885-2DE2-4DB2-A6A7-FDB66FB40B61}
59980,EB60 10:18:27.65945 CMFAttributesDetours::GetString @02D17618 attribute not found guidKey = {39384300-D0EB-40B1-87A0-3318871B5A53}
59980,EB60 10:18:27.65946 CMFAttributesDetours::GetUINT32 @02D17618 attribute not found guidKey = {43AD19CE-F33F-4BA9-A580-E4CD12F2D144}
### BuffersWritten : 7
59980,EB60 10:18:27.65959 ### Trace session stopped
CMFReadWriteClassFactoryDetours::HandleObject @ New sink writer @04B890F8
59980,EB60 10:18:27.65970 CMFAttributesDetours::GetUnknown @04B8A1F8 attribute not found guidKey = MFT_FIELDOFUSE_UNLOCK_Attribute
           __M_F_T_R_A_C_E___LOG__
Total events received: 25

Initialize:

CComPtr<IMFAttributes> attribs;
CComPtr<IMFDXGIDeviceManager> pDeviceManager = NULL;
UINT resetToken = 0;
CComPtr<ID3D11Device> pD3dDevice = NULL;
CComQIPtr<ID3D10Multithread> pMultithread = nullptr;

VideoEncoderMF (std::array<unsigned short, 2> dimensions, unsigned int fps, IMFByteStream * stream, ID3D11Device* D3dDevice = NULL) : VideoEncoderMF(dimensions, fps) {
const unsigned int bit_rate = static_cast<unsigned int>(0.78f*fps*m_width*m_height); // yields 40Mb/s for 1920x1080@25fps

  CComPtr<IMFAttributes> attribs;
  COM_CHECK(MFCreateAttributes(&attribs, 0));

  pD3dDevice = D3dDevice;
  pMultithread = pD3dDevice;

  if (pMultithread) {
      pMultithread->SetMultithreadProtected(TRUE);
  }

  if (pD3dDevice) {
      MFCreateDXGIDeviceManager(&resetToken, &pDeviceManager);
  }

  COM_CHECK(attribs->SetGUID(MF_TRANSCODE_CONTAINERTYPE, MFTranscodeContainerType_FMPEG4));
  COM_CHECK(attribs->SetUINT32(MF_LOW_LATENCY, TRUE));

  //COM_CHECK(attribs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE));// un-commenting this leads WriteSample to fail with error E_FAIL

  if (pDeviceManager) {
      COM_CHECK(attribs->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, pDeviceManager));
  }

  // create sink writer with specified output format
  IMFMediaTypePtr mediaTypeOut = MediaTypeutput(fps, bit_rate);
  COM_CHECK(MFCreateFMPEG4MediaSink(stream, mediaTypeOut, nullptr, &m_media_sink)); // "fragmented" MPEG4 does not require seekable byte-stream
  COM_CHECK(MFCreateSinkWriterFromMediaSink(m_media_sink, attribs, &m_sink_writer));

  // connect input to output
  IMFMediaTypePtr mediaTypeIn = MediaTypeInput(fps);
  COM_CHECK(m_sink_writer->SetInputMediaType(m_stream_index, mediaTypeIn, nullptr));

  COM_CHECK(m_sink_writer->BeginWriting());
}

Configure input media type:

IMFMediaTypePtr MediaTypeInput (unsigned int fps) {
    IMFMediaTypePtr mediaTypeIn;
    COM_CHECK(MFCreateMediaType(&mediaTypeIn));
    COM_CHECK(mediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
    COM_CHECK(mediaTypeIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12));
    COM_CHECK(mediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive));
    COM_CHECK(mediaTypeIn->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE));

    LONG stride = 0;
    COM_CHECK(MFGetStrideForBitmapInfoHeader(MFVideoFormat_NV12.Data1, m_width, &stride));
    COM_CHECK(mediaTypeIn->SetUINT32(MF_MT_DEFAULT_STRIDE, UINT32(stride)));

    UINT32 vidSampleSize = (UINT32)(m_height * stride + (m_height >> 1)* stride);
    COM_CHECK(mediaTypeIn->SetUINT32(MF_MT_SAMPLE_SIZE, vidSampleSize));        //Set the new sample size

    COM_CHECK(MFSetAttributeSize(mediaTypeIn, MF_MT_FRAME_SIZE, m_width, m_height));
    COM_CHECK(MFSetAttributeRatio(mediaTypeIn, MF_MT_FRAME_RATE, fps, 1));
    COM_CHECK(MFSetAttributeRatio(mediaTypeIn, MF_MT_PIXEL_ASPECT_RATIO, 1, 1));

    return mediaTypeIn;
}

Configure output media type:

IMFMediaTypePtr MediaTypeutput (unsigned int fps, unsigned int bit_rate) {
  IMFMediaTypePtr mediaTypeOut;
  COM_CHECK(MFCreateMediaType(&mediaTypeOut));
  COM_CHECK(mediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
  COM_CHECK(mediaTypeOut->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264)); // H.264 format
  COM_CHECK(mediaTypeOut->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High));
  COM_CHECK(mediaTypeOut->SetUINT32(MF_MT_MPEG2_LEVEL, eAVEncH264VLevel5_1));
  COM_CHECK(mediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE, bit_rate));
  COM_CHECK(mediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive));
  COM_CHECK(MFSetAttributeSize(mediaTypeOut, MF_MT_FRAME_SIZE, m_width, m_height));
  COM_CHECK(MFSetAttributeRatio(mediaTypeOut, MF_MT_FRAME_RATE, fps, 1));
  COM_CHECK(MFSetAttributeRatio(mediaTypeOut, MF_MT_PIXEL_ASPECT_RATIO, 1, 1));

  return mediaTypeOut;
}

Write Sample:

HRESULT WriteSample(CComPtr<IMFSample> &sample) override {

  // Set the time stamp and the duration.
  COM_CHECK(sample->SetSampleTime(m_time_stamp));
  COM_CHECK(sample->SetSampleDuration(m_frame_duration));

  // send sample to Sink Writer.
  HRESULT hr = m_sink_writer->WriteSample(m_stream_index, sample); // fails on I/O error
  if (FAILED(hr))
      return hr;

  // increment time
  m_time_stamp += m_frame_duration;

  return S_OK;
}

I'm puzzled with this behavior of SinkWriter and unable to solve the problem for quite some time. Any help would be appreciated.