I created an RSA Certificate with a Private Key in LocalMachine/My store.

I then created a simple Windows console app in C to:

• locate the cert by thumbprint
• encrypt a buffer with CryptEncryptMessage
• write the buffer out to a file

I then wrote a small service app, also in C to:

• read the buffer
• decrypt with CryptDecryptMessage

It all works fine as long as the service app is not running under the Local System Account (LSA). In that case, CryptDecryptMessage generates CRYPT_E_NO_DECRYPT_CERT (0x8009200C). This is despite that the Cert is in LocalMachine/My

I tried to strip down the code as much as possible but it is still rather long. I'm hoping there is some flag setting I am getting wrong. Or is this expected behavior? Thanks for taking a look.

Windows console app

int __cdecl main(int argc, CHAR* argv[])
{
    HCERTSTORE hSysStore = NULL;
    DWORD err;
    PCHAR    certThumbprint   = "14FE20556FC106DA4C561707ABDFEACEB8CAAC98";
    PCERT_CONTEXT pContext = NULL;
    BYTE*    pbEncryptedBlob = NULL;
    DWORD    cbEncryptedBlob;
    BYTE bPlain[255] = {0};
    DWORD cbContent;
    strcpy_s(bPlain, sizeof(bPlain)-1, argv[1]);
    cbContent = strlen(bPlain)+1;
    err = myGetCertFromThumb(CERT_SYSTEM_STORE_LOCAL_MACHINE, L"MY", certThumbprint, &hSysStore, &pContext);
    if (err != 0)
        goto cleanup;
    pbEncryptedBlob = malloc(BUF_SZ);
    cbEncryptedBlob = BUF_SZ;
    /**** code below ****/
    err = myEncryptMsg(pContext, bPlain, cbContent, pbEncryptedBlob, &cbEncryptedBlob);
    if (err != 0)
        goto cleanup;
    printf("Encrypted msg size=%d\n", cbEncryptedBlob);

    /****  write cbEncryptedBlob to file  ****/

cleanup:
    if (pbEncryptedBlob)
        free(pbEncryptedBlob);
    if (pContext)
        CertFreeCertificateContext(pContext);
    if (hSysStore != NULL)
        CertCloseStore(hSysStore, CERT_CLOSE_STORE_CHECK_FLAG);
    return(err);
}

DWORD myEncryptMsg(PCERT_CONTEXT pContext, BYTE* pbPlain, DWORD cbPlainszsz, BYTE* pbCipher, DWORD *pcbCiphersz)
{
    BOOL rc;
    DWORD err;
    PCCERT_CONTEXT RecipientCertArray[1];
    DWORD EncryptAlgSize;
    HCRYPTPROV hCryptProv;
    CRYPT_ALGORITHM_IDENTIFIER EncryptAlgorithm;
    CRYPT_ENCRYPT_MESSAGE_PARA EncryptParams;
    DWORD EncryptParamsSize;
    rc = CryptAcquireContext(
        &hCryptProv,        // Address for handle to be returned.
        NULL,               // Use the current user's logon name.
        NULL,               // Use the default provider.
        PROV_RSA_FULL,      // Need to both encrypt and sign.
        0);              // No flags needed.
    if (rc) 
        goto keyset_exists;
    err = GetLastError();
    if (err != NTE_BAD_KEYSET)
        goto cleanup;
    rc = CryptAcquireContext(
        &hCryptProv,        // Address for handle to be returned.
        NULL,               // Use the current user's logon name.
        NULL,               // Use the default provider.
        PROV_RSA_AES,      // Need to both encrypt and sign.
        CRYPT_NEWKEYSET);              // No flags needed.
    if (!rc) 
    {
        err = GetLastError();
        goto cleanup;
    }
keyset_exists:
    RecipientCertArray[0] = pContext;
    EncryptAlgSize = sizeof(EncryptAlgorithm);
    memset(&EncryptAlgorithm, 0, EncryptAlgSize);
    EncryptAlgorithm.pszObjId = szOID_NIST_AES256_CBC;
    EncryptParamsSize = sizeof(EncryptParams);
    memset(&EncryptParams, 0, EncryptParamsSize);
    EncryptParams.cbSize =  EncryptParamsSize;
    EncryptParams.dwMsgEncodingType = (PKCS_7_ASN_ENCODING | X509_ASN_ENCODING);
    EncryptParams.hCryptProv = hCryptProv;
    EncryptParams.ContentEncryptionAlgorithm = EncryptAlgorithm;
    rc = CryptEncryptMessage(
        &EncryptParams,
        1,
        RecipientCertArray,
        pbPlain,
        cbPlainszsz,
        pbCipher,
        pcbCiphersz);
    if (!rc)
    {
        err = GetLastError();
        goto cleanup;
    }
    err = 0;
cleanup:
    if(hCryptProv)
        CryptReleaseContext(hCryptProv,0);
    return(err);
}

Windows Service App

BYTE * someRoutineInsideService()
{
    DWORD err;
    BYTE*    pbEncryptedBlob = NULL;
    DWORD    cbEncryptedBlob;
    BYTE* pbDecryptedMessage = NULL;
    DWORD cbDecryptedMessage;
    pbEncryptedBlob = malloc(BUF_SZ);
    cbEncryptedBlob = BUF_SZ;

    /**** read pbEncryptedBlob from file ****/

    pbDecryptedMessage = malloc(BUF_SZ);
    cbDecryptedMessage = BUF_SZ;
    /**** code below ****/
    err = myDecryptMessage("MY", pbEncryptedBlob, cbEncryptedBlob, pbDecryptedMessage, &cbDecryptedMessage);
    if (err != 0)
        goto cleanup;
cleanup:
    if (pbEncryptedBlob)
        free(pbEncryptedBlob);
    return(pbDecryptedMessage);
}

DWORD myDecryptMessage(LPSTR cStore, BYTE *pbEncryptedBlob, DWORD cbEncryptedBlob, BYTE *pbDecryptedMessage, DWORD *cbDecryptedMessage)
{
    HCERTSTORE CertStoreArray[1];
    CRYPT_DECRYPT_MESSAGE_PARA  DecryptParams;
    DWORD  DecryptParamsSize = sizeof(DecryptParams);
    HCRYPTPROV hCryptProv = 0;                      // CSP handle
    HCERTSTORE hStoreHandle = 0;
    DWORD err = 0;
    BOOL rc;
    rc = CryptAcquireContext(
        &hCryptProv,        // Address for handle to be returned.
        NULL,               // Use the current user's logon name.
        NULL,               // Use the default provider.
        PROV_RSA_FULL,      // Need to both encrypt and sign.
        0);              // No flags needed.
    if (!rc)
    {
        err = GetLastError();
        goto cleanup;
    }
    hStoreHandle = CertOpenSystemStore(hCryptProv, cStore);
    if (hStoreHandle == NULL)
    {
        err = GetLastError();
        goto cleanup;
    }
    CertStoreArray[0] = hStoreHandle;
    memset(&DecryptParams, 0, DecryptParamsSize);
    DecryptParams.cbSize = DecryptParamsSize;
    DecryptParams.dwMsgAndCertEncodingType = (PKCS_7_ASN_ENCODING | X509_ASN_ENCODING);
    DecryptParams.cCertStore = 1;
    DecryptParams.rghCertStore = CertStoreArray;
    rc = CryptDecryptMessage(
              &DecryptParams,
              pbEncryptedBlob,
              cbEncryptedBlob,
              pbDecryptedMessage,
              cbDecryptedMessage,
              NULL);
    if (!rc)
    {
        err = GetLastError();    
        /**** returns 0x8009200C (CRYPT_E_NO_DECRYPT_CERT) if service run under LSA ****/
        goto cleanup;
    }
cleanup:
    if(hCryptProv)
        rc = CryptReleaseContext(hCryptProv,0);
    return(err);
}