How to use diffie Hellman Sessionkey as password for AES Encryption

-1

I need to create a server and client in c++ which exchange Diffie Hellman public key and encryption with AES_256 so far I am using MSDN sample for DH public key Generatinghttps://docs.microsoft.com/en-us/windows/win32/seccrypto/diffie-hellman-keys and its fine with RC4 in both side (client and server) but after Converting sample to AES_256 I get error 0x80090005(NET_BAD_DATA) on client-side EncryptDecrypt API.strange part is if both client and server runs on the same machine (not the same OS) result is OK . the code I am using for Converting Public key to AES password is added below for both server and client. my question is:

  1. am i doing correct in this (Converting Public key to AES) way or not?

  2. why the result is Ok in my host machine but if i move client to other VM the error (Net_BAD_DATA) occurs?

*I removed API result testing parts from code every API call is tested in original code no error in any API.

any help would be appreciated.

server side:

    CryptImportKey(hProvParty1,pbKeyBlob2,dwDataLen2,hPrivateKey1,0,&hSessionKey2);
    DWORD dwpassLength = 32;
    CryptAcquireContext(&hCryptProv,NULL,MS_ENH_RSA_AES_PROV,PROV_RSA_AES,0);
    CryptCreateHash(hCryptProv, CALG_SHA_256,0, 0,&hHash);
    CryptHashData(hHash,(BYTE*)hSessionKey2,dwpassLength,0);
    hKey = (HCRYPTKEY )(malloc(100));
    CryptDeriveKey(hCryptProv,CALG_AES_128,hHash,CRYPT_EXPORTABLE,&hKey);
    DWORD dwLength = sizeof(g_rgbData);
    CryptEncrypt(   hKey,0,TRUE,0,NULL, &dwLength,sizeof(g_rgbData));       
    DWORD dwpbdataLength = dwLength;
    BYTE * pbEncryptedData = (PBYTE)malloc(dwpbdataLength);     
    memcpy(pbEncryptedData, g_rgbData, sizeof(g_rgbData));
    dwLength = sizeof(g_rgbData);       
    CryptEncrypt(hKey,NULL, TRUE,0, pbEncryptedData,&dwLength,dwpbdataLength); 
    send(newsocket, (const char*)pbEncryptedData, dwLength, 0);

CLient:

CryptImportKey(hProvParty1,pbKeyBlob2,dwDataLen2,hPrivateKey1,0,&hSessionKey2); 
BYTE * pbEncryptedData = (PBYTE)malloc(1024);
recv(ConnectSocket, (char *)pbEncryptedData, DEFAULT_BUFLEN, 0);//receiving encrypted data
DWORD dwpassLength = 32;
CryptAcquireContext(&hCryptProv,NULL,MS_ENH_RSA_AES_PROV,PROV_RSA_AES,0);
CryptCreateHash(hCryptProv,CALG_SHA_256,0,0,&hHash);
CryptHashData(hHash,(BYTE*)hSessionKey2,dwpassLength,0);
CryptDeriveKey(hCryptProv,CALG_AES_256,hHash,CRYPT_EXPORTABLE,&hKey);
CryptDecrypt(hKey,0,TRUE,0, pbEncryptedData,&dlength);
c++
aes
diffie-hellman
wincrypt
asked on Stack Overflow Dec 9, 2019 by deimen • edited Dec 16, 2019 by deimen

1 Answer

0

According to the page https://docs.microsoft.com/en-us/windows/win32/seccrypto/base-provider-algorithms, CALG_AES_256 is not a supported algorithm for this call.

answered on Stack Overflow Dec 12, 2019 by stark

User contributions licensed under CC BY-SA 3.0