CryptDecrypt fails with NT_BAD_DATA (0x80090005)

0

The following code by Alejandro Magencio generate key pairs, encrypts and decrypts files with the Microsoft CryptoAPI works well, but only for files smaller than the key: http://blogs.msdn.com/b/alejacma/archive/2008/01/28/how-to-generate-key-pairs-encrypt-and-decrypt-data-with-cryptoapi.aspx

I am trying to extend this code in order to encrypt and decrypt larger files.

Here is the complete listing as far as I got. It encrypts but on the second time in the decrypt loop I get a bad data error.

Any help will be greatly appreciated and I promise to post the full working code.

Thanks in advance!

/*
    EncryptDecrypt.c

    This program was compiled on Windows 7 64-bit with Visual Studio 2013 Desktop Express. To run:

    EncryptDecrypt.exe k C:\temp\myprivate.key C:\temp\mypublic.key

    EncryptDecrypt.exe e C:\temp\myprivate.key C:\temp\todo.txt C:\temp\done.txt

    EncryptDecrypt.exe d C:\temp\mypublic.key C:\temp\done.txt C:\temp\redone.txt  
*/
#include "stdio.h"
#include "conio.h"
#include "windows.h"
#include "wincrypt.h"
#include "tchar.h"

#define ENCRYPTEDLENGTH (128)
#define MAXSIZE (128 - 11) // size of chunk we can use for this program

// FUNCTIONS
int Keys(_TCHAR* strPublicKeyFile, _TCHAR* strPrivateKeyFile);
int Encrypt(_TCHAR* strPublicKeyFile, _TCHAR* strPlainFile, _TCHAR* strEncryptedFile);
int Decrypt(_TCHAR* strPrivateKeyFile, _TCHAR* strEncryptedFile, _TCHAR* strPlainFile);

// Main
int _tmain(int argc, _TCHAR* argv[])
{
    int iResult = 0;

    if ((argc == 4) && (_tcscmp(argv[1], _T("k")) == 0))
    {
        // Generate a new key pair
        iResult = Keys(argv[2], argv[3]);
    }
    else if ((argc == 5) && (_tcscmp(argv[1], _T("e")) == 0))
    {
        // Encrypt
        iResult = Encrypt(argv[2], argv[3], argv[4]);
    }
    else if ((argc == 5) && (_tcscmp(argv[1], _T("d")) == 0))
    {
        // Decrypt
        iResult = Decrypt(argv[2], argv[3], argv[4]);
    }
else
 {
        // Show usage
        _tprintf(_T("Usage:\n"));
        _tprintf(_T("   - New key pair: EncryptDecrypt k public_key_file private_key_file\n"));
        _tprintf(_T("   - Encrypt:      EncryptDecrypt e public_key_file plain_file encrypted_file\n"));
        _tprintf(_T("   - Decrypt:      EncryptDecrypt d private_key_file encrypted_file plain_file\n"));
        iResult = 1;
    }

    _tprintf(_T("\n<< Press any key to continue >>\n"));
    _getch();

    return iResult;
}
// End of Main

// Keys
int Keys(_TCHAR* strPublicKeyFile, _TCHAR* strPrivateKeyFile)
 {
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPublicKeyLen = 0;
    DWORD dwPrivateKeyLen = 0;
    BYTE* pbPublicKey = NULL;
    BYTE* pbPrivateKey = NULL;
    HANDLE hPublicKeyFile = NULL;
    HANDLE hPrivateKeyFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    __try
    {
        // Acquire access to key container
        _tprintf(_T("CryptAcquireContext...\n"));
        if (!CryptAcquireContext(&hCryptProv, _T("ACMEENCRYPT.EncryptDecrypt"), NULL, PROV_RSA_FULL, 0))
        {
            // Error
            _tprintf(_T("(ref 12) CryptAcquireContext error 0x%x\n"), GetLastError());

            // Try to create a new key container
            if (!CryptAcquireContext(&hCryptProv, _T("ACMEENCRYPT.EncryptDecrypt"), NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET))
            {
                // Error
                _tprintf(_T("(ref 13) CryptAcquireContext error 0x%x\n"), GetLastError());
                return 1;
            }
        }

        // Generate new key pair
        _tprintf(_T("CryptGenKey...\n"));
        if (!CryptGenKey(hCryptProv, AT_KEYEXCHANGE, CRYPT_ARCHIVABLE, &hKey))
        {
            // Error
            _tprintf(_T("CryptGenKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get public key size
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PUBLICKEYBLOB, 0, NULL, &dwPublicKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the public key
        _tprintf(_T("malloc...\n"));
        if (!(pbPublicKey = (BYTE *)malloc(dwPublicKeyLen)))
        {
            // Error
            _tprintf(_T("(ref 29) malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get public key
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PUBLICKEYBLOB, 0, pbPublicKey, &dwPublicKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get private key size
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PRIVATEKEYBLOB, 0, NULL, &dwPrivateKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the private key
        _tprintf(_T("malloc...\n"));
        if (!(pbPrivateKey = (BYTE *)malloc(dwPrivateKeyLen)))
    {
            // Error
        _tprintf(_T("(ref 30) malloc error 0x%x\n"), GetLastError());
        return 1;
    }

        // Get private key
        _tprintf(_T("CryptExportKey...\n"));
        if (!CryptExportKey(hKey, NULL, PRIVATEKEYBLOB, 0, pbPrivateKey, &dwPrivateKeyLen))
        {
            // Error
            _tprintf(_T("CryptExportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the public key
        _tprintf(_T("CreateFile...\n"));
        if ((hPublicKeyFile = CreateFile(
            strPublicKeyFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 21) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Write the public key to the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPublicKeyFile,
            (LPCVOID)pbPublicKey,
            dwPublicKeyLen,
            &lpNumberOfBytesWritten,
            NULL
            ))
        {
            // Error
            _tprintf(_T("(ref 51) WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a file to save the private key
        _tprintf(_T("CreateFile...\n"));
        if ((hPrivateKeyFile = CreateFile(
            strPrivateKeyFile,
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 22) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Write the private key to the file
        _tprintf(_T("WriteFile...\n"));
        if (!WriteFile(
            hPrivateKeyFile,
            (LPCVOID)pbPrivateKey,
            dwPrivateKeyLen,
            &lpNumberOfBytesWritten,
            NULL
            ))
        {
            // Error
            _tprintf(_T("(ref 52) WriteFile error 0x%x\n"), GetLastError());
            return 1;
        }

        return 0;
    }
    __finally
    {
        // Clean up       
        if (!pbPublicKey) {
            _tprintf(_T("free...\n"));
            free(pbPublicKey);
        }
        if (!pbPrivateKey) {
            _tprintf(_T("free...\n"));
            free(pbPrivateKey);
        }
        if (hPublicKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPublicKeyFile);
        }
        if (hPrivateKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPrivateKeyFile);
        }
        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }
        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Keys

// Encrypt
int Encrypt(_TCHAR* strPublicKeyFile, _TCHAR* strPlainFile, _TCHAR* strEncryptedFile)
{
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPublicKeyLen = 0;
    DWORD dwDataLen = 0;
    DWORD dwEncryptedLen = 0;
    DWORD myloopcount = 0;
    DWORD mymodulus = 0;
    DWORD i = 0;
    DWORD myencryptedlength = 0;
    DWORD mymaxsize = MAXSIZE;
    BYTE* pbPublicKey = NULL;
    BYTE* pbData = NULL;
    BYTE* pbEncData = NULL;
    HANDLE hPublicKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    __try
    {
        // Acquire access to key container
        _tprintf(_T("CryptAcquireContext...\n"));
        if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
        {
            // Error
            _tprintf(_T("(ref 9) CryptAcquireContext error 0x%x\n"), GetLastError());
            return 1;
        }

        // Open public key file
        _tprintf(_T("CreateFile...\n"));
        if ((hPublicKeyFile = CreateFile(
            strPublicKeyFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 14) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwPublicKeyLen = GetFileSize(hPublicKeyFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("(ref 43) GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the public key
        _tprintf(_T("malloc...\n"));
        if (!(pbPublicKey = (BYTE *)malloc(dwPublicKeyLen)))
        {
            // Error
            _tprintf(_T("(ref 22) malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Read public key
        _tprintf(_T("ReadFile...\n"));
        if (!ReadFile(hPublicKeyFile, pbPublicKey, dwPublicKeyLen, &dwPublicKeyLen, NULL))
        {
            // Error
            _tprintf(_T("(ref 31) ReadFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Import public key
        _tprintf(_T("CryptImportKey...\n"));
        if (!CryptImportKey(hCryptProv, pbPublicKey, dwPublicKeyLen, 0, 0, &hKey))
        {
            // Error
            _tprintf(_T("(ref 38) CryptImportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Open plain text file
        _tprintf(_T("CreateFile...\n"));
        if ((hPlainFile = CreateFile(
            strPlainFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 15) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwDataLen = GetFileSize(hPlainFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("(ref 40) GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        _tprintf(_T("my data length is %d\n"), dwDataLen);
        _tprintf(_T("my max size is %d\n"), MAXSIZE);

        // Get length for encrypted data
        if (!CryptEncrypt(hKey, NULL, TRUE, 0, NULL, &dwEncryptedLen, 0))
        {
            // Error
            _tprintf(_T("(ref 1) CryptEncrypt error 0x%x\n"), GetLastError());
            return 1;
        }

        if (dwDataLen < MAXSIZE)
        {
            // as before... Just one run through.

            // BEGIN OLD BLOCK
            // Create a buffer for the plain text
            _tprintf(_T("malloc...\n"));
            if (!(pbData = (BYTE *)malloc(dwDataLen)))
            {
                // Error
                _tprintf(_T("(ref 23) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            // Read plain text
            _tprintf(_T("ReadFile...\n"));
            if (!ReadFile(hPlainFile, pbData, dwDataLen, &dwDataLen, NULL))
            {
                // Error
                _tprintf(_T("(ref 32) ReadFile error 0x%x\n"), GetLastError());
                return 1;
            }

            // Next line: dwEncryptedLen is the length of key! Ergo, when decrypting
            //  use 128 instead of MAXSIZE:
            _tprintf(_T("My encrypted length is %d\n"), dwEncryptedLen); 

            // Create a buffer for encrypted data
            _tprintf(_T("realloc...\n"));
            if (!(pbData = (BYTE *)realloc(pbData, dwEncryptedLen)))
            {
                // Error
                _tprintf(_T("(ref 24) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            // Encrypt data
            if (!CryptEncrypt(hKey, NULL, TRUE, 0, pbData, &dwDataLen, dwEncryptedLen))
            {
                // Error
                _tprintf(_T("(ref 2) CryptEncrypt error 0x%x\n"), GetLastError());
                return 1;
            }

            // Create a file to save the encrypted data
            _tprintf(_T("CreateFile...\n"));
            if ((hEncryptedFile = CreateFile(
                strEncryptedFile,
                GENERIC_WRITE,
                0,
                NULL,
                CREATE_ALWAYS,          // will truncate if already existing
                FILE_ATTRIBUTE_NORMAL,
                NULL
                )) == INVALID_HANDLE_VALUE)
            {
                // Error
                _tprintf(_T("(ref 16) CreateFile error 0x%x\n"), GetLastError());
                return 1;
            }

            // Write the public key to the file
            _tprintf(_T("WriteFile...\n"));
            if (!WriteFile(
                hEncryptedFile,
                (LPCVOID)pbData,
                dwDataLen,
                &lpNumberOfBytesWritten,
                NULL
                ))
            {
                // Error
                _tprintf(_T("(ref 47) WriteFile error 0x%x\n"), GetLastError());
                return 1;
            }
            // END OLD BLOCK
            return 0;
        }
        else
        {
            // File is bigger than key.

            // Figure out how many times we will need to loop.
            myloopcount = (DWORD)(dwDataLen / MAXSIZE);
            _tprintf(_T("Loop counter is %d\n"), myloopcount);

            mymodulus = dwDataLen % MAXSIZE;
            _tprintf(_T("Remainder is %d\n"), mymodulus);

            if (mymodulus == 0) // no remainder
            {
                myloopcount -= 1; // decrement by one
            }

            // Create a file to save the encrypted data with append flag.
            // The parameter for appending data to a file is FILE_APPEND_DATA in the CreateFile function. 
            // Ref: http://msdn.microsoft.com/en-us/library/windows/desktop/aa363778(v=vs.85).aspx
            _tprintf(_T("CreateFile...\n"));
            if ((hEncryptedFile = CreateFile(
                strEncryptedFile,  // if you hardcode a filename here, use syntax: _T("C:\\temp\\append.txt"),
                GENERIC_WRITE,
                0,
                NULL,
                CREATE_ALWAYS,
                FILE_APPEND_DATA,
                NULL
                )) == INVALID_HANDLE_VALUE)
            {
                // Error
                _tprintf(_T("(ref 17) CreateFile error saving the encrypted data 0x%x\n"), GetLastError());
                return 1;
            }

            // Create a buffer for the plain text
            _tprintf(_T("malloc...\n"));
            pbData = NULL;
            if (!(pbData = (BYTE *)malloc(MAXSIZE)))
            {
                // Error
                _tprintf(_T("(ref 25) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            _tprintf(_T("malloc...\n"));
            pbEncData = NULL;
            if (!(pbEncData = (BYTE *)malloc(ENCRYPTEDLENGTH)))
            {
                // Error
                _tprintf(_T("(ref 53) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            for (i = 0; i < myloopcount; i++)
            {
                // ref https://msdn.microsoft.com/en-us/library/windows/desktop/aa365541(v=vs.85).aspx
                if (i > 0)
                {
                    SetFilePointer(
                        hPlainFile,     //     HANDLE hFile,
                        MAXSIZE,        //    LONG lDistanceToMove,
                        NULL,           //  PLONG lpDistanceToMoveHigh,
                        FILE_CURRENT
                        );
                }

                mymaxsize = MAXSIZE;

                // Read plain text
                _tprintf(_T("ReadFile...\n"));
                if (!ReadFile(hPlainFile, pbData, mymaxsize, &mymaxsize, NULL))
                {
                    // Error
                    _tprintf(_T("(ref 33) ReadFile error 0x%x\n"), GetLastError());
                    return 1;
                }

                // Encrypt data
                if (!CryptEncrypt(hKey, NULL, FALSE, 0, pbEncData, &mymaxsize, dwEncryptedLen))
                {
                    // Error
                    _tprintf(_T("(ref 4) CryptEncrypt error in loop number %i 0x%x\n"),i, GetLastError());
                    return 1;
                }

                _tprintf(_T("WriteFile...\n"));
                if (!WriteFile(
                    hEncryptedFile,
                    (LPCVOID)pbEncData,
                    dwDataLen,
                    &lpNumberOfBytesWritten,
                    NULL
                    ))
                {
                    // Error
                    _tprintf(_T("(ref 48) WriteFile error, i is %d 0x%x\n"), i, GetLastError());
                    return 1;
                }
            } // end for loop

            SetFilePointer(
                hPlainFile,     //     HANDLE hFile,
                MAXSIZE,        //     LONG lDistanceToMove,
                NULL,           //     PLONG lpDistanceToMoveHigh,
                FILE_CURRENT    
                );

            mymaxsize = MAXSIZE;

            if (mymodulus == 0)
            {
                _tprintf(_T("ReadFile...\n"));
                if (!ReadFile(hPlainFile, pbData, mymaxsize, &mymaxsize, NULL))
                {
                    // Error
                    _tprintf(_T("(ref 34) ReadFile error 0x%x\n"), GetLastError());
                    return 1;
                }

                // Encrypt data; last chunk must have TRUE flag when encrypting.
                if (!CryptEncrypt(hKey, NULL, TRUE, 0, pbEncData, &dwDataLen, dwEncryptedLen))
                {
                    // Error
                    _tprintf(_T("(ref 6) CryptEncrypt error 0x%x\n"), GetLastError());
                    return 1;
                }
            }
            else
            {
                _tprintf(_T("ReadFile...\n"));
                if (!ReadFile(hPlainFile, pbData, mymodulus, &mymodulus, NULL))
                {
                    // Error
                    _tprintf(_T("(ref 35) ReadFile error 0x%x\n"), GetLastError());
                    return 1;
                }

                // Encrypt data
                if (!CryptEncrypt(hKey, NULL, TRUE, 0, pbEncData, &mymodulus, dwEncryptedLen))
                {
                    // Error
                    _tprintf(_T("(ref 8) CryptEncrypt error 0x%x\n"), GetLastError());
                    return 1;
                }
            }

            // Finish writing.
            _tprintf(_T("WriteFile...\n"));
            if (!WriteFile(
                hEncryptedFile,
                (LPCVOID)pbEncData,
                dwDataLen,
                &lpNumberOfBytesWritten,
                NULL
                ))
            {
                // Error
                _tprintf(_T("(ref 49) WriteFile error 0x%x\n"), GetLastError());
                return 1;
            }
        }
        return 0;
    }
    __finally
    {
        // Clean up
        if (!pbPublicKey) {
            _tprintf(_T("free...\n"));
            free(pbPublicKey);
        }
        if (!pbData) {
            _tprintf(_T("free...\n"));
            free(pbData);
        }
        if (hPublicKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPublicKeyFile);
        }
        if (hPlainFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPlainFile);
        }
        if (hEncryptedFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hEncryptedFile);
        }
        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
        CryptDestroyKey(hKey);
        }
        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Encrypt

// Decrypt
int Decrypt(_TCHAR* strPrivateKeyFile, _TCHAR* strEncryptedFile, _TCHAR* strPlainFile)
{
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwPrivateKeyLen = 0;
    DWORD dwDataLen = 0;
    DWORD myloopcount = 0;
    DWORD mymodulus = 0;
    DWORD i = 0;
    DWORD mysize = 0;
    BYTE* pbPrivateKey = NULL;
    BYTE* pbData = NULL;
    BYTE* pbEncData = NULL;
    HANDLE hPrivateKeyFile = NULL;
    HANDLE hEncryptedFile = NULL;
    HANDLE hPlainFile = NULL;
    DWORD lpNumberOfBytesWritten = 0;

    __try
    {
        // Acquire access to key container
        _tprintf(_T("CryptAcquireContext...\n"));
        if (!CryptAcquireContext(&hCryptProv, _T("ACMEENCRYPT.EncryptDecrypt"), NULL, PROV_RSA_FULL, 0))
        {
            // Error
            _tprintf(_T("(ref 10) CryptAcquireContext error 0x%x\n"), GetLastError());

            // Try to create a new key container
            if (!CryptAcquireContext(&hCryptProv, _T("ACMEENCRYPT.EncryptDecrypt"), NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET))
            {
                // Error
                _tprintf(_T("(ref 11) CryptAcquireContext error 0x%x\n"), GetLastError());
                return 1;
            }
        }

        // Open private key file
        _tprintf(_T("CreateFile...\n"));
        if ((hPrivateKeyFile = CreateFile(
            strPrivateKeyFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 18) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwPrivateKeyLen = GetFileSize(hPrivateKeyFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("(ref 41) GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        // Create a buffer for the private key
        _tprintf(_T("malloc...\n"));
        if (!(pbPrivateKey = (BYTE *)malloc(dwPrivateKeyLen)))
        {
            // Error
            _tprintf(_T("(ref 27) malloc error 0x%x\n"), GetLastError());
            return 1;
        }

        // Read private key
        _tprintf(_T("ReadFile...\n"));
        if (!ReadFile(hPrivateKeyFile, pbPrivateKey, dwPrivateKeyLen, &dwPrivateKeyLen, NULL))
        {
            // Error
            _tprintf(_T("(ref 36) ReadFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Import private key
        _tprintf(_T("CryptImportKey...\n"));
        if (!CryptImportKey(hCryptProv, pbPrivateKey, dwPrivateKeyLen, 0, 0, &hKey))
        {
            // Error
            _tprintf(_T("(ref 39) CryptImportKey error 0x%x\n"), GetLastError());
            return 1;
        }

        // Open encrypted file
        _tprintf(_T("CreateFile...\n"));
        if ((hEncryptedFile = CreateFile(
            strEncryptedFile,
            GENERIC_READ,
            FILE_SHARE_READ,
            NULL,
            OPEN_EXISTING,
            FILE_FLAG_SEQUENTIAL_SCAN,
            NULL
            )) == INVALID_HANDLE_VALUE)
        {
            // Error
            _tprintf(_T("(ref 19) CreateFile error 0x%x\n"), GetLastError());
            return 1;
        }

        // Get file size
        _tprintf(_T("GetFileSize...\n"));
        if ((dwDataLen = GetFileSize(hEncryptedFile, NULL)) == INVALID_FILE_SIZE)
        {
            // Error
            _tprintf(_T("(ref 42) GetFileSize error 0x%x\n"), GetLastError());
            return 1;
        }

        if (dwDataLen == ENCRYPTEDLENGTH)
        {
            // Create a buffer for the encrypted data
            _tprintf(_T("malloc...\n"));
            if (!(pbData = (BYTE *)malloc(dwDataLen)))
            {
                // Error
                _tprintf(_T("(ref 28) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            // Read encrypted data
            _tprintf(_T("ReadFile...\n"));
            if (!ReadFile(hEncryptedFile, pbData, dwDataLen, &dwDataLen, NULL))
            {
                // Error
                _tprintf(_T("(ref 37) ReadFile error 0x%x\n"), GetLastError());
                return 1;
            }

            if (!CryptDecrypt(hKey, NULL, TRUE, 0, pbData, &dwDataLen))
            {
                // Error
                _tprintf(_T("(ref 54) CryptDecrypt error 0x%x\n"), GetLastError());
                return 1;
            }

            // Create a file to save the plain text
            _tprintf(_T("CreateFile...\n"));
            if ((hPlainFile = CreateFile(
                strPlainFile,
                GENERIC_WRITE,
                0,
                NULL,
                CREATE_ALWAYS,
                FILE_ATTRIBUTE_NORMAL,
                NULL
                )) == INVALID_HANDLE_VALUE)
            {
                // Error
                _tprintf(_T("(ref 20) CreateFile error 0x%x\n"), GetLastError());
                return 1;
            }

            // Write the plain text the file
            _tprintf(_T("WriteFile...\n"));
            if (!WriteFile(
                hPlainFile,
                (LPCVOID)pbData,
                dwDataLen,
                &lpNumberOfBytesWritten,
                NULL
                ))
            {
                // Error
                _tprintf(_T("(ref 50) WriteFile error 0x%x\n"), GetLastError());
                return 1;
            }
        }
        else
        {
            // encrypted file is bigger than 128 bytes
            // Figure out how many times we will need to loop.
            myloopcount = (DWORD)(dwDataLen / ENCRYPTEDLENGTH);
            _tprintf(_T("Loop counter is %d\n"), myloopcount);

            mymodulus = dwDataLen % ENCRYPTEDLENGTH;
            _tprintf(_T("Remainder is %d\n"), mymodulus);

            if (mymodulus == 0) // no remainder
            {
                myloopcount -= 1; // decrement by one
            }

            // Create a file to save the plain text
            _tprintf(_T("CreateFile...\n"));
            if ((hPlainFile = CreateFile(
                strPlainFile,
                GENERIC_WRITE,
                0,
                NULL,
                CREATE_ALWAYS,
                FILE_APPEND_DATA,
                NULL
                )) == INVALID_HANDLE_VALUE)
            {
                // Error
                _tprintf(_T("(ref 55) CreateFile error 0x%x\n"), GetLastError());
                return 1;
            }

            // Create a buffer for the encrypted data
            _tprintf(_T("malloc...\n"));
            if (!(pbData = (BYTE *)malloc(ENCRYPTEDLENGTH)))
            {
                // Error
                _tprintf(_T("(ref 56) malloc error 0x%x\n"), GetLastError());
                return 1;
            }

            for (i = 0; i < myloopcount; i++)
            {
                // ref https://msdn.microsoft.com/en-us/library/windows/desktop/aa365541(v=vs.85).aspx
                if (i > 0)
                {
                    SetFilePointer(
                        hEncryptedFile,     //  HANDLE hFile,
                        ENCRYPTEDLENGTH,    //  LONG lDistanceToMove,
                        NULL,               //  PLONG lpDistanceToMoveHigh,
                        FILE_CURRENT
                        );
                }

                mysize = ENCRYPTEDLENGTH;

                // Read encrypted data
                _tprintf(_T("ReadFile...\n"));
                if (!ReadFile(hEncryptedFile, pbData, mysize, &mysize, NULL))
                {
                    // Error
                    _tprintf(_T("(ref 37) ReadFile error 0x%x\n"), GetLastError());
                    return 1;
                }

                if (!CryptDecrypt(hKey, NULL, FALSE, 0, pbData, &mysize))
                {
                    // Error
                    _tprintf(_T("(ref 54) CryptDecrypt error in loop number %d 0x%x\n"),i, GetLastError());
                    return 1;
                }

                // Write the plain text the file
                _tprintf(_T("WriteFile...\n"));
                if (!WriteFile(
                    hPlainFile,
                    (LPCVOID)pbData,
                    dwDataLen,
                    &lpNumberOfBytesWritten,
                    NULL
                    ))
                {
                    // Error
                    _tprintf(_T("(ref 50) WriteFile error 0x%x\n"), GetLastError());
                    return 1;
                }
            } // end for loop

            SetFilePointer(
                hEncryptedFile,     //     HANDLE hFile,
                ENCRYPTEDLENGTH,        //     LONG lDistanceToMove,
                NULL,           //     PLONG lpDistanceToMoveHigh,
                FILE_CURRENT    //  DWORD dwMoveMethod, with FILE_CURRENT it's more efficient, not forced 
                //  to do (i * MAXSIZE) for second parameter size
                );

            mysize = ENCRYPTEDLENGTH;

            // Read encrypted data
            _tprintf(_T("ReadFile...\n"));
            if (!ReadFile(hEncryptedFile, pbData, mysize, &mysize, NULL))
            {
                // Error
                _tprintf(_T("(ref 37) ReadFile error 0x%x\n"), GetLastError());
                return 1;
            }

            if (!CryptDecrypt(hKey, NULL, TRUE, 0, pbData, &mysize))
            {
                // Error
                _tprintf(_T("(ref 54) CryptDecrypt error 0x%x\n"), GetLastError());
                return 1;
            }

            // Write the plain text the file
            _tprintf(_T("WriteFile...\n"));
            if (!WriteFile(
                hPlainFile,
                (LPCVOID)pbData,
                dwDataLen,
                &lpNumberOfBytesWritten,
                NULL
                ))
            {
                // Error
                _tprintf(_T("(ref 50) WriteFile error 0x%x\n"), GetLastError());
                return 1;
            }
        }

        return 0;
    }
    __finally
    {
        // Clean up       
        if (!pbPrivateKey) {
            _tprintf(_T("free...\n"));
            free(pbPrivateKey);
        }
        if (!pbData) {
            _tprintf(_T("free...\n"));
            free(pbData);
        }
        if (hPrivateKeyFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPrivateKeyFile);
        }
        if (hEncryptedFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hEncryptedFile);
        }
        if (hPlainFile) {
            _tprintf(_T("CloseHandle...\n"));
            CloseHandle(hPlainFile);
        }
        if (hKey) {
            _tprintf(_T("CryptDestroyKey...\n"));
            CryptDestroyKey(hKey);
        }
        if (hCryptProv) {
            _tprintf(_T("CryptReleaseContext...\n"));
            CryptReleaseContext(hCryptProv, 0);
        }
    }
}
// End of Decrypt
c
winapi
encryption
cryptoapi
asked on Stack Overflow Apr 21, 2015 by Bertrand_Szoghy

1 Answer

1

Yes; in general you cannot use RSA to encrypt data larger than the key size in a single pass, the limit is infact slightly shorter than the key size, depending on padding.

To encrypt larger amounts of data you would typically use a hybrid approach whereby you generate a symmetric key (say AES) and use that to encrypt your data.

You can then use your RSA public key to encrypt the AES key, combine it with the ciphertext and you have a blob that requires the private key to decrypt.

answered on Stack Overflow Apr 22, 2015 by Alex K.

User contributions licensed under CC BY-SA 3.0