Upgrading from makecert.exe to CertEnroll - issues with certificate trust

I have an app that up until now used makecert.exe to generate self certificates. However as makecert does't have the ability to add a SubjectAltName field, I am needing to migrate the code to certenroll.dll

This is the original makecert code:

public static X509Certificate2 MakeCert(string subjectName)
    {
        X509Certificate2 cert;
        string certFile = Path.Combine(Path.GetTempPath(), subjectName + ".cer");

        var process = new Process
        {
            StartInfo = new ProcessStartInfo
            {
                FileName = "makecert.exe",
                Arguments = " -pe -ss my -n \"CN=" + subjectName + ", O=myCert, OU=Created by me\" -sky exchange -in MyCustomRoot -is my -eku 1.3.6.1.5.5.7.3.1 -cy end -a sha1 -m 132 -b 10/08/2018 " + certFile,
                UseShellExecute = false,
                RedirectStandardOutput = true,
                CreateNoWindow = true
            }
        };

        process.Start();
        string str = "";
        while (!process.StandardOutput.EndOfStream)
        {
            var line = process.StandardOutput.ReadLine();
            str += line;
            //Console.WriteLine(line);
        }
        process.WaitForExit();

        cert = new X509Certificate2(certFile);
        // Install Cert
        try
        {

            var store = new X509Store(StoreName.My, StoreLocation.CurrentUser);
            store.Open(OpenFlags.ReadWrite);
            try
            {
                var contentType = X509Certificate2.GetCertContentType(certFile);
                var pfx = cert.Export(contentType);
                cert = new X509Certificate2(pfx, (string)null, X509KeyStorageFlags.PersistKeySet | X509KeyStorageFlags.MachineKeySet);
                store.Add(cert);
            }
            finally
            {
                store.Close();
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine(String.Format("Could not create the certificate from file from {0}", certFile), ex);
        }
        return cert;
    }

And this is the certenroll.dll code:

   public static X509Certificate2 CertOpen(string subjectName)
    {
        try
        {
            X509Store store = new X509Store("My", StoreLocation.CurrentUser);
            store.Open(OpenFlags.ReadOnly);
            try
            {
                var cer = store.Certificates.Find(
                    X509FindType.FindBySubjectName,
                    subjectName,
                    false);

                if (cer.Count > 0)
                {
                    return cer[0];
                }
                else
                {
                    return null;
                }
            }
            finally
            {
                store.Close();
            }
        }
        catch
        {
            return null;
        }
    }

    public static X509Certificate2 CertCreateNew(string subjectName)
    {
        // create DN for subject and issuer
        var dn = new CX500DistinguishedName();
        dn.Encode("CN=" + subjectName, X500NameFlags.XCN_CERT_NAME_STR_NONE);


        // create a new private key for the certificate
        CX509PrivateKey privateKey = new CX509PrivateKey();
        privateKey.ProviderName = "Microsoft Base Cryptographic Provider v1.0";
        privateKey.MachineContext = false;
        privateKey.Length = 2048;
        privateKey.KeySpec = X509KeySpec.XCN_AT_SIGNATURE; // use is not limited
        privateKey.ExportPolicy = X509PrivateKeyExportFlags.XCN_NCRYPT_ALLOW_PLAINTEXT_EXPORT_FLAG;
        privateKey.Create();


        var hashobj = new CObjectId();
        hashobj.InitializeFromAlgorithmName(ObjectIdGroupId.XCN_CRYPT_HASH_ALG_OID_GROUP_ID,
            ObjectIdPublicKeyFlags.XCN_CRYPT_OID_INFO_PUBKEY_ANY,
            AlgorithmFlags.AlgorithmFlagsNone, "SHA256");

        // add extended key usage if you want - look at MSDN for a list of possible OIDs
        var oid = new CObjectId();
        oid.InitializeFromValue("1.3.6.1.5.5.7.3.1"); // SSL server
        var oidlist = new CObjectIds();
        oidlist.Add(oid);
        var eku = new CX509ExtensionEnhancedKeyUsage();
        eku.InitializeEncode(oidlist);

        // Create the self signing request
        var cert = new CX509CertificateRequestCertificate();

        cert.InitializeFromPrivateKey(X509CertificateEnrollmentContext.ContextUser, privateKey, "");

        X509Certificate2 signer = CertOpen("MyCustomRoot");
        if (signer == null)
        {
            throw new CryptographicException("Signer not found");
        }
        String base64str = Convert.ToBase64String(signer.RawData);


        ISignerCertificate signerCertificate = new CSignerCertificate();
        signerCertificate.Initialize(false, X509PrivateKeyVerify.VerifySilent, EncodingType.XCN_CRYPT_STRING_BASE64, base64str);
        // this line MUST be called AFTER IX509CertificateRequestCertificate.InitializeFromPrivateKey call,
        // otherwise you will get OLE_E_BLANK uninitialized object error.
        cert.SignerCertificate = (CSignerCertificate)signerCertificate;


        cert.Subject = dn;
        cert.Issuer.Encode(signer.Subject, X500NameFlags.XCN_CERT_NAME_STR_NONE); ; // the issuer and the subject are the same
        cert.NotBefore = DateTime.Now;
        // this cert expires immediately. Change to whatever makes sense for you
        cert.NotAfter = DateTime.Now.AddYears(10);
        cert.X509Extensions.Add((CX509Extension)eku); // add the EKU
        cert.HashAlgorithm = hashobj; // Specify the hashing algorithm
        cert.Encode(); // encode the certificate

        // Do the final enrollment process
        var enroll = new CX509Enrollment();
        enroll.InitializeFromRequest(cert); // load the certificate
        enroll.CertificateFriendlyName = subjectName; // Optional: add a friendly name

        string csr = enroll.CreateRequest(); // Output the request in base64
                                             // and install it back as the response
        enroll.InstallResponse(InstallResponseRestrictionFlags.AllowUntrustedCertificate,
            csr, EncodingType.XCN_CRYPT_STRING_BASE64, ""); // no password
                                                            // output a base64 encoded PKCS#12 so we can import it back to the .Net security classes
        var base64encoded = enroll.CreatePFX("", // no password, this is for internal consumption
            PFXExportOptions.PFXExportChainWithRoot);

        // instantiate the target class with the PKCS#12 data (and the empty password)
        return new System.Security.Cryptography.X509Certificates.X509Certificate2(
            System.Convert.FromBase64String(base64encoded), "",
            // mark the private key as exportable (this is usually what you want to do)
            System.Security.Cryptography.X509Certificates.X509KeyStorageFlags.Exportable
        );
    }

Further to the assistance from Crypt32 I am now having issues with the signerCertificate.Initialize line. I can't seem to get it to use my self cert. root certificate. I assume that I'm trying to feed it in the wrong format as I am getting the following error:

The certificate does not have the property that references a private key. 0x8009200a (CRYPT_E_UNEXPECTED_MSG_TYPE)

You have to specify a signer certificate in SignerCertificate property of IX509CertificateRequestCertificate object (cert variable in your code). Signer certificate must be supplied in a form of ISignerCertificate instance. More information: ISignerCertificate interface

Update 1 (13.12.2019)

sorry to tell, but almost every piece in ISignerCertificate call is incorrect.

1. If you specify X509PrivateKeyVerify.VerifyNone, then private key existence is not checked. You need to use X509PrivateKeyVerify.VerifySilent flag.

2. You are using Base64 formatting with PEM header and footer to format certificate as a string. You are using EncodingType.XCN_CRYPT_STRING_BASE64 which expects raw Base64 string without PEM envelope. PEM-formatted certificate uses EncodingType.XCN_CRYPT_STRING_BASE64HEADER encoding type. In your case I would do:

X509Certificate signer = CertOpen("MyCustomRoot");
if (signer == null) {
    throw new CryptographicException("Signer not found");
}
String base64str = Convert.ToBase64String(signer.RawData);
signerCertificate.Initialize(false, X509PrivateKeyVerify.VerifySilent, EncodingType.XCN_CRYPT_STRING_BASE64, base64str);

<...>
// put issuer directly from issuer cert:
issuer.Encode(signer.Subject, X500NameFlags.XCN_CERT_NAME_STR_NONE);
<...>
cert.InitializeFromPrivateKey(X509CertificateEnrollmentContext.ContextUser, privateKey, "");
// this line MUST be called AFTER IX509CertificateRequestCertificate.InitializeFromPrivateKey call,
// otherwise you will get OLE_E_BLANK uninitialized object error.
cert.SignerCertificate = signerCertificate;

Also, some minor improvements:

1. In CertOpen method you do not close the store.
2. if (cer != null && cer.Count >0) -- IIRC, X509Certificate2Collection.Find never returns null, so just check if returned collection is non-empty.
3. You are assigning ISignerCertificate object to request before initializing it. See my comments above.
4. Bear in mind, that SHA512 is not enabled by default in all cryptographic modules. SHA512 is disabled in Windows when you use TLS 1.2

Update 2 (14.12.2019)

I reproed the code with my modifications I provided yesterday, the code works. What CRYPT_E_UNEXPECTED_MSG_TYPE error suggests is that signer certificate doesn't have a private key in certificate store.