Windows NtCreateFile returning STATUS_INVALID_PARAMETER (using undocumented NT syscall interface)

I thought it might be interesting/fun to write a program in assembly (for 64-bit x86, NASM) that uses the undocumented Windows NT system call interface. Unfourtunately, it appears as though somewhere along the line I'm incorrectly setting up a structure or some other parameter, leading to NtCreateFile returning STATUS_INVALID_PARAMETER (0xC000000D). I suspect that this could also be due to the fact that I might be putting the parameters in the wrong order. My information on how to use the system call interface comes from another SO question where someone mentioned decompiling the function in ntdll, which led me to the fact that I could invoke a system call in a fashion similar to Linux, i.e. putting the magic number in RAX and then using the syscall instruction (int 2e also works). This generates the same result as directly calling NtCreateFile. After having read this Microsoft document, I figured out the order to put the parameters in. The magic number for NtCreateFile is from here. Here's the program:

BITS 64

section .data
;; Declare a quadword (pointer size) to store the file handle in
file: resq 1

;; Declare the memory necessary for an OBJECT_ATTRIBUTES structure (five quadwords)
;; typedef struct _OBJECT_ATTRIBUTES {
;;   ULONG           Length;
;;   HANDLE          RootDirectory;
;;   PUNICODE_STRING ObjectName;
;;   ULONG           Attributes;
;;   PVOID           SecurityDescriptor;
;;   PVOID           SecurityQualityOfService;
;; } OBJECT_ATTRIBUTES;
atrs: resq 5

;; Declare a UNICODE_STRING for the file path, and the text to be written
fname:
    resq 2

fname_str: dw "\SystemRoot\test.txt", 0

;; An IO_STATUS_BLOCK
iostat: resq 3

section .code
;; The signature of NtCreateFile:
;; __kernel_entry NTSTATUS NtCreateFile(
;;   PHANDLE            FileHandle,
;;   ACCESS_MASK        DesiredAccess,
;;   POBJECT_ATTRIBUTES ObjectAttributes,
;;   PIO_STATUS_BLOCK   IoStatusBlock,
;;   PLARGE_INTEGER     AllocationSize,
;;   ULONG              FileAttributes,
;;   ULONG              ShareAccess,
;;   ULONG              CreateDisposition,
;;   ULONG              CreateOptions,
;;   PVOID              EaBuffer,
;;   ULONG              EaLength
;; );

;; Still counting this as a win since this function is just for a structure
extern RtlInitUnicodeString

global mainCRTstartup
mainCRTstartup:
    ;; Initialize a Unicode string
    lea rcx, [fname] ;; The memory for the structure
    lea rdx, [fname_str] ;; The string
    call RtlInitUnicodeString

    ;; NtCreateFile parameters
    mov r10, rcx ;; Save rcx (this is what happens in ntdll, not sure why yet)
    mov eax, 55h ;; NtCreateFile's number is 0x55 in all versions thus far
    lea rcx, [file] ;; Put the file handle address into rcx
    mov rdx, 40100000h ;; The desired file access, which is GENERIC_WRITE | SYNCHRONIZE

    ;; Set up attributes for the handle
    mov rbx, 40 ;; Structure size, 40 bytes/5 quadwords
    mov QWORD [atrs], rbx
    xor rbx, rbx ;; Set RootDirectory to NULL
    mov QWORD [atrs + 8], rbx
    mov rbx, QWORD [fname] ;; Move the start of the filename data to ObjectName
    mov QWORD [atrs + 16], rbx
    mov rbx, 40h
    mov QWORD [atrs + 12], rbx ;; OBJ_CASE_INSENSITIVE

    mov r8, atrs ;; Move the pointer to the OBJECT_ATTRIBUTES structure to r8
    xor r9, r9 ;; NULL

    ;; Now that we've reached 4 arguments, stuff goes on the stack in reverse
    xor rbx, rbx ;; Zero rbx for general use as zero
    push rbx ;; EaLength, not used
    push rbx ;; EaBuffer, NULL
    mov rsi, 0x20 ;; CreateOptions is FILE_SYNCHRONOUS_IO_NONALERT
    push rsi
    xor rsi, rsi ;; FILE_OVERWRITE_IF (create/overwrite)
    mov rsi, 5
    push rsi
    push rbx ;; We don't have other threads
    mov rsi, 80h
    push rsi ;; FILE_ATTRIBUTE_NORMAL
    push rbx ;; NULL
    syscall ;; Jump into kernel mode and call NtCreateFile

    ;; Exit
    mov eax, 2ch ;; NtTerminateProcess
    xor ecx, ecx ;; We want to kill this process, not another one
    xor edx, edx ;; Exit with code 0
    syscall ;; Enter kernel mode again

I assemble and link it with these commands:

nasm -o ntsyscall.obj -fwin64 ntsyscall.s
cl ntsyscall.obj -link -entry:mainCRTstartup -subsystem:console -largeaddressaware:no -debug ntdll.lib # ntdll's RtlInitUnicodeString is used

I've also written this C program which is attempting the same thing and works as expected:

#define _AMD64_

#include <ntdef.h>

/* Definitions for symbols and structures needed */
#define GENERIC_WRITE 0x40000000L
#define SYNCHRONIZE 0x00100000L
#define FILE_SYNCHRONOUS_IO_NONALERT 0x20
#define FILE_OVERWRITE_IF 0x5
#define FILE_ATTRIBUTE_NORMAL 0x80

typedef struct _IO_STATUS_BLOCK {
    union {
        long Status;
        long Pointer;
    } DUMMYUNIONNAME;
    unsigned long *Information;
} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK;

extern long NtCreateFile(HANDLE *file_ret, unsigned long desired_access,
             OBJECT_ATTRIBUTES *oattrs, IO_STATUS_BLOCK *iostat,
             LARGE_INTEGER *alloc_size, unsigned long fattrs, 
             unsigned long share_access, unsigned long create_disp,
             unsigned long create_opts, void *ea_buf,
             unsigned long ea_len);
extern void RtlFillMemory(void *dst, unsigned long n, int c);
extern void RtlInitUnicodeString(UNICODE_STRING *dst, unsigned short *src);

int mainCRTstartup(void)
{
    HANDLE file;
    OBJECT_ATTRIBUTES atrs;
    UNICODE_STRING fname;
    //UNICODE_STRING ftext;
    IO_STATUS_BLOCK iostat;

    /* Initialize obsoletely designed Windows structures (not a fan of the Windows API) */
    RtlFillMemory(&atrs, sizeof(atrs), 0);
    RtlInitUnicodeString(&fname, L"\\SystemRoot\\test.txt");
    atrs.Length = sizeof(atrs);
    atrs.RootDirectory = NULL;
    atrs.ObjectName = &fname;
    atrs.Attributes = OBJ_CASE_INSENSITIVE;

    /* Call the function */
    NtCreateFile(&file, GENERIC_WRITE | SYNCHRONIZE, &atrs, &iostat, NULL,
             FILE_ATTRIBUTE_NORMAL, 0, FILE_OVERWRITE_IF,
             FILE_SYNCHRONOUS_IO_NONALERT, NULL, 0);

    /* Return to the program loader */
    return 0;
}

which is compiled with

cl ntsyscall_equiv.c -link -entry:mainCRTstartup -debug ntdll.lib

If anyone can help with this, it's very much appreciated.

If you're actually writing production code you should not invoke the syscall gate yourself; import and call the function in NTDLL. The syscall gate can change in patch releases and has changed before.

The actual syscall gate in NTDLL is a function containing only the syscall instruction; your code doesn't work because you tried to do it yourself and your stack is misaligned with what the kernel expects.

Syscall numbers have changed in service packs; again you could theoretically have an issue there.