open syscall failes to create a file without a reason

section .text
global _start   ;must be declared for linker (ld)
_start:
    mov eax,5
    mov ebx,plname
    mov ecx,0x202
    mov edx,0700o
    int 0x80

    mov eax,4
    mov ecx,plaintext
    mov edx,256
    int 0x80

    xor eax,eax
    inc eax
    xor ebx,ebx
    int 0x80


section .data
key db '123456passwordqwerty',0x0
keylen equ $ - key     ;length of our dear string
plname  db 'plname.bin',0x0
plaintext times 256 db 1

first part planned to create a file specified in plname, first time I'd tryed create it into /tmp/plname.bin and after fail, try to create at least into excuting directory.I've also tried create syscall and got the same results.

programm fails on open syscall, after excuting int 0x80 instruction, eax contains -2, programm ends normally, but doesn't create file. here i got flags and mods https://sourceware.org/gdb/onlinedocs/gdb/mode_005ft-Values.html#mode_005ft-Values

here is gdb output

Dump of assembler code for function _start:
0x08048080 <+0>:    mov    $0x8,%eax
0x08048085 <+5>:    mov    $0x80490c9,%ebx
0x0804808a <+10>:   mov    $0x700,%ecx
0x0804808f <+15>:   int    $0x80
0x08048091 <+17>:   mov    $0x4,%eax
0x08048096 <+22>:   mov    $0x80490e3,%ecx
0x0804809b <+27>:   mov    $0x100,%edx
0x080480a0 <+32>:   int    $0x80
0x080480a2 <+34>:   xor    %eax,%eax
0x080480a4 <+36>:   inc    %eax
0x080480a5 <+37>:   xor    %ebx,%ebx
0x080480a7 <+39>:   int    $0x80
End of assembler dump.

Breakpoint 1, 0x0804808f in _start ()
(gdb) i r eax
eax            0x5  5
(gdb) stepi
0x08048094 in _start () 
(gdb) i r eax
eax            0x5  5
(gdb) i r eax ebx ecx edx esi edi
eax            0x5  5
ebx            0x80490d1    134516945
ecx            0x202    514
edx            0x1c0    448
esi            0x0  0
edi            0x0  0 
(gdb) stepi
0x08048096 in _start ()
(gdb) i r eax ebx ecx edx esi edi
eax            0xfffffffe   -2
ebx            0x80490d1    134516945
ecx            0x202    514
edx            0x1c0    448
esi            0x0  0
edi            0x0  0

You used the wrong reference manual. What you linked to is the flags used in the gdb protocol, not the ones used by system calls.

O_CREAT is actually 0100 octal, so you should do mov ecx,0102o.

Also note you have forgotten to move the returned file descriptor from eax to ebx for the sys_write.

Working code:

section .text
global _start   ;must be declared for linker (ld)
_start:
    mov eax,5
    mov ebx,plname
    mov ecx,0102o
    mov edx,0700o
    int 0x80

    mov ebx, eax
    mov eax,4
    mov ecx,plaintext
    mov edx,256
    int 0x80

    xor eax,eax
    inc eax
    xor ebx,ebx
    int 0x80


section .data
key db '123456passwordqwerty',0x0
keylen equ $ - key     ;length of our dear string
plname  db 'plname.bin',0x0
plaintext times 256 db 1