Docker container with --restart=always not restarting after Out of Memory error

1

As you can see bellow my restart police is set to "always", but almost every night I get an out of memory (expected and not the issue I'm trying to solve here) and in the morning the container is stopped. If we check the RestartCount property it says 2, does this mean that it restarted twice and then stopped? Is it possible that it failed during the start process for some reason and stopped trying? Everything works fine when I start it again manually.

Here is the docker inspect result, let me know if you have a clue on what's happening.

{
    "Id": "...",
    "Created": "2020-11-26T16:32:57.611591274Z",
    "Path": "/bin/sh",
    "Args": [
        "-c",
        "/entrypoint.sh $profile $service_name"
    ],
    "State": {
        "Status": "exited",
        "Running": false,
        "Paused": false,
        "Restarting": false,
        "OOMKilled": false,
        "Dead": false,
        "Pid": 0,
        "ExitCode": 137,
        "Error": "OCI runtime create failed: container_linux.go:349: starting container process caused \"process_linux.go:449: container init caused \\\"process_linux.go:432: running prestart hook 0 caused \\\\\\\"error running hook: exit status 2, stdout: , stderr: fatal error: runtime: out of memory\\\\\\\\n\\\\\\\\nruntime stack:\\\\\\\\nruntime.throw(0x5630cae5b359, 0x16)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/panic.go:774 +0x74 fp=0x7ffffee76988 sp=0x7ffffee76958 pc=0x5630c9507714\\\\\\\\nruntime.sysMap(0xc000000000, 0x4000000, 0x5630cd2a71f8)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mem_linux.go:169 +0xc7 fp=0x7ffffee769c8 sp=0x7ffffee76988 pc=0x5630c94f0457\\\\\\\\nruntime.(*mheap).sysAlloc(0x5630cd28aa40, 0x2000, 0x0, 0x0)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/malloc.go:701 +0x1cf fp=0x7ffffee76a70 sp=0x7ffffee769c8 pc=0x5630c94e33ff\\\\\\\\nruntime.(*mheap).grow(0x5630cd28aa40, 0x1, 0xffffffff)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mheap.go:1255 +0xa5 fp=0x7ffffee76ac0 sp=0x7ffffee76a70 pc=0x5630c94fe605\\\\\\\\nruntime.(*mheap).allocSpanLocked(0x5630cd28aa40, 0x1, 0x5630cd2a7208, 0x1)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mheap.go:1170 +0x268 fp=0x7ffffee76b40 sp=0x7ffffee76ac0 pc=0x5630c94fe498\\\\\\\\nruntime.(*mheap).alloc_m(0x5630cd28aa40, 0x1, 0x5630c953002a, 0x0)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mheap.go:1022 +0xc6 fp=0x7ffffee76b90 sp=0x7ffffee76b40 pc=0x5630c94fdd36\\\\\\\\nruntime.(*mheap).alloc.func1()\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mheap.go:1093 +0x4e fp=0x7ffffee76bc8 sp=0x7ffffee76b90 pc=0x5630c95339de\\\\\\\\nruntime.(*mheap).alloc(0x5630cd28aa40, 0x1, 0x1002a, 0x40000)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mheap.go:1092 +0x8c fp=0x7ffffee76c18 sp=0x7ffffee76bc8 pc=0x5630c94fe01c\\\\\\\\nruntime.(*mcentral).grow(0x5630cd28b878, 0x0)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mcentral.go:255 +0x7d fp=0x7ffffee76c58 sp=0x7ffffee76c18 pc=0x5630c94efe7d\\\\\\\\nruntime.(*mcentral).cacheSpan(0x5630cd28b878, 0x7f7e1619e000)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mcentral.go:106 +0x300 fp=0x7ffffee76cb8 sp=0x7ffffee76c58 pc=0x5630c94ef9a0\\\\\\\\nruntime.(*mcache).refill(0x7f7e1619e008, 0x2a)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/mcache.go:138 +0x87 fp=0x7ffffee76cd8 sp=0x7ffffee76cb8 pc=0x5630c94ef437\\\\\\\\nruntime.(*mcache).nextFree(0x7f7e1619e008, 0x5630cd2a722a, 0x7f7e1619e008, 0x8, 0xfffffffffffffff8)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/malloc.go:854 +0x89 fp=0x7ffffee76d10 sp=0x7ffffee76cd8 pc=0x5630c94e3c29\\\\\\\\nruntime.mallocgc(0x180, 0x5630cc009ea0, 0x1, 0x5630cd2a7260)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/malloc.go:1022 +0x7a5 fp=0x7ffffee76db0 sp=0x7ffffee76d10 pc=0x5630c94e4575\\\\\\\\nruntime.newobject(0x5630cc009ea0, 0x4000)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/malloc.go:1151 +0x3a fp=0x7ffffee76de0 sp=0x7ffffee76db0 pc=0x5630c94e497a\\\\\\\\nruntime.malg(0x1196e00008000, 0x5630cd28d0e8)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/proc.go:3232 +0x33 fp=0x7ffffee76e20 sp=0x7ffffee76de0 pc=0x5630c9510be3\\\\\\\\nruntime.mpreinit(...)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/os_linux.go:324\\\\\\\\nruntime.mcommoninit(0x5630cd285500)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/proc.go:626 +0xc6 fp=0x7ffffee76e58 sp=0x7ffffee76e20 pc=0x5630c950a516\\\\\\\\nruntime.schedinit()\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/proc.go:543 +0x76 fp=0x7ffffee76eb0 sp=0x7ffffee76e58 pc=0x5630c950a176\\\\\\\\nruntime.rt0_go(0x7ffffee76fb8, 0x4, 0x7ffffee76fb8, 0x0, 0x7f7e1515c02a, 0x4, 0x7ffffee76fb8, 0x4fee76fe0, 0x5630c9535da0, 0x0, ...)\\\\\\\\n\\\\\\\\t/usr/lib/golang/src/runtime/asm_amd64.s:214 +0x129 fp=0x7ffffee76eb8 sp=0x7ffffee76eb0 pc=0x5630c9535ed9\\\\\\\\n\\\\\\\"\\\"\": unknown",
        "StartedAt": "2020-11-27T02:53:11.621911146Z",
        "FinishedAt": "2020-11-28T02:53:12.35136947Z"
    },
    "Image": "...",
    "ResolvConfPath": "/.../resolv.conf",
    "HostnamePath": "/.../hostname",
    "HostsPath": "/.../hosts",
    "LogPath": "/...-json.log",
    "Name": "/service",
    "RestartCount": 2,
    "Driver": "overlay2",
    "Platform": "linux",
    "MountLabel": "",
    "ProcessLabel": "",
    "AppArmorProfile": "",
    "ExecIDs": null,
    "HostConfig": {
        "Binds": [
            "/home/ec2-user/logs:/logs",
            "/var/www:/www"
        ],
        "ContainerIDFile": "",
        "LogConfig": {
            "Type": "json-file",
            "Config": {}
        },
        "NetworkMode": "default",
        "PortBindings": {
            "8082/tcp": [
                {
                    "HostIp": "",
                    "HostPort": "8082"
                }
            ]
        },
        "RestartPolicy": {
            "Name": "always",
            "MaximumRetryCount": 0
        },
        "AutoRemove": false,
        "VolumeDriver": "",
        "VolumesFrom": null,
        "CapAdd": null,
        "CapDrop": null,
        "Capabilities": null,
        "Dns": [],
        "DnsOptions": [],
        "DnsSearch": [],
        "ExtraHosts": null,
        "GroupAdd": null,
        "IpcMode": "private",
        "Cgroup": "",
        "Links": null,
        "OomScoreAdj": 0,
        "PidMode": "",
        "Privileged": false,
        "PublishAllPorts": false,
        "ReadonlyRootfs": false,
        "SecurityOpt": null,
        "UTSMode": "",
        "UsernsMode": "",
        "ShmSize": 67108864,
        "Runtime": "runc",
        "ConsoleSize": [
            0,
            0
        ],
        "Isolation": "",
        "CpuShares": 0,
        "Memory": 629145600,
        "NanoCpus": 0,
        "CgroupParent": "",
        "BlkioWeight": 0,
        "BlkioWeightDevice": [],
        "BlkioDeviceReadBps": null,
        "BlkioDeviceWriteBps": null,
        "BlkioDeviceReadIOps": null,
        "BlkioDeviceWriteIOps": null,
        "CpuPeriod": 0,
        "CpuQuota": 0,
        "CpuRealtimePeriod": 0,
        "CpuRealtimeRuntime": 0,
        "CpusetCpus": "",
        "CpusetMems": "",
        "Devices": [],
        "DeviceCgroupRules": null,
        "DeviceRequests": null,
        "KernelMemory": 0,
        "KernelMemoryTCP": 0,
        "MemoryReservation": 0,
        "MemorySwap": 1258291200,
        "MemorySwappiness": null,
        "OomKillDisable": false,
        "PidsLimit": null,
        "Ulimits": [
            {
                "Name": "nofile",
                "Hard": 4096,
                "Soft": 1024
            }
        ],
        "CpuCount": 0,
        "CpuPercent": 0,
        "IOMaximumIOps": 0,
        "IOMaximumBandwidth": 0,
        "MaskedPaths": [
            "/proc/asound",
            "/proc/acpi",
            "/proc/kcore",
            "/proc/keys",
            "/proc/latency_stats",
            "/proc/timer_list",
            "/proc/timer_stats",
            "/proc/sched_debug",
            "/proc/scsi",
            "/sys/firmware"
        ],
        "ReadonlyPaths": [
            "/proc/bus",
            "/proc/fs",
            "/proc/irq",
            "/proc/sys",
            "/proc/sysrq-trigger"
        ]
    },
    "GraphDriver": {
        "Data": {
            "LowerDir": "/.../diff",
            "MergedDir": "/.../merged",
            "UpperDir": "/.../diff",
            "WorkDir": "/.../work"
        },
        "Name": "overlay2"
    },
    "Mounts": [
        {
            "Type": "bind",
            "Source": "/home/ec2-user/logs/service",
            "Destination": "/logs",
            "Mode": "",
            "RW": true,
            "Propagation": "rprivate"
        },
        {
            "Type": "bind",
            "Source": "/var/www/service",
            "Destination": "/www",
            "Mode": "",
            "RW": true,
            "Propagation": "rprivate"
        }
    ],
    "Config": {
        "Hostname": "b20730462c1d",
        "Domainname": "",
        "User": "",
        "AttachStdin": false,
        "AttachStdout": false,
        "AttachStderr": false,
        "ExposedPorts": {
            "8082/tcp": {}
        },
        "Tty": true,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "TZ=Europe/Lisbon",
            "JAVA_OPTS=-XX:MinRAMPercentage=60 -XX:MaxRAMPercentage=95 -XshowSettings:vm",
            "PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
            "LANG=en_US.UTF-8",
            "LANGUAGE=en_US:en",
            "LC_ALL=en_US.UTF-8",
            "JAVA_VERSION=jdk-11.0.7+10",
            "JAVA_HOME=/opt/java/openjdk",
            "profile=prod",
            "service_name=service"
        ],
        "Cmd": null,
        "Image": "...:latest",
        "Volumes": null,
        "WorkingDir": "",
        "Entrypoint": [
            "/bin/sh",
            "-c",
            "/entrypoint.sh $profile $service_name"
        ],
        "OnBuild": null,
        "Labels": {
            "name": "...",
            "version": "\"0.0.1\""
        }
    },
    "NetworkSettings": {
        "Bridge": "",
        "SandboxID": "...",
        "HairpinMode": false,
        "LinkLocalIPv6Address": "",
        "LinkLocalIPv6PrefixLen": 0,
        "Ports": {},
        "SandboxKey": "/var/run/docker/netns/3873dd087817",
        "SecondaryIPAddresses": null,
        "SecondaryIPv6Addresses": null,
        "EndpointID": "",
        "Gateway": "",
        "GlobalIPv6Address": "",
        "GlobalIPv6PrefixLen": 0,
        "IPAddress": "",
        "IPPrefixLen": 0,
        "IPv6Gateway": "",
        "MacAddress": "",
        "Networks": {
            "bridge": {
                "IPAMConfig": null,
                "Links": null,
                "Aliases": null,
                "NetworkID": "...",
                "EndpointID": "",
                "Gateway": "",
                "IPAddress": "",
                "IPPrefixLen": 0,
                "IPv6Gateway": "",
                "GlobalIPv6Address": "",
                "GlobalIPv6PrefixLen": 0,
                "MacAddress": "",
                "DriverOpts": null
            }
        }
    }
}
java
docker
out-of-memory
restart
asked on Stack Overflow Nov 28, 2020 by Bruno

0 Answers

Nobody has answered this question yet.


User contributions licensed under CC BY-SA 3.0