So, my Hetzner cloud server keeps crashing / freezing me out of SSH. I constantly need to power reset. I know its Mailcow because when i shut down the service everything is fine. I only have Mailcow running on there.
I’ve attached a bunch of logs. I’ve been debugging for hours and haven’t found the issue.
I’ve ran every command imaginable to try to narrow down the issue.
gen@internal-tools:~$ sudo dmesg | tail -10 | grep -i "memory\|oom"

gen@internal-tools:~$ sudo journalctl -u docker --since "10 minutes ago" | tail -10
Aug 09 02:58:10 internal-tools dockerd[967]: time="2025-08-09T02:58:10.721667928Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=4ddb54792ebc6d7d0edab5aee137af86694b564a59f1204232e8a251968a0b32 ep=mailcowdockerized-php-fpm-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=f3dec17f0a885e83 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:10 internal-tools dockerd[967]: time="2025-08-09T02:58:10.948689951Z" level=info msg="No non-localhost DNS nameservers are left in resolv.conf. Using default external servers"
Aug 09 02:58:10 internal-tools dockerd[967]: time="2025-08-09T02:58:10.968692431Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=aac4ed75e2ca0ad7cc32be76e5c32021001b27224290ed178fcf1b0e42a9b0c7 ep=mailcowdockerized-ofelia-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=23a38726bc73026c traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:40 internal-tools dockerd[967]: time="2025-08-09T02:58:40.728171288Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=2563d8103558eb65e4e1627070a335983409e7d58ec0e79e1e74534901f33fac ep=mailcowdockerized-postfix-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=b77256b70796486d traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:40 internal-tools dockerd[967]: time="2025-08-09T02:58:40.963854803Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=c566bacea4cd477ac21e468d201827c09d6d59b7ca7e96d3b5b0b9e5bec35738 ep=mailcowdockerized-clamd-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=ec149f61a7bdf076 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:41 internal-tools dockerd[967]: time="2025-08-09T02:58:41.117255889Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=25ef55aebad6c9d15019005158a3eb8cd1b50fdef4d021567f393d435ffb9d54 ep=mailcowdockerized-rspamd-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=6391b8aaefda60b8 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:41 internal-tools dockerd[967]: time="2025-08-09T02:58:41.272895223Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=fa4b43b2ace20827056098738a89478d1f5591d5a5708fe165cf72d2d1b58d44 ep=mailcowdockerized-nginx-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=525b919612f68860 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:41 internal-tools dockerd[967]: time="2025-08-09T02:58:41.976127401Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=44da874a95de2384af6a8d2115ad970cc2678bce0ff3613d2cd03b031b1dc0f0 ep=mailcowdockerized-acme-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=109787b82eab1404 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:42 internal-tools dockerd[967]: time="2025-08-09T02:58:42.175317900Z" level=info msg="Skipping check for route to send NA, EMSGSIZE" eid=52c5741f4ca64b4b372f06f3276868b9ae58360a0cdff9d674f41a711a989ccf ep=mailcowdockerized-watchdog-mailcow-1 net=mailcowdockerized_mailcow-network nid=7bdf9f67429dfe57491702431f5894c261fa7e151e9bfab2e2c2455c8e7bc37c spanID=33567f8946f7dc02 traceID=ce19543925a1c0d66f5a6428c937b09a
Aug 09 02:58:51 internal-tools dockerd[967]: time="2025-08-09T02:58:51.156405846Z" level=info msg="ignoring event" container=30bdbe44f9a44c1885986ce5e1e463f0ce24ad0474a7a739efca80bbb72d7e60 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"


gen@internal-tools:~$ sudo docker ps -a | grep -E "Restarting|Exited"

gen@internal-tools:~$ sudo tail -f /var/log/pre-crash-monitor.log
[sudo] password for gen:
Sat Aug  9 01:13:15 AM UTC 2025: Docker containers: 19
---
Sat Aug  9 01:13:45 AM UTC 2025: Memory: 40.5%
Sat Aug  9 01:13:45 AM UTC 2025: Load:  0.11, 0.12, 0.09
Sat Aug  9 01:13:45 AM UTC 2025: Docker containers: 19
---
Sat Aug  9 01:14:15 AM UTC 2025: Memory: 40.3%
Sat Aug  9 01:14:15 AM UTC 2025: Load:  0.38, 0.18, 0.11
Sat Aug  9 01:14:15 AM UTC 2025: Docker containers: 19
---

gen@internal-tools:~$ sudo docker inspect mailcowdockerized-clamd-mailcow-1 | grep -A 10 "Memory"
[sudo] password for gen:
Sorry, try again.
[sudo] password for gen:
            "Memory": 0,
            "NanoCpus": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
--
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": null,
            "PidsLimit": null,
            "Ulimits": null,
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0,
            "MaskedPaths": [
                "/proc/asound",
                "/proc/acpi",

gen@internal-tools:~$ docker stats --no-stream | grep clamd
permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.51/containers/json": dial unix /var/run/docker.sock: connect: permission denied

gen@internal-tools:~$ sudo docker stats --no-stream | grep clamd
5c8b42e2628c   mailcowdockerized-clamd-mailcow-1       0.00%     844KiB / 7.57GiB     0.01%     11.9kB / 1.4kB    0B / 0B           3

gen@internal-tools:~$ sudo docker compose ps | grep clamd
no configuration file provided: not found

gen@internal-tools:~$ sudo docker logs --tail 20 mailcowdockerized-clamd-mailcow-1
SKIP_CLAMD=y, skipping ClamAV...
gen@internal-tools:~$ sudo uptime && free -h
 03:05:24 up 45 min,  8 users,  load average: 0.33, 0.16, 0.07
               total        used        free      shared  buff/cache   available
Mem:           7.6Gi       1.7Gi       5.0Gi        20Mi       1.1Gi       5.9Gi
Swap:             0B          0B          0B
gen@internal-tools:~$ sudo docker exec mailcowdockerized-clamd-mailcow-1 clamdscan /tmp || echo "ClamAV test completed"
ERROR: Could not connect to clamd on LocalSocket /run/clamav/clamd.sock: No such file or directory

----------- SCAN SUMMARY -----------
Infected files: 0
Total errors: 1
Time: 0.000 sec (0 m 0 s)
Start Date: 2025:08:09 03:05:38
End Date:   2025:08:09 03:05:38
ClamAV test completed
gen@internal-tools:~$ sudo docker exec mailcowdockerized-clamd-mailcow-1 clamd --version
ClamAV 1.4.2/27726/Fri Aug  8 08:32:39 2025
gen@internal-tools:~$

gen@internal-tools:~$ sudo docker logs -f mailcowdockerized-clamd-mailcow-1
[sudo] password for gen:
SKIP_CLAMD=y, skipping ClamAV...

gen@internal-tools:~$ sudo watch -n 5 'free -h && echo "=== Load ===" && uptime && echo "=== Top Memory Users ===" && docker stats --no-stream | head -5'
[sudo] password for gen:
Every 5.0s: free -h && echo "=== Load ===" && uptime && echo "=...  internal-tools: Sat Aug  9 03:06:56 2025

               total        used        free      shared  buff/cache   available
Mem:           7.6Gi       1.7Gi       5.0Gi        20Mi       1.2Gi       5.9Gi
Swap:             0B          0B          0B
=== Load ===
 03:06:56 up 46 min, 10 users,  load average: 0.13, 0.13, 0.06
=== Top Memory Users ===
CONTAINER ID   NAME                                    CPU %     MEM USAGE / LIMIT    MEM %     NET I/O
      BLOCK I/O         PIDS
496c05a0dcd7   mailcowdockerized-ipv6nat-mailcow-1     0.22%     4.871MiB / 7.57GiB   0.06%     0B / 0B
      20.5kB / 0B       9
e286f35ce053   mailcowdockerized-watchdog-mailcow-1    0.00%     21.24MiB / 7.57GiB   0.27%     322kB / 259k
B     0B / 0B           37
08805f847cec   mailcowdockerized-acme-mailcow-1        0.00%     1.211MiB / 7.57GiB   0.02%     91.8kB / 33.
9kB   0B / 12.3kB       3
75f63413676d   mailcowdockerized-nginx-mailcow-1       0.00%     7.332MiB / 7.57GiB   0.09%     1.02MB / 504
kB    4.1kB / 20.5kB    6

gen@internal-tools:~$ sudo watch -n 2 'docker stats --no-stream | grep clamd && echo "=== Memory Limit Check ===" && docker inspect mailcowdockerized-clamd-mailcow-1 | grep -A 2 -B 2 Memory'
Every 2.0s: docker stats --no-stream | grep clamd && echo "=== ...  internal-tools: Sat Aug  9 03:07:02 2025

5c8b42e2628c   mailcowdockerized-clamd-mailcow-1       0.00%     5.363MiB / 7.57GiB   0.07%     12.7kB / 1.4
kB    4.99MB / 0B       3
=== Memory Limit Check ===
            "Isolation": "",
            "CpuShares": 0,
            "Memory": 0,
            "NanoCpus": 0,
            "CgroupParent": "",
--
            "DeviceCgroupRules": null,
            "DeviceRequests": null,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": null,
            "PidsLimit": null,