From 9f8e51e0e2b7a56e5157c681cd683e554e5c7680 Mon Sep 17 00:00:00 2001
From: Ivan Malison <IvanMalison@gmail.com>
Date: Thu, 9 Jul 2026 11:32:53 -0700
Subject: [PATCH] gitea-runner: redirect job TMPDIR off the DynamicUser tmpfs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CI on railbird-sf failed intermittently with postgres DiskFull
("could not create file base/5/...", "No space left on device")
while `df /` showed 200G+ free. Root cause, confirmed live: the
upstream services.gitea-actions-runner module sets DynamicUser=true,
which gives the service a private /tmp backed by a tmpfs capped at
10% of RAM (~3.2G) — despite our PrivateTmp=false. Every host-executor
job inherits it; leaked `nix-shell.*` dirs from cancelled jobs plus
pytest's ephemeral postgres (conftest tempfile.mkdtemp -> PGDATA) fill
the 3.2G ceiling and the next test dies with ENOSPC, invisible to a
host-disk df.

Point the service's TMPDIR at the disk-backed state dir and prune
leaked entries after 2 days. Socket paths under it (~52 chars) stay
well under postgres's 107-char unix-socket limit.

Also add daily nix GC + min-free/max-free on railbird-sf: the store
had grown 20k dead paths / 40G+ reclaimable with no automatic GC, so
busy build evenings could still run the real disk low; min-free makes
nix GC mid-build instead of hitting ENOSPC.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 nixos/gitea-runner.nix         | 12 ++++++++++++
 nixos/machines/railbird-sf.nix | 17 +++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/nixos/gitea-runner.nix b/nixos/gitea-runner.nix
index 4c010fb6..4a47534f 100644
--- a/nixos/gitea-runner.nix
+++ b/nixos/gitea-runner.nix
@@ -58,9 +58,21 @@ makeEnable config "myModules.gitea-runner" false {
     in {
       XDG_CONFIG_HOME = gitea-runner-directory;
       XDG_CACHE_HOME = "${gitea-runner-directory}/.cache";
+      # Despite PrivateTmp=false below, the upstream module's DynamicUser=true
+      # still gives the service a private /tmp: a tmpfs capped at 10% of RAM
+      # (~3.2G here) with 400k inodes. CI jobs inherit it — `nix develop`
+      # shell dirs (leaked on cancelled jobs) and pytest's ephemeral postgres
+      # (tempfile.mkdtemp) slowly fill it until tests die with
+      # DiskFull/ENOSPC while `df /` shows hundreds of GB free. Point job
+      # temp at the disk-backed state dir instead; tmpfiles rule below prunes
+      # leaked entries.
+      TMPDIR = "${gitea-runner-directory}/tmp";
     };
     serviceConfig.PrivateTmp = false;
   };
+  systemd.tmpfiles.rules = [
+    "d /var/lib/private/gitea-runner/tmp 0750 gitea-runner gitea-runner 2d"
+  ];
   users.groups.gitea-runner = {};
   users.users.gitea-runner = {
     isSystemUser = true;
diff --git a/nixos/machines/railbird-sf.nix b/nixos/machines/railbird-sf.nix
index ad85d7c2..011dcaf7 100644
--- a/nixos/machines/railbird-sf.nix
+++ b/nixos/machines/railbird-sf.nix
@@ -55,6 +55,23 @@
   myModules.postgres.enable = true;
   features.full.enable = true;
 
+  # CI on this box grows /nix unboundedly (per-commit `nix develop
+  # '.?submodules=1'` source copies + railbird-full CUDA image builds); the
+  # store had 20k dead paths / 40G+ reclaimable when this was added. Daily GC
+  # keeps the baseline down; min-free makes nix GC mid-build when free space
+  # drops below 50G instead of running into ENOSPC. (The CI postgres DiskFull
+  # failures were a separate issue — the runner's private 3.2G tmpfs /tmp;
+  # see gitea-runner.nix TMPDIR.)
+  nix.gc = {
+    automatic = true;
+    dates = "daily";
+    options = "--delete-older-than 7d";
+  };
+  nix.settings = {
+    min-free = 50 * 1024 * 1024 * 1024; # trigger GC below 50G free
+    max-free = 150 * 1024 * 1024 * 1024; # GC until 150G free
+  };
+
   hardware.nvidia = {
     powerManagement.enable = false;
     # Fine-grained power management. Turns off GPU when not in use.