diff --git a/nixos/gitea-runner.nix b/nixos/gitea-runner.nix index 4c010fb6..4a47534f 100644 --- a/nixos/gitea-runner.nix +++ b/nixos/gitea-runner.nix @@ -58,9 +58,21 @@ makeEnable config "myModules.gitea-runner" false { in { XDG_CONFIG_HOME = gitea-runner-directory; XDG_CACHE_HOME = "${gitea-runner-directory}/.cache"; + # Despite PrivateTmp=false below, the upstream module's DynamicUser=true + # still gives the service a private /tmp: a tmpfs capped at 10% of RAM + # (~3.2G here) with 400k inodes. CI jobs inherit it — `nix develop` + # shell dirs (leaked on cancelled jobs) and pytest's ephemeral postgres + # (tempfile.mkdtemp) slowly fill it until tests die with + # DiskFull/ENOSPC while `df /` shows hundreds of GB free. Point job + # temp at the disk-backed state dir instead; tmpfiles rule below prunes + # leaked entries. + TMPDIR = "${gitea-runner-directory}/tmp"; }; serviceConfig.PrivateTmp = false; }; + systemd.tmpfiles.rules = [ + "d /var/lib/private/gitea-runner/tmp 0750 gitea-runner gitea-runner 2d" + ]; users.groups.gitea-runner = {}; users.users.gitea-runner = { isSystemUser = true; diff --git a/nixos/machines/railbird-sf.nix b/nixos/machines/railbird-sf.nix index ad85d7c2..011dcaf7 100644 --- a/nixos/machines/railbird-sf.nix +++ b/nixos/machines/railbird-sf.nix @@ -55,6 +55,23 @@ myModules.postgres.enable = true; features.full.enable = true; + # CI on this box grows /nix unboundedly (per-commit `nix develop + # '.?submodules=1'` source copies + railbird-full CUDA image builds); the + # store had 20k dead paths / 40G+ reclaimable when this was added. Daily GC + # keeps the baseline down; min-free makes nix GC mid-build when free space + # drops below 50G instead of running into ENOSPC. (The CI postgres DiskFull + # failures were a separate issue — the runner's private 3.2G tmpfs /tmp; + # see gitea-runner.nix TMPDIR.) + nix.gc = { + automatic = true; + dates = "daily"; + options = "--delete-older-than 7d"; + }; + nix.settings = { + min-free = 50 * 1024 * 1024 * 1024; # trigger GC below 50G free + max-free = 150 * 1024 * 1024 * 1024; # GC until 150G free + }; + hardware.nvidia = { powerManagement.enable = false; # Fine-grained power management. Turns off GPU when not in use.