From 591e87364d95a38d7f89cc607ccdafe1a20c357d Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Sat, 11 Jul 2026 15:42:57 -0700 Subject: [PATCH] Audit direnv Nix retention --- .../skills/disk-space-assessment/SKILL.md | 26 ++- .../references/direnv-gc-roots.md | 49 +++++ .../scripts/direnv_gc_roots_audit.py | 196 ++++++++++++++++++ 3 files changed, 262 insertions(+), 9 deletions(-) create mode 100644 dotfiles/agents/skills/disk-space-assessment/references/direnv-gc-roots.md create mode 100644 dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py diff --git a/dotfiles/agents/skills/disk-space-assessment/SKILL.md b/dotfiles/agents/skills/disk-space-assessment/SKILL.md index 4992bcd2..93c130a2 100644 --- a/dotfiles/agents/skills/disk-space-assessment/SKILL.md +++ b/dotfiles/agents/skills/disk-space-assessment/SKILL.md @@ -7,18 +7,18 @@ description: Measure and explain disk usage without deleting data, producing reu Build a reproducible picture of disk usage. Stop at findings and proposed actions; use `disk-space-cleanup` when the user authorizes remediation. -Read `references/ignore-paths.md` before scanning. Read `references/observed-heavy-hitters.md` when triaging this machine or interpreting familiar paths. +Read `references/ignore-paths.md` before scanning. Read `references/direnv-gc-roots.md` for Nix development-root attribution. Read `references/observed-heavy-hitters.md` when triaging this machine or interpreting familiar paths. ## Required Artifact Contract Always create a reusable, timestamped `ncdu` export for every filesystem or major root assessed. Never make an interactive `ncdu` session, transient `/tmp` export, or `du` output the only record of an assessment. -Use the local wrapper: +Always scan `/` with privilege. An unprivileged root scan is incomplete by design because it cannot measure private service state. Also scan each separately mounted pressured filesystem because `safe_ncdu -x` does not cross mount boundaries: ```bash +/run/wrappers/bin/sudo -n env HOME=/home/imalison safe_ncdu / safe_ncdu /home safe_ncdu /nix/store -sudo -n env HOME=/home/imalison safe_ncdu / ``` `safe_ncdu` writes these durable artifacts under `~/.cache/ncdu/`: @@ -33,7 +33,7 @@ Keep timestamped files intact for iterative analysis. Report their absolute path ## Workflow 1. Record filesystem pressure and topology. -2. Choose scan roots that cover the pressured filesystem without crossing mounts. +2. Run a privileged `/` scan and choose additional roots for separately mounted filesystems. 3. Create reusable `safe_ncdu` snapshots before any cleanup. 4. Analyze snapshots repeatedly with `top` and `open`; do not rescan for every question. 5. Attribute special stores such as Nix separately. @@ -52,12 +52,12 @@ Record used/free space and whether `/home`, `/nix`, or other large paths are sep ## 2. Select Scan Roots -Prefer one-filesystem coverage: +Require one-filesystem coverage: -- Scan `/` for root accounting. +- Always scan `/` with `/run/wrappers/bin/sudo -n env HOME=/home/imalison`; never substitute an unprivileged root scan. On NixOS, use the setuid wrapper explicitly because a non-setuid `sudo` store binary may appear earlier on `PATH`. - Scan separately mounted `/home` and `/nix/store` independently. - Add a focused root such as `~/Projects` when the first snapshot identifies it as dominant. -- Use privileged root scans when unprivileged results undercount private service state. +- Treat a failed privileged root scan as an explicit coverage gap; do not silently fall back to an unprivileged scan. Inspect exclusions before a long scan: @@ -72,14 +72,14 @@ Update `references/ignore-paths.md` and the implementation of `safe_ncdu` togeth Run scans early enough that cleanup does not destroy the evidence: ```bash +/run/wrappers/bin/sudo -n env HOME=/home/imalison safe_ncdu / safe_ncdu /home safe_ncdu /nix/store -sudo -n env HOME=/home/imalison safe_ncdu / ``` If `safe_ncdu` is unavailable, source or run `/srv/dotfiles/dotfiles/lib/functions/safe_ncdu`. If `ncdu` itself is missing, use Nix temporarily rather than substituting a non-reusable interactive scan. -Do not store the privileged scan in root's home. Set `HOME=/home/imalison` so all artifacts remain together and are available to later sessions. +Do not store the privileged scan in root's home. Set `HOME=/home/imalison` so all artifacts remain together and are available to later sessions. Restore ownership to `imalison:users` if `sudo` creates root-owned snapshot artifacts. ## 4. Analyze Iteratively @@ -113,6 +113,14 @@ nix-store --gc --print-roots Use `/srv/dotfiles/dotfiles/lib/functions/find_store_path_gc_roots` and `nix why-depends` to explain why a large path is retained. Inspect `.direnv/flake-profile-*`, `result*` symlinks, system generations, and current/booted system closures. Prefer `nix_store_audit` over an initial `du -sh /nix/store`, which is slow and does not explain retention. +Always create a reusable `.direnv` GC-root audit artifact when direnv roots exist: + +```bash +python /srv/dotfiles/dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py --top 30 +``` + +This separates collectively direnv-only paths from paths retained by non-direnv roots and estimates each project's marginal uniquely retained footprint. Read `references/direnv-gc-roots.md` before interpreting or acting on the result. + ## Assessment Handoff Return: diff --git a/dotfiles/agents/skills/disk-space-assessment/references/direnv-gc-roots.md b/dotfiles/agents/skills/disk-space-assessment/references/direnv-gc-roots.md new file mode 100644 index 00000000..a759f737 --- /dev/null +++ b/dotfiles/agents/skills/disk-space-assessment/references/direnv-gc-roots.md @@ -0,0 +1,49 @@ +# Auditing `.direnv` Nix GC Roots + +Audit direnv roots as a retention graph, not as a list of symlink sizes. + +## Required Audit + +Run: + +```bash +python /srv/dotfiles/dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py --top 30 +``` + +The script writes a timestamped JSON artifact under `~/.cache/ncdu/` and updates `latest-direnv-gc-roots.json`. Preserve the timestamped artifact with the related ncdu snapshots. + +## Interpret the Measures + +- **Raw roots**: every `.direnv` entry reported by `nix-store --gc --print-roots`; several may point to the same store target. +- **Unique targets**: deduplicated store paths directly referenced by those roots. +- **Project closure**: union of the transitive closures of one project's direnv targets. +- **Collectively direnv-only**: direnv closure paths absent from the union of every observed non-direnv root closure. Removing all direnv roots could make these collectible. +- **Outside non-direnv roots**: one project's closure after subtracting non-direnv-retained paths. This remains an upper bound because other direnv projects may share it. +- **Marginal unique**: paths retained by exactly one direnv project and no non-direnv root. This is the best logical NAR-size estimate of what removing only that project's `.direnv` could make collectible. + +Rank projects by marginal unique size, then inspect age, worktree state, active shells/builds, and project importance. Large closure size with near-zero marginal unique size indicates heavy sharing and little immediate benefit from removing that project alone. + +## Validate Candidates + +For a candidate project: + +```bash +find /.direnv -maxdepth 1 -type l -printf '%TY-%Tm-%Td %TH:%TM %p -> %l\n' | sort +git -C status --short +git worktree list --porcelain +ps aux | rg '|direnv|nix develop|nix-shell' +``` + +Resolve a surprising retained store path with: + +```bash +/srv/dotfiles/dotfiles/lib/functions/find_store_path_gc_roots /nix/store/ +nix why-depends /nix/store/ +``` + +## Limitations + +- NAR sizes are logical database sizes, not guaranteed physical bytes reclaimed. +- The graph is a point-in-time view; shells and agents can add or replace roots during the run. +- Nix paths may also be retained by roots created immediately after enumeration. +- Run the audit immediately before and after any cleanup campaign and use actual `df` change as the final measure. diff --git a/dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py b/dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py new file mode 100644 index 00000000..0ee488d2 --- /dev/null +++ b/dotfiles/agents/skills/disk-space-assessment/scripts/direnv_gc_roots_audit.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +import argparse +import collections +import datetime as dt +import json +import os +import socket +import sqlite3 +import subprocess +from pathlib import Path + + +DB_PATH = Path("/nix/var/nix/db/db.sqlite") + + +def run(*args: str) -> str: + return subprocess.run(args, check=True, text=True, capture_output=True).stdout + + +def chunks(values: list[str], size: int = 100): + for index in range(0, len(values), size): + yield values[index : index + size] + + +def closure(targets: set[str]) -> set[str]: + result: set[str] = set() + existing = sorted(target for target in targets if Path(target).exists()) + for batch in chunks(existing): + result.update(line for line in run("nix-store", "-qR", *batch).splitlines() if line) + return result + + +def sizes(paths: set[str]) -> dict[str, int]: + if not paths: + return {} + connection = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True) + try: + result: dict[str, int] = {} + ordered = sorted(paths) + for batch in chunks(ordered, 900): + placeholders = ",".join("?" for _ in batch) + query = f"select path, narSize from ValidPaths where path in ({placeholders})" + result.update({path: int(size or 0) for path, size in connection.execute(query, batch)}) + return result + finally: + connection.close() + + +def human_size(value: int) -> str: + amount = float(value) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if amount < 1024 or unit == "TiB": + return f"{amount:.2f} {unit}" + amount /= 1024 + raise AssertionError("unreachable") + + +def parse_roots() -> list[tuple[str, str]]: + entries: list[tuple[str, str]] = [] + for line in run("nix-store", "--gc", "--print-roots").splitlines(): + if " -> " not in line: + continue + source, target = line.split(" -> ", 1) + source = source.strip('"') + target = target.strip('"') + if target.startswith("/nix/store/"): + entries.append((source, target)) + return entries + + +def project_for_root(source: str) -> str | None: + marker = "/.direnv/" + if marker not in source: + return None + return source.split(marker, 1)[0] + + +def top_paths(paths: set[str], path_sizes: dict[str, int], limit: int) -> list[dict]: + ranked = sorted(paths, key=lambda path: path_sizes.get(path, 0), reverse=True)[:limit] + return [{"path": path, "nar_size_bytes": path_sizes.get(path, 0)} for path in ranked] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Audit Nix paths retained by .direnv GC roots.") + parser.add_argument("--top", type=int, default=30, help="Projects and store paths to show.") + parser.add_argument("--output", help="JSON artifact path; defaults under ~/.cache/ncdu.") + args = parser.parse_args() + if args.top < 1: + parser.error("--top must be positive") + if not DB_PATH.is_file(): + parser.error(f"Nix database not found: {DB_PATH}") + + generated_at = dt.datetime.now().astimezone() + roots = parse_roots() + grouped: dict[str, list[tuple[str, str]]] = collections.defaultdict(list) + non_direnv_targets: set[str] = set() + for source, target in roots: + project = project_for_root(source) + if project is None: + non_direnv_targets.add(target) + else: + grouped[project].append((source, target)) + + project_closures = { + project: closure({target for _source, target in entries}) + for project, entries in sorted(grouped.items()) + } + non_direnv_closure = closure(non_direnv_targets) + all_direnv_closure = set().union(*project_closures.values()) if project_closures else set() + direnv_only = all_direnv_closure - non_direnv_closure + + membership: collections.Counter[str] = collections.Counter() + for project_paths in project_closures.values(): + membership.update(project_paths) + + all_relevant = all_direnv_closure | non_direnv_closure + path_sizes = sizes(all_relevant) + total = lambda paths: sum(path_sizes.get(path, 0) for path in paths) + + projects = [] + for project, entries in grouped.items(): + project_paths = project_closures[project] + outside_non_direnv = project_paths - non_direnv_closure + marginal = {path for path in outside_non_direnv if membership[path] == 1} + direnv_path = Path(project) / ".direnv" + mtime = None + age_days = None + if direnv_path.exists(): + timestamp = direnv_path.stat().st_mtime + mtime = dt.datetime.fromtimestamp(timestamp).astimezone().isoformat() + age_days = (generated_at.timestamp() - timestamp) / 86400 + projects.append( + { + "project": project, + "direnv_mtime": mtime, + "direnv_age_days": age_days, + "raw_root_count": len(entries), + "unique_target_count": len({target for _source, target in entries}), + "closure_path_count": len(project_paths), + "closure_nar_bytes": total(project_paths), + "outside_non_direnv_nar_bytes": total(outside_non_direnv), + "marginal_unique_nar_bytes": total(marginal), + "roots": [{"source": source, "target": target} for source, target in entries], + "top_marginal_paths": top_paths(marginal, path_sizes, args.top), + } + ) + projects.sort(key=lambda item: (item["marginal_unique_nar_bytes"], item["closure_nar_bytes"]), reverse=True) + + artifact = { + "format_version": 1, + "generated_at": generated_at.isoformat(), + "hostname": socket.gethostname(), + "measurement": "logical NAR size from the Nix database", + "raw_gc_root_count": len(roots), + "raw_direnv_root_count": sum(len(entries) for entries in grouped.values()), + "unique_direnv_target_count": len({target for entries in grouped.values() for _source, target in entries}), + "direnv_project_count": len(grouped), + "all_direnv_closure_nar_bytes": total(all_direnv_closure), + "collectively_direnv_only_nar_bytes": total(direnv_only), + "collectively_direnv_only_path_count": len(direnv_only), + "top_collectively_direnv_only_paths": top_paths(direnv_only, path_sizes, args.top), + "projects": projects, + } + + out_dir = Path.home() / ".cache" / "ncdu" + out_dir.mkdir(parents=True, exist_ok=True) + output = Path(args.output).expanduser() if args.output else out_dir / f"direnv-gc-roots-{generated_at:%Y%m%d-%H%M%S}.json" + output = output.resolve() + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(artifact, indent=2) + "\n") + os.replace(temporary, output) + latest = out_dir / "latest-direnv-gc-roots.json" + latest.unlink(missing_ok=True) + latest.symlink_to(output) + + print(f"Direnv GC-root artifact: {output}") + print(f"Raw direnv roots: {artifact['raw_direnv_root_count']}") + print(f"Unique direnv targets: {artifact['unique_direnv_target_count']}") + print(f"Projects: {artifact['direnv_project_count']}") + print(f"All direnv closures: {human_size(artifact['all_direnv_closure_nar_bytes'])}") + print(f"Collectively direnv-only: {human_size(artifact['collectively_direnv_only_nar_bytes'])}") + print() + print(f"{'MARGINAL':>11} {'CLOSURE':>11} {'AGE(d)':>8} PROJECT") + for item in projects[: args.top]: + age = "?" if item["direnv_age_days"] is None else f"{item['direnv_age_days']:.1f}" + print( + f"{human_size(item['marginal_unique_nar_bytes']):>11} " + f"{human_size(item['closure_nar_bytes']):>11} {age:>8} {item['project']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())