bootstrap_retention.py (21139B)
1 #!/usr/bin/env python3 2 """Finish a verified compiler checkpoint: plan, isolated checks, prune, recheck. 3 4 Linux/Haskell only. Trusted local scratch trees, not adversarial filesystems. 5 Only objects/ directories under an explicit .bootstrap-run root are removed. 6 No archive discovery/deletion, no age-based compiler selection, no Nix GC. 7 The config is an explicit retention registry; see compiler-rebuilding.md. 8 """ 9 import argparse 10 import collections 11 from contextlib import ExitStack 12 import fcntl 13 import hashlib 14 import json 15 import os 16 from pathlib import Path 17 import re 18 import shutil 19 import stat 20 import subprocess 21 import sys 22 import time 23 24 from bootstrap_retention_lock import acquire 25 26 REPO = Path(__file__).resolve().parent.parent 27 SCHEMA = 'tricu.bootstrap-retention.v1' 28 29 30 def sha(path): 31 h = hashlib.sha256() 32 with Path(path).open('rb') as f: 33 for block in iter(lambda: f.read(1024 * 1024), b''): 34 h.update(block) 35 return h.hexdigest() 36 37 38 def save(path, value): 39 temporary = path.with_name(path.name + '.tmp') 40 with temporary.open('w') as f: 41 json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno()) 42 os.replace(temporary, path) 43 fd = os.open(path.parent, os.O_DIRECTORY) 44 try: 45 os.fsync(fd) 46 finally: 47 os.close(fd) 48 49 50 def within(root, relative, exists=True): 51 p = Path(relative) 52 if p.is_absolute() or not p.parts or any(x in ('.', '..') for x in p.parts): 53 raise ValueError('expected normalized scratch-relative path: ' + str(relative)) 54 result = root / p 55 if str(p) != relative or result.resolve() != result: 56 raise ValueError('symlink/non-normalized path: ' + relative) 57 if exists and not result.exists(): 58 raise ValueError('missing path: ' + relative) 59 return result 60 61 62 def files(path): 63 for parent, dirs, names in os.walk(path, followlinks=False): 64 for name in dirs + names: 65 p = Path(parent) / name 66 s = p.lstat() 67 if stat.S_ISLNK(s.st_mode): 68 # Retained cohorts can contain Nix runtime roots; record links, 69 # never traverse or modify their targets. 70 if name in dirs: 71 dirs.remove(name) 72 yield p 73 elif not (stat.S_ISREG(s.st_mode) or stat.S_ISDIR(s.st_mode)): 74 raise ValueError('special file: ' + str(p)) 75 for name in names: 76 yield Path(parent) / name 77 for name in dirs[:]: 78 p = Path(parent) / name 79 if p.is_symlink(): 80 dirs.remove(name) 81 yield p 82 83 84 def identity(p): 85 s = p.lstat() 86 return [s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, 87 s.st_nlink, s.st_blocks * 512, os.readlink(p) if p.is_symlink() else None] 88 89 90 def inventory(path): 91 return {str(p.relative_to(path)): identity(p) for p in sorted(files(path))} 92 93 94 def fingerprint(root, protected): 95 payload = {p: inventory(root / p) for p in protected} 96 # Unlinking old hardlinks must not count as modifying a retained object. 97 for records in payload.values(): 98 for value in records.values(): 99 value[5] = None 100 return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() 101 102 103 def load_config(root, path): 104 config = json.loads(path.read_text()) 105 if config.get('schema') != SCHEMA: 106 raise ValueError('invalid retention registry schema') 107 protected = config['protected'] 108 if not protected or len(set(protected)) != len(protected): 109 raise ValueError('explicit unique protected paths required') 110 for p in protected: 111 if not within(root, p).is_dir(): 112 raise ValueError('protected path is not a directory') 113 def covered(relative): 114 within(root, relative) 115 if not any(relative == p or relative.startswith(p + '/') for p in protected): 116 raise ValueError('registry input is not protected: ' + relative) 117 roles = [t['role'] for t in config['toolchains']] 118 if roles.count('current') != 1 or roles.count('rollback') != 1 or len(set(roles)) != len(roles): 119 raise ValueError('one current and one rollback toolchain required; roles must be unique') 120 for t in config['toolchains']: 121 if not re.fullmatch(r'[a-z][a-z0-9-]*', t['role']): 122 raise ValueError('unsafe toolchain role') 123 for key in ('generation', 'source', 'acceptance'): 124 covered(t[key]) 125 accepted = json.loads((root / t['acceptance']).read_text()) 126 if not accepted.get('accepted') or accepted.get('backend') != 'stable': 127 raise ValueError('accepted stable pair required') 128 if Path(accepted['generations'][1]).resolve() != root / t['generation']: 129 raise ValueError('acceptance must select this second generation; relocate acceptance first') 130 for generation in accepted['generations']: 131 path = Path(generation).resolve() 132 if not path.is_relative_to(root): 133 raise ValueError('retained acceptance pair must be relocated inside scratch root') 134 covered(str(path.relative_to(root))) 135 runtime = Path(t['runtime']).resolve() 136 if not str(runtime).startswith('/nix/store/') or runtime.name != 'tricu': 137 raise ValueError('explicit Nix-built Haskell tricu runtime required') 138 if sha(runtime) != accepted['inputs'][1]['runtime_sha256']: 139 raise ValueError('accepted runtime identity mismatch') 140 for name, digest in accepted['artifacts'].items(): 141 if sha(root / t['generation'] / name) != digest: 142 raise ValueError('accepted artifact mismatch') 143 covered(config['support']['preparation']) 144 if config['support']['toolchain'] not in roles: 145 raise ValueError('unknown support toolchain') 146 if not config.get('sibling_users_checked') or not config.get('uncoordinated_jobs_stopped'): 147 raise ValueError('confirm sibling users and stop uncoordinated jobs before planning') 148 return config 149 150 151 def object_directories(root, protected): 152 found = [] 153 for parent, dirs, _ in os.walk(root, followlinks=False): 154 parent = Path(parent) 155 relative = str(parent.relative_to(root)) 156 if any(relative == p or relative.startswith(p + '/') for p in protected): 157 dirs[:] = []; continue 158 for name in dirs[:]: 159 p = parent / name 160 if p.is_symlink(): 161 if name == 'objects': 162 raise ValueError('symlink objects directory: ' + str(p)) 163 dirs.remove(name) 164 elif name == 'objects': 165 found.append(str(p.relative_to(root))); dirs.remove(name) 166 return sorted(found) 167 168 169 def make_plan(root, config, out): 170 protected = config['protected'] 171 harness = out / 'harness'; harness.mkdir() 172 for name in ['bootstrap_retention.py', 'bootstrap_retention_probe.py', 'bootstrap_retention_lock.py']: 173 shutil.copyfile(REPO / 'test' / name, harness / name) 174 candidates = object_directories(root, protected + [str(out.relative_to(root))]) 175 entries = [] 176 metadata = out / 'historical-metadata'; metadata.mkdir() 177 deleting = collections.Counter(); allocated = {}; all_links = collections.Counter() 178 for p in files(root): 179 s = p.lstat() 180 if stat.S_ISREG(s.st_mode): 181 all_links[s.st_dev, s.st_ino] += 1 182 allocated[s.st_dev, s.st_ino] = (s.st_blocks * 512, s.st_nlink) 183 for relative in candidates: 184 target = within(root, relative) 185 records = inventory(target) 186 evidence = [] 187 for name, s in records.items(): 188 p = target / name 189 if not stat.S_ISREG(s[2]): 190 raise ValueError('symlink/non-file inside objects: ' + str(p)) 191 deleting[s[0], s[1]] += 1 192 with p.open('rb') as f: 193 head = f.read(128) 194 domain = head.split(b'\n', 1)[0] 195 if re.fullmatch(rb'arboricx\.[a-zA-Z0-9_.-]+', domain) and b'\n' in head: 196 digest = sha(p); saved = metadata / digest 197 if not saved.exists(): 198 shutil.copyfile(p, saved) 199 evidence.append({'path': name, 'sha256': digest, 'domain': domain.decode()}) 200 entries.append(dict(objects=relative, files=records, metadata=evidence, state='pending')) 201 reclaim = sum(allocated[k][0] for k, count in deleting.items() 202 if count == all_links[k] == allocated[k][1]) 203 return dict(schema=SCHEMA, root=str(root), config=config, complete=False, 204 phase='planned', planned_at=time.time(), stores=entries, 205 reclaim_bytes=reclaim, free_bytes_before=shutil.disk_usage(root).free, 206 protected_fingerprint=fingerprint(root, protected), 207 tool_sha256=sha(Path(__file__)), probe_sha256=sha(REPO / 'test/bootstrap_retention_probe.py')) 208 209 210 def validate_plan(root, plan, out, same_tools=True): 211 if plan['schema'] != SCHEMA or plan['root'] != str(root): 212 raise ValueError('plan/root mismatch') 213 if same_tools: 214 if plan['tool_sha256'] != sha(Path(__file__)) or plan['probe_sha256'] != sha(REPO / 'test/bootstrap_retention_probe.py'): 215 raise ValueError('retention tooling changed; do not reuse this plan') 216 else: 217 # Completed historical evidence can be checked by newer tooling, but 218 # deletion/resumption must always use the original reviewed harness. 219 for name, key in [('bootstrap_retention.py', 'tool_sha256'), 220 ('bootstrap_retention_probe.py', 'probe_sha256')]: 221 if sha(out / 'harness' / name) != plan[key]: 222 raise ValueError('archived retention harness changed') 223 protected = plan['config']['protected'] 224 if fingerprint(root, protected) != plan['protected_fingerprint']: 225 raise ValueError('protected inputs changed since planning') 226 seen = set() 227 for row in plan['stores']: 228 relative = row['objects'] 229 target = within(root, relative, exists=False) 230 if target.name != 'objects' or relative in seen: 231 raise ValueError('invalid/duplicate deletion target') 232 seen.add(relative) 233 if any(target == root / p or target.is_relative_to(root / p) or (root / p).is_relative_to(target) 234 for p in protected + [str(out.relative_to(root))]): 235 raise ValueError('deletion intersects protected inputs') 236 actual = inventory(target) if target.exists() else {} 237 def unchanged(name, value): 238 old = row['files'].get(name) 239 if old is None: 240 return False 241 # Deletion of another planned hardlink may lower this count. 242 return value[:5] == old[:5] and 0 < value[5] <= old[5] and value[6:] == old[6:] 243 if row['state'] == 'pending' and (not target.is_dir() or set(actual) != set(row['files']) 244 or not all(unchanged(k, v) for k, v in actual.items())): 245 raise ValueError('candidate changed since planning: ' + relative) 246 if row['state'] == 'deleting' and not all(unchanged(k, v) for k, v in actual.items()): 247 raise ValueError('interrupted candidate changed: ' + relative) 248 if row['state'] == 'removed' and target.exists(): 249 raise ValueError('removed candidate reappeared: ' + relative) 250 if row['state'] not in ('pending', 'deleting', 'removed'): 251 raise ValueError('unknown deletion state') 252 for e in row['metadata']: 253 if not re.fullmatch('[a-f0-9]{64}', e['sha256']) or sha(out / 'historical-metadata' / e['sha256']) != e['sha256']: 254 raise ValueError('preserved metadata changed') 255 256 257 def isolated_checks(root, plan, out, phase): 258 destination = out / (phase + '-' + str(time.time_ns())) 259 destination.mkdir() 260 command = ['bwrap', '--die-with-parent', '--unshare-pid', '--ro-bind', '/', '/', 261 '--proc', '/proc', '--tmpfs', '/tmp', '--tmpfs', str(root)] 262 for relative in plan['config']['protected']: 263 command += ['--ro-bind', str(root / relative), str(root / relative)] 264 command += ['--bind', str(out), str(out), '--setenv', 'PYTHONDONTWRITEBYTECODE', '1', 265 '--', sys.executable, str(REPO / 'test/bootstrap_retention_probe.py'), 266 '--root', str(root), '--plan', str(out / 'plan.json'), '--directory', str(destination)] 267 with (destination / 'checks.log').open('w') as log: 268 subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=3600) 269 report = json.loads((destination / 'report.json').read_text()) 270 if not report.get('complete') or not report.get('obsolete_stores_hidden'): 271 raise ValueError('incomplete isolated checks') 272 return str(destination.relative_to(out)) 273 274 275 def prune(root, plan, out): 276 # Validate ALL inputs before the first unlink. Revalidate on every resumption. 277 validate_plan(root, plan, out) 278 for row in plan['stores']: 279 if row['state'] == 'removed': 280 continue 281 target = root / row['objects'] 282 save(target.parent / 'PRUNED-CAS.json', dict(objects_removed=row['objects'], 283 retention_report=str(out / 'plan.json'), complete=False, resumable=False, 284 note='Historical records only; full CAS/pair audit is no longer available here.')) 285 row['state'] = 'deleting'; plan['phase'] = 'deleting'; save(out / 'plan.json', plan) 286 if target.exists(): 287 shutil.rmtree(target) 288 row['state'] = 'removed'; save(out / 'plan.json', plan) 289 validate_plan(root, plan, out) 290 plan['phase'] = 'deleted'; save(out / 'plan.json', plan) 291 292 293 def validate_check_reports(root, plan, out, phases): 294 for phase in phases: 295 report_path = within(out, plan[phase]) / 'report.json' 296 report = json.loads(report_path.read_text()) 297 if phase + '_sha256' in plan and sha(report_path) != plan[phase + '_sha256']: 298 raise ValueError('checkpoint check report changed') 299 if not all(report.get(k) for k in ['complete', 'obsolete_stores_hidden', 300 'support_receipts_reproduced', 'fixture_sources_current']): 301 raise ValueError('checkpoint verification incomplete') 302 expected_roles = {t['role'] for t in plan['config']['toolchains']} 303 if {t['role'] for t in report['toolchains']} != expected_roles or len(report['toolchains']) != len(expected_roles): 304 raise ValueError('checkpoint toolchain coverage mismatch') 305 for t in plan['config']['toolchains']: 306 checked = next(c for c in report['toolchains'] if c['role'] == t['role']) 307 accepted = json.loads((root / t['acceptance']).read_text()) 308 manifest = (root / t['generation'] / 'cas/cache/source-build/native_math').read_text().split('\t')[2] 309 if (checked['artifacts'] != accepted['artifacts'] or 310 checked['acceptance_sha256'] != sha(root / t['acceptance']) or 311 checked['runtime_sha256'] != sha(t['runtime']) or 312 checked['complete_results'] != 12 or not checked['portable_cli'] or 313 checked['scoped_map_manifest'] != manifest): 314 raise ValueError('checkpoint toolchain evidence mismatch') 315 if set(report['support_sha256']) != {'support-library.arboricx', 'support-rebuilt.arboricx', 'support-library.v1'}: 316 raise ValueError('checkpoint support coverage mismatch') 317 for name, digest in report['support_sha256'].items(): 318 if sha(REPO / 'test/fixtures' / name) != digest: 319 raise ValueError('checkpoint support publication changed') 320 321 322 def validate_completion(root, plan, out): 323 if not plan['complete'] or plan['phase'] != 'complete' or any(r['state'] != 'removed' for r in plan['stores']): 324 raise ValueError('checkpoint retention is pending, not complete') 325 validate_check_reports(root, plan, out, ['prechecks', 'postchecks']) 326 327 328 def checkpoint_main(locks): 329 p = argparse.ArgumentParser(description=__doc__) 330 p.add_argument('action', choices=['plan', 'apply', 'check']) 331 p.add_argument('--root', type=Path, required=True) 332 p.add_argument('--directory', type=Path, required=True, help='checkpoint evidence directory under root') 333 p.add_argument('--config', type=Path, help='explicit registry JSON, required for plan') 334 p.add_argument('--approve-plan', help='reviewed plan SHA256, required for apply') 335 args = p.parse_args() 336 root = args.root.absolute() 337 if root.name != '.bootstrap-run' or root.resolve() != root or not root.is_dir(): 338 p.error('explicit real .bootstrap-run directory required') 339 out = args.directory.absolute() 340 if out.resolve() != out or not out.is_relative_to(root) or out == root: 341 p.error('evidence directory must be real and inside root') 342 locks.enter_context(acquire(root, exclusive=True)) 343 # Also refuse pre-existing generation writers that predate root-lock support. 344 generation_locks = [] 345 for parent, dirs, names in os.walk(root, followlinks=False): 346 if '.generation.lock' in names: 347 path = Path(parent) / '.generation.lock' 348 if path.is_symlink(): 349 raise ValueError('symlink generation lock') 350 handle = locks.enter_context(path.open('r')) 351 fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) 352 generation_locks.append(handle) 353 if args.action == 'plan': 354 if not args.config: 355 p.error('--config required') 356 config = load_config(root, args.config.resolve()) 357 if any(out.is_relative_to(root / k) or (root / k).is_relative_to(out) for k in config['protected']): 358 raise ValueError('evidence directory intersects protected inputs') 359 out.mkdir(parents=True, exist_ok=False) 360 runtime_roots = {} 361 for t in config['toolchains']: 362 runtime_closure = Path(t['runtime']).resolve().parent.parent 363 runtime_root = out / ('runtime-' + t['role']) 364 subprocess.run(['nix-store', '--realise', str(runtime_closure), 365 '--add-root', str(runtime_root), '--indirect'], 366 check=True, capture_output=True, text=True, timeout=120) 367 runtime_roots[t['role']] = str(runtime_root) 368 plan = make_plan(root, config, out) 369 plan['runtime_roots'] = runtime_roots 370 save(out / 'plan.json', plan) 371 print(f'PLAN: {len(plan["stores"])} object directories; {plan["reclaim_bytes"] / 2**30:.2f} GiB reclaim estimate') 372 print('Review plan.json; apply requires --approve-plan ' + sha(out / 'plan.json')) 373 return 374 plan_path = out / 'plan.json' 375 if args.action == 'apply': 376 if args.approve_plan != sha(plan_path): 377 p.error('--approve-plan must equal the current reviewed plan SHA256 (also on resumption)') 378 plan = json.loads(plan_path.read_text()) 379 load_config(root, plan_path_for_config(out, plan['config'])) 380 validate_plan(root, plan, out, same_tools=args.action != 'check') 381 if args.action == 'check': 382 validate_completion(root, plan, out) 383 print('PASS: compiler checkpoint retention complete'); return 384 if plan['complete']: 385 print('Checkpoint retention already complete'); return 386 # A previously successful precheck remains valid only under unchanged inputs. 387 if 'prechecks' not in plan: 388 print('Running isolated pre-deletion checks (see checkpoint logs)...', flush=True) 389 plan['prechecks'] = isolated_checks(root, plan, out, 'before') 390 plan['prechecks_sha256'] = sha(out / plan['prechecks'] / 'report.json') 391 validate_plan(root, plan, out) 392 plan['phase'] = 'verified'; save(plan_path, plan) 393 validate_check_reports(root, plan, out, ['prechecks']) 394 print('Prechecks passed; pruning reviewed object directories...', flush=True) 395 prune(root, plan, out) 396 print('Running isolated post-deletion checks...', flush=True) 397 plan['postchecks'] = isolated_checks(root, plan, out, 'after') 398 plan['postchecks_sha256'] = sha(out / plan['postchecks'] / 'report.json') 399 validate_plan(root, plan, out) 400 plan.update(complete=True, phase='complete', completed_at=time.time(), 401 free_bytes_after=shutil.disk_usage(root).free) 402 plan['observed_free_bytes_gain'] = plan['free_bytes_after'] - plan['free_bytes_before'] 403 validate_completion(root, plan, out) 404 save(plan_path, plan) 405 print(f'COMPLETE: estimated unlink reclaim {plan["reclaim_bytes"] / 2**30:.2f} GiB; ' 406 f'observed filesystem free-space gain {plan["observed_free_bytes_gain"] / 2**30:.2f} GiB') 407 408 409 def plan_path_for_config(out, config): 410 # Snapshot for the validator/probe; never overwrite the user's registry. 411 path = out / 'registry.json' 412 if path.exists(): 413 if json.loads(path.read_text()) != config: 414 raise ValueError('registry snapshot changed') 415 else: 416 save(path, config) 417 return path 418 419 420 def main(): 421 with ExitStack() as locks: 422 checkpoint_main(locks) 423 424 425 if __name__ == '__main__': 426 main()