bootstrap_retention.py (23081B)
1 #!/usr/bin/env python3 2 """Finish a verified compiler checkpoint: plan, isolated checks, prune, recheck. 3 4 Linux/Haskell only. Trusted local scratch trees, not adversarial filesystems. 5 Only objects/ directories under an explicit .bootstrap-run root are removed. 6 No archive discovery/deletion, no age-based compiler selection, no Nix GC. 7 The config is an explicit retention registry; see compiler-rebuilding.md. 8 """ 9 import argparse 10 import collections 11 from contextlib import ExitStack 12 import fcntl 13 import hashlib 14 import json 15 import os 16 from pathlib import Path 17 import re 18 import shutil 19 import stat 20 import subprocess 21 import sys 22 import time 23 24 from bootstrap_retention_lock import acquire 25 26 REPO = Path(__file__).resolve().parent.parent 27 SCHEMA = 'tricu.bootstrap-retention.v1' 28 29 30 def sha(path): 31 h = hashlib.sha256() 32 with Path(path).open('rb') as f: 33 for block in iter(lambda: f.read(1024 * 1024), b''): 34 h.update(block) 35 return h.hexdigest() 36 37 38 def save(path, value): 39 temporary = path.with_name(path.name + '.tmp') 40 with temporary.open('w') as f: 41 json.dump(value, f, indent=2); f.write('\n'); f.flush(); os.fsync(f.fileno()) 42 os.replace(temporary, path) 43 fd = os.open(path.parent, os.O_DIRECTORY) 44 try: 45 os.fsync(fd) 46 finally: 47 os.close(fd) 48 49 50 def within(root, relative, exists=True): 51 p = Path(relative) 52 if p.is_absolute() or not p.parts or any(x in ('.', '..') for x in p.parts): 53 raise ValueError('expected normalized scratch-relative path: ' + str(relative)) 54 result = root / p 55 if str(p) != relative or result.resolve() != result: 56 raise ValueError('symlink/non-normalized path: ' + relative) 57 if exists and not result.exists(): 58 raise ValueError('missing path: ' + relative) 59 return result 60 61 62 def files(path): 63 for parent, dirs, names in os.walk(path, followlinks=False): 64 for name in dirs + names: 65 p = Path(parent) / name 66 s = p.lstat() 67 if stat.S_ISLNK(s.st_mode): 68 # Retained cohorts can contain Nix runtime roots; record links, 69 # never traverse or modify their targets. 70 if name in dirs: 71 dirs.remove(name) 72 yield p 73 elif not (stat.S_ISREG(s.st_mode) or stat.S_ISDIR(s.st_mode)): 74 raise ValueError('special file: ' + str(p)) 75 for name in names: 76 yield Path(parent) / name 77 for name in dirs[:]: 78 p = Path(parent) / name 79 if p.is_symlink(): 80 dirs.remove(name) 81 yield p 82 83 84 def identity(p): 85 s = p.lstat() 86 return [s.st_dev, s.st_ino, s.st_mode, s.st_size, s.st_mtime_ns, 87 s.st_nlink, s.st_blocks * 512, os.readlink(p) if p.is_symlink() else None] 88 89 90 def inventory(path): 91 return {str(p.relative_to(path)): identity(p) for p in sorted(files(path))} 92 93 94 def fingerprint(root, protected): 95 payload = {p: inventory(root / p) for p in protected} 96 # Unlinking old hardlinks must not count as modifying a retained object. 97 for records in payload.values(): 98 for value in records.values(): 99 value[5] = None 100 return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() 101 102 103 def load_config(root, path): 104 config = json.loads(path.read_text()) 105 if config.get('schema') != SCHEMA: 106 raise ValueError('invalid retention registry schema') 107 protected = config['protected'] 108 if not protected or len(set(protected)) != len(protected): 109 raise ValueError('explicit unique protected paths required') 110 for p in protected: 111 if not within(root, p).is_dir(): 112 raise ValueError('protected path is not a directory') 113 def covered(relative): 114 within(root, relative) 115 if not any(relative == p or relative.startswith(p + '/') for p in protected): 116 raise ValueError('registry input is not protected: ' + relative) 117 roles = [t['role'] for t in config['toolchains']] 118 if roles.count('current') != 1 or roles.count('rollback') != 1 or len(set(roles)) != len(roles): 119 raise ValueError('one current and one rollback toolchain required; roles must be unique') 120 for t in config['toolchains']: 121 if not re.fullmatch(r'[a-z][a-z0-9-]*', t['role']): 122 raise ValueError('unsafe toolchain role') 123 for key in ('generation', 'source', 'acceptance'): 124 covered(t[key]) 125 accepted = json.loads((root / t['acceptance']).read_text()) 126 if not accepted.get('accepted') or accepted.get('backend') != 'stable': 127 raise ValueError('accepted stable pair required') 128 if Path(accepted['generations'][1]).resolve() != root / t['generation']: 129 raise ValueError('acceptance must select this second generation; relocate acceptance first') 130 for generation in accepted['generations']: 131 path = Path(generation).resolve() 132 if not path.is_relative_to(root): 133 raise ValueError('retained acceptance pair must be relocated inside scratch root') 134 covered(str(path.relative_to(root))) 135 runtime = Path(t['runtime']).resolve() 136 if not str(runtime).startswith('/nix/store/') or runtime.name != 'tricu': 137 raise ValueError('explicit Nix-built Haskell tricu runtime required') 138 if sha(runtime) != accepted['inputs'][1]['runtime_sha256']: 139 raise ValueError('accepted runtime identity mismatch') 140 for name, digest in accepted['artifacts'].items(): 141 if sha(root / t['generation'] / name) != digest: 142 raise ValueError('accepted artifact mismatch') 143 for preparation in support_preparations(config['support']): 144 covered(preparation) 145 if 'source' in config['support']: 146 covered(config['support']['source']) 147 check_support_source(root, config['support']) 148 if config['support']['toolchain'] not in roles: 149 raise ValueError('unknown support toolchain') 150 if not config.get('sibling_users_checked') or not config.get('uncoordinated_jobs_stopped'): 151 raise ValueError('confirm sibling users and stop uncoordinated jobs before planning') 152 return config 153 154 155 def support_preparations(support): 156 """Return the ordered, explicitly selected preparation directories.""" 157 extra = support.get('additional_preparations', []) 158 if not isinstance(extra, list): 159 raise ValueError('additional preparations must be a list') 160 paths = [support['preparation'], *extra] 161 if any(not isinstance(path, str) or not path for path in paths) or len(set(paths)) != len(paths): 162 raise ValueError('unique preparation paths required') 163 return paths 164 165 166 def check_support_source(root, support): 167 from fixture_sources import current_mismatches, verify 168 source = within(root, support['source']) if 'source' in support else REPO 169 if not source.is_dir(): 170 raise ValueError('support source must be a directory') 171 for _, _, _, path, _ in verify(REPO / 'test/fixtures'): 172 within(source, path) 173 within(source, 'bootstrap/compiler-source-map.v1') 174 changed = current_mismatches(REPO / 'test/fixtures', source) 175 if changed: 176 raise ValueError('selected support source differs from current fixture provenance: ' + ', '.join(changed)) 177 return source 178 179 180 def object_directories(root, protected): 181 found = [] 182 for parent, dirs, _ in os.walk(root, followlinks=False): 183 parent = Path(parent) 184 relative = str(parent.relative_to(root)) 185 if any(relative == p or relative.startswith(p + '/') for p in protected): 186 dirs[:] = []; continue 187 for name in dirs[:]: 188 p = parent / name 189 if p.is_symlink(): 190 if name == 'objects': 191 raise ValueError('symlink objects directory: ' + str(p)) 192 dirs.remove(name) 193 elif name == 'objects': 194 found.append(str(p.relative_to(root))); dirs.remove(name) 195 return sorted(found) 196 197 198 def make_plan(root, config, out): 199 protected = config['protected'] 200 harness = out / 'harness'; harness.mkdir() 201 for name in ['bootstrap_retention.py', 'bootstrap_retention_probe.py', 'bootstrap_retention_lock.py']: 202 shutil.copyfile(REPO / 'test' / name, harness / name) 203 candidates = object_directories(root, protected + [str(out.relative_to(root))]) 204 entries = [] 205 metadata = out / 'historical-metadata'; metadata.mkdir() 206 deleting = collections.Counter(); allocated = {}; all_links = collections.Counter() 207 for p in files(root): 208 s = p.lstat() 209 if stat.S_ISREG(s.st_mode): 210 all_links[s.st_dev, s.st_ino] += 1 211 allocated[s.st_dev, s.st_ino] = (s.st_blocks * 512, s.st_nlink) 212 for relative in candidates: 213 target = within(root, relative) 214 records = inventory(target) 215 evidence = [] 216 for name, s in records.items(): 217 p = target / name 218 if not stat.S_ISREG(s[2]): 219 raise ValueError('symlink/non-file inside objects: ' + str(p)) 220 deleting[s[0], s[1]] += 1 221 with p.open('rb') as f: 222 head = f.read(128) 223 domain = head.split(b'\n', 1)[0] 224 if re.fullmatch(rb'arboricx\.[a-zA-Z0-9_.-]+', domain) and b'\n' in head: 225 digest = sha(p); saved = metadata / digest 226 if not saved.exists(): 227 shutil.copyfile(p, saved) 228 evidence.append({'path': name, 'sha256': digest, 'domain': domain.decode()}) 229 entries.append(dict(objects=relative, files=records, metadata=evidence, state='pending')) 230 reclaim = sum(allocated[k][0] for k, count in deleting.items() 231 if count == all_links[k] == allocated[k][1]) 232 return dict(schema=SCHEMA, root=str(root), config=config, complete=False, 233 phase='planned', planned_at=time.time(), stores=entries, 234 reclaim_bytes=reclaim, free_bytes_before=shutil.disk_usage(root).free, 235 protected_fingerprint=fingerprint(root, protected), 236 tool_sha256=sha(Path(__file__)), probe_sha256=sha(REPO / 'test/bootstrap_retention_probe.py')) 237 238 239 def validate_plan(root, plan, out, same_tools=True): 240 if plan['schema'] != SCHEMA or plan['root'] != str(root): 241 raise ValueError('plan/root mismatch') 242 if same_tools: 243 if plan['tool_sha256'] != sha(Path(__file__)) or plan['probe_sha256'] != sha(REPO / 'test/bootstrap_retention_probe.py'): 244 raise ValueError('retention tooling changed; do not reuse this plan') 245 else: 246 # Completed historical evidence can be checked by newer tooling, but 247 # deletion/resumption must always use the original reviewed harness. 248 for name, key in [('bootstrap_retention.py', 'tool_sha256'), 249 ('bootstrap_retention_probe.py', 'probe_sha256')]: 250 if sha(out / 'harness' / name) != plan[key]: 251 raise ValueError('archived retention harness changed') 252 protected = plan['config']['protected'] 253 if fingerprint(root, protected) != plan['protected_fingerprint']: 254 raise ValueError('protected inputs changed since planning') 255 seen = set() 256 for row in plan['stores']: 257 relative = row['objects'] 258 target = within(root, relative, exists=False) 259 if target.name != 'objects' or relative in seen: 260 raise ValueError('invalid/duplicate deletion target') 261 seen.add(relative) 262 if any(target == root / p or target.is_relative_to(root / p) or (root / p).is_relative_to(target) 263 for p in protected + [str(out.relative_to(root))]): 264 raise ValueError('deletion intersects protected inputs') 265 actual = inventory(target) if target.exists() else {} 266 def unchanged(name, value): 267 old = row['files'].get(name) 268 if old is None: 269 return False 270 # Deletion of another planned hardlink may lower this count. 271 return value[:5] == old[:5] and 0 < value[5] <= old[5] and value[6:] == old[6:] 272 if row['state'] == 'pending' and (not target.is_dir() or set(actual) != set(row['files']) 273 or not all(unchanged(k, v) for k, v in actual.items())): 274 raise ValueError('candidate changed since planning: ' + relative) 275 if row['state'] == 'deleting' and not all(unchanged(k, v) for k, v in actual.items()): 276 raise ValueError('interrupted candidate changed: ' + relative) 277 if row['state'] == 'removed' and target.exists(): 278 raise ValueError('removed candidate reappeared: ' + relative) 279 if row['state'] not in ('pending', 'deleting', 'removed'): 280 raise ValueError('unknown deletion state') 281 for e in row['metadata']: 282 if not re.fullmatch('[a-f0-9]{64}', e['sha256']) or sha(out / 'historical-metadata' / e['sha256']) != e['sha256']: 283 raise ValueError('preserved metadata changed') 284 285 286 def isolated_checks(root, plan, out, phase): 287 destination = out / (phase + '-' + str(time.time_ns())) 288 destination.mkdir() 289 command = ['bwrap', '--die-with-parent', '--unshare-pid', '--ro-bind', '/', '/', 290 '--proc', '/proc', '--tmpfs', '/tmp', '--setenv', 'TMPDIR', '/tmp', 291 '--setenv', 'TMP', '/tmp', '--setenv', 'TEMP', '/tmp', '--tmpfs', str(root)] 292 for relative in plan['config']['protected']: 293 command += ['--ro-bind', str(root / relative), str(root / relative)] 294 command += ['--bind', str(out), str(out), '--setenv', 'PYTHONDONTWRITEBYTECODE', '1', 295 '--', sys.executable, str(REPO / 'test/bootstrap_retention_probe.py'), 296 '--root', str(root), '--plan', str(out / 'plan.json'), '--directory', str(destination)] 297 with (destination / 'checks.log').open('w') as log: 298 subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=3600) 299 report = json.loads((destination / 'report.json').read_text()) 300 if not report.get('complete') or not report.get('obsolete_stores_hidden'): 301 raise ValueError('incomplete isolated checks') 302 return str(destination.relative_to(out)) 303 304 305 def prune(root, plan, out): 306 # Validate ALL inputs before the first unlink. Revalidate on every resumption. 307 validate_plan(root, plan, out) 308 for row in plan['stores']: 309 if row['state'] == 'removed': 310 continue 311 target = root / row['objects'] 312 save(target.parent / 'PRUNED-CAS.json', dict(objects_removed=row['objects'], 313 retention_report=str(out / 'plan.json'), complete=False, resumable=False, 314 note='Historical records only; full CAS/pair audit is no longer available here.')) 315 row['state'] = 'deleting'; plan['phase'] = 'deleting'; save(out / 'plan.json', plan) 316 if target.exists(): 317 shutil.rmtree(target) 318 row['state'] = 'removed'; save(out / 'plan.json', plan) 319 validate_plan(root, plan, out) 320 plan['phase'] = 'deleted'; save(out / 'plan.json', plan) 321 322 323 def validate_check_reports(root, plan, out, phases): 324 for phase in phases: 325 report_path = within(out, plan[phase]) / 'report.json' 326 report = json.loads(report_path.read_text()) 327 if phase + '_sha256' in plan and sha(report_path) != plan[phase + '_sha256']: 328 raise ValueError('checkpoint check report changed') 329 if not all(report.get(k) for k in ['complete', 'obsolete_stores_hidden', 330 'support_receipts_reproduced']): 331 raise ValueError('checkpoint verification incomplete') 332 support = plan['config']['support'] 333 if 'source' in support: 334 check_support_source(root, support) 335 if (not report.get('fixture_sources_match_selected') or 336 report.get('fixture_source_scope') != 'frozen-reference' or 337 report.get('fixture_source') != support['source']): 338 raise ValueError('checkpoint support source selection mismatch') 339 elif not report.get('fixture_sources_current'): 340 raise ValueError('checkpoint working-source freshness incomplete') 341 expected_roles = {t['role'] for t in plan['config']['toolchains']} 342 if {t['role'] for t in report['toolchains']} != expected_roles or len(report['toolchains']) != len(expected_roles): 343 raise ValueError('checkpoint toolchain coverage mismatch') 344 for t in plan['config']['toolchains']: 345 checked = next(c for c in report['toolchains'] if c['role'] == t['role']) 346 accepted = json.loads((root / t['acceptance']).read_text()) 347 manifest = (root / t['generation'] / 'cas/cache/source-build/native_math').read_text().split('\t')[2] 348 if (checked['artifacts'] != accepted['artifacts'] or 349 checked['acceptance_sha256'] != sha(root / t['acceptance']) or 350 checked['runtime_sha256'] != sha(t['runtime']) or 351 checked['complete_results'] != 12 or not checked['portable_cli'] or 352 checked['scoped_map_manifest'] != manifest): 353 raise ValueError('checkpoint toolchain evidence mismatch') 354 if set(report['support_sha256']) != {'support-library.arboricx', 'support-rebuilt.arboricx', 'support-library.v1'}: 355 raise ValueError('checkpoint support coverage mismatch') 356 for name, digest in report['support_sha256'].items(): 357 if sha(REPO / 'test/fixtures' / name) != digest: 358 raise ValueError('checkpoint support publication changed') 359 360 361 def validate_completion(root, plan, out): 362 if not plan['complete'] or plan['phase'] != 'complete' or any(r['state'] != 'removed' for r in plan['stores']): 363 raise ValueError('checkpoint retention is pending, not complete') 364 validate_check_reports(root, plan, out, ['prechecks', 'postchecks']) 365 366 367 def checkpoint_main(locks): 368 p = argparse.ArgumentParser(description=__doc__) 369 p.add_argument('action', choices=['plan', 'apply', 'check']) 370 p.add_argument('--root', type=Path, required=True) 371 p.add_argument('--directory', type=Path, required=True, help='checkpoint evidence directory under root') 372 p.add_argument('--config', type=Path, help='explicit registry JSON, required for plan') 373 p.add_argument('--approve-plan', help='reviewed plan SHA256, required for apply') 374 args = p.parse_args() 375 root = args.root.absolute() 376 if root.name != '.bootstrap-run' or root.resolve() != root or not root.is_dir(): 377 p.error('explicit real .bootstrap-run directory required') 378 out = args.directory.absolute() 379 if out.resolve() != out or not out.is_relative_to(root) or out == root: 380 p.error('evidence directory must be real and inside root') 381 locks.enter_context(acquire(root, exclusive=True)) 382 # Also refuse pre-existing generation writers that predate root-lock support. 383 generation_locks = [] 384 for parent, dirs, names in os.walk(root, followlinks=False): 385 if '.generation.lock' in names: 386 path = Path(parent) / '.generation.lock' 387 if path.is_symlink(): 388 raise ValueError('symlink generation lock') 389 handle = locks.enter_context(path.open('r')) 390 fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) 391 generation_locks.append(handle) 392 if args.action == 'plan': 393 if not args.config: 394 p.error('--config required') 395 config = load_config(root, args.config.resolve()) 396 if any(out.is_relative_to(root / k) or (root / k).is_relative_to(out) for k in config['protected']): 397 raise ValueError('evidence directory intersects protected inputs') 398 out.mkdir(parents=True, exist_ok=False) 399 runtime_roots = {} 400 for t in config['toolchains']: 401 runtime_closure = Path(t['runtime']).resolve().parent.parent 402 runtime_root = out / ('runtime-' + t['role']) 403 subprocess.run(['nix-store', '--realise', str(runtime_closure), 404 '--add-root', str(runtime_root), '--indirect'], 405 check=True, capture_output=True, text=True, timeout=120) 406 runtime_roots[t['role']] = str(runtime_root) 407 plan = make_plan(root, config, out) 408 plan['runtime_roots'] = runtime_roots 409 save(out / 'plan.json', plan) 410 print(f'PLAN: {len(plan["stores"])} object directories; {plan["reclaim_bytes"] / 2**30:.2f} GiB reclaim estimate') 411 print('Review plan.json; apply requires --approve-plan ' + sha(out / 'plan.json')) 412 return 413 plan_path = out / 'plan.json' 414 if args.action == 'apply': 415 if args.approve_plan != sha(plan_path): 416 p.error('--approve-plan must equal the current reviewed plan SHA256 (also on resumption)') 417 plan = json.loads(plan_path.read_text()) 418 load_config(root, plan_path_for_config(out, plan['config'])) 419 validate_plan(root, plan, out, same_tools=args.action != 'check') 420 if args.action == 'check': 421 validate_completion(root, plan, out) 422 print('PASS: compiler checkpoint retention complete'); return 423 if plan['complete']: 424 print('Checkpoint retention already complete'); return 425 # A previously successful precheck remains valid only under unchanged inputs. 426 if 'prechecks' not in plan: 427 print('Running isolated pre-deletion checks (see checkpoint logs)...', flush=True) 428 plan['prechecks'] = isolated_checks(root, plan, out, 'before') 429 plan['prechecks_sha256'] = sha(out / plan['prechecks'] / 'report.json') 430 validate_plan(root, plan, out) 431 plan['phase'] = 'verified'; save(plan_path, plan) 432 validate_check_reports(root, plan, out, ['prechecks']) 433 print('Prechecks passed; pruning reviewed object directories...', flush=True) 434 prune(root, plan, out) 435 print('Running isolated post-deletion checks...', flush=True) 436 plan['postchecks'] = isolated_checks(root, plan, out, 'after') 437 plan['postchecks_sha256'] = sha(out / plan['postchecks'] / 'report.json') 438 validate_plan(root, plan, out) 439 plan.update(complete=True, phase='complete', completed_at=time.time(), 440 free_bytes_after=shutil.disk_usage(root).free) 441 plan['observed_free_bytes_gain'] = plan['free_bytes_after'] - plan['free_bytes_before'] 442 validate_completion(root, plan, out) 443 save(plan_path, plan) 444 print(f'COMPLETE: estimated unlink reclaim {plan["reclaim_bytes"] / 2**30:.2f} GiB; ' 445 f'observed filesystem free-space gain {plan["observed_free_bytes_gain"] / 2**30:.2f} GiB') 446 447 448 def plan_path_for_config(out, config): 449 # Snapshot for the validator/probe; never overwrite the user's registry. 450 path = out / 'registry.json' 451 if path.exists(): 452 if json.loads(path.read_text()) != config: 453 raise ValueError('registry snapshot changed') 454 else: 455 save(path, config) 456 return path 457 458 459 def main(): 460 with ExitStack() as locks: 461 checkpoint_main(locks) 462 463 464 if __name__ == '__main__': 465 main()