bootstrap_archive.py (22901B)
1 #!/usr/bin/env python3 2 """Opt-in recovery archive creation; never compiles source or builds a generation. 3 4 Preserves a completed seed, matching historical source, current tracked working 5 files/history, checked compiler/support fixtures and an exact Nix runtime closure. 6 Only explicitly selected inputs are included; unrelated scratch runs are omitted. 7 """ 8 import argparse 9 import hashlib 10 import json 11 from pathlib import Path 12 import re 13 import shutil 14 import struct 15 import subprocess 16 import sys 17 import tarfile 18 import tempfile 19 20 ROOT = Path(__file__).resolve().parent.parent 21 22 23 def run(*args, cwd=ROOT): 24 return subprocess.check_output(list(map(str, args)), cwd=cwd, text=True, timeout=600) 25 26 27 def sha(path): 28 h = hashlib.sha256() 29 with path.open('rb') as f: 30 for chunk in iter(lambda: f.read(1024 * 1024), b''): 31 h.update(chunk) 32 return h.hexdigest() 33 34 35 def select_exports(bundle, destination, names): 36 """Re-label selected verified DAG roots without expanding logical trees.""" 37 from stable_bundle import assembled, sections 38 manifest, nodes = sections(bundle) 39 offset = 12 # ARBMNFST and version 40 41 def word(): 42 nonlocal offset 43 value = struct.unpack_from('>I', manifest, offset)[0]; offset += 4 44 return value 45 46 def string(): 47 nonlocal offset 48 size = word(); value = manifest[offset:offset + size]; offset += size 49 return value 50 51 def text(value): 52 value = value.encode() 53 return struct.pack('>I', len(value)) + value 54 55 for _ in range(9): 56 string() 57 for _ in range(word()): 58 string() 59 offset += 1 # closure mode 60 prefix = manifest[:offset] 61 for _ in range(word()): 62 word(); string() 63 exports = {} 64 for _ in range(word()): 65 name, index, kind, abi = string().decode(), word(), string(), string() 66 exports[name] = index 67 assert kind == b'term' and abi == b'arboricx.abi.tree.v1' 68 suffix = manifest[offset:] 69 selected = [(new, exports[old]) for old, new in names] 70 roots = struct.pack('>I', len(selected)) + b''.join( 71 struct.pack('>I', index) + text(name) for name, index in selected) 72 entries = struct.pack('>I', len(selected)) + b''.join( 73 text(name) + struct.pack('>I', index) + text('term') + text('arboricx.abi.tree.v1') 74 for name, index in selected) 75 destination.write_bytes(assembled(prefix + roots + entries + suffix, nodes)) 76 77 78 def recovery_profile(seed, inputs, adapter, accepted): 79 """Bind recovery execution to produced artifacts, never to interpreter success.""" 80 from generation_io import versions, produced_adapter 81 execution, output = versions(inputs) 82 if (execution, output) != (1, 1) and not accepted: 83 raise ValueError('IO recovery requires an independently accepted pair') 84 if output == 2 and adapter.read_bytes() != produced_adapter(seed, inputs): 85 raise ValueError('recovery adapter differs from produced adapter') 86 if accepted and sha(adapter) != inputs.get('produced_adapter_sha256', inputs['adapter_sha256']): 87 raise ValueError('recovery adapter differs from accepted output') 88 return output 89 90 91 def main(): 92 p = argparse.ArgumentParser(description=__doc__) 93 p.add_argument('--generation', type=Path, required=True) 94 p.add_argument('--seed-commit', required=True, help='commit matching every recorded seed source') 95 p.add_argument('--first-generation', type=Path, help='first fresh stable generation of an accepted pair') 96 p.add_argument('--acceptance', type=Path, help='strict acceptance JSON for that pair') 97 p.add_argument('--adapter', type=Path, required=True, help='standard six-argument retained adapter') 98 p.add_argument('--tricu', type=Path, required=True, help='Nix-built Haskell executable') 99 p.add_argument('--output', type=Path, required=True, help='new transfer directory') 100 a = p.parse_args() 101 if bool(a.first_generation) != bool(a.acceptance): 102 p.error('--first-generation and --acceptance must be selected together') 103 seed, tricu, adapter = a.generation.resolve(), a.tricu.resolve(), a.adapter.resolve() 104 first = a.first_generation.resolve() if a.first_generation else None 105 acceptance = None 106 if first: 107 from bootstrap_acceptance import verify 108 recorded = json.loads(a.acceptance.read_text()) 109 acceptance = verify(first, seed, tricu) 110 assert recorded.get('accepted') and recorded['inputs'] == acceptance['inputs'] 111 assert recorded['artifacts'] == acceptance['artifacts'], 'stale acceptance evidence' 112 status = json.loads((seed / 'status.json').read_text()) 113 assert status['complete'] and status['root'] == 'source_map', 'incomplete seed' 114 inputs = json.loads((seed / 'inputs.json').read_text()) 115 from generation_io import compiler_flags 116 io_version = recovery_profile(seed, inputs, adapter, bool(acceptance)) 117 rows = [r.split('\t') for r in (seed / 'source-map.v1').read_text().splitlines()[1:]] 118 for name in ('rebuilt-compiler.arboricx', 'build.arboricx', 'build-stable.arboricx', 'io-adapter.arboricx'): 119 assert (seed / name).is_file(), name 120 output = a.output.resolve() 121 # Pair extraction needs the two full CAS stores plus compressed output. 122 required = sum(f.stat().st_size for directory in ([seed, first] if first else [seed]) 123 for f in directory.rglob('*') if f.is_file()) 124 assert shutil.disk_usage(output.parent).free > required * 2 + 1024**3, 'insufficient archive/restore space' 125 output.mkdir(parents=True, exist_ok=False) 126 with tempfile.TemporaryDirectory(prefix='tricu-recovery-') as temp: 127 work = Path(temp) 128 payload = work / 'recovery'; payload.mkdir() 129 source = payload / 'source'; source.mkdir() 130 tracked = run('git', 'ls-files', '-z').split('\0') 131 for name in filter(None, tracked): 132 # Historical tracked Zig build-output link is not a recovery input. 133 if name == 'ext/zig/result': 134 continue 135 src = ROOT / name; dst = source / name 136 dst.parent.mkdir(parents=True, exist_ok=True) 137 if src.is_symlink(): 138 dst.symlink_to(src.readlink()) 139 else: 140 shutil.copy2(src, dst) 141 # Preserve this creator even before it is committed; mounts are not copied. 142 shutil.copy2(__file__, source / 'test/bootstrap_archive.py') 143 run('git', 'bundle', 'create', payload / 'history.bundle', '--all') 144 (payload / 'working-tree.patch').write_text(run('git', 'diff', 'HEAD', '--binary')) 145 historical = payload / 'seed-source'; historical.mkdir() 146 historical_tar = work / 'seed-source.tar' 147 with historical_tar.open('wb') as f: 148 subprocess.run(['git', 'archive', a.seed_commit], cwd=ROOT, stdout=f, check=True) 149 with tarfile.open(historical_tar) as tar: 150 tar.extractall(historical, members=(m for m in tar if m.name != 'ext/zig/result'), filter='data') 151 for _, name, _, path, _ in rows: 152 data = subprocess.check_output(['git', 'show', a.seed_commit + ':' + path], cwd=ROOT) 153 assert hashlib.sha256(data).hexdigest() == inputs['sources'][name], ('seed source mismatch', path) 154 dst = historical / path; dst.parent.mkdir(parents=True, exist_ok=True); dst.write_bytes(data) 155 audit = (acceptance['audits'][1] if acceptance else 156 run(sys.executable, historical / 'test/bootstrap_store_check.py', seed, '--source-root', historical)) 157 if acceptance: 158 (payload / 'acceptance.json').write_text(json.dumps(acceptance, indent=2) + '\n') 159 (payload / 'seed-audit.txt').write_text(audit) 160 print(audit, end='', flush=True) 161 fixtures = source / 'test/fixtures' 162 from fixture_sources import verify as verify_sources, current_mismatches 163 verify_sources(fixtures) 164 fixture_differences = current_mismatches(fixtures, source) 165 # Seed-only legacy packing still requires current compiler fixture sources. 166 if not acceptance: 167 from fixture_sources import source_rows 168 compiler_paths = {r[3] for r in source_rows(fixtures) if r[0] == 'compiler-library'} 169 if compiler_paths.intersection(fixture_differences): 170 raise ValueError('seed-only fixture compiler sources are not current') 171 tools = payload / 'tools'; tools.mkdir() 172 shutil.copy2(adapter, tools / 'io-adapter.arboricx') 173 shutil.copy2(seed / 'source-map.v1', tools / 'source-map.v1') 174 store = work / 'fixture-store' 175 selections = { 176 'build.arboricx': [('source_build.sourceBuildGraph', 'sourceBuildGraph')], 177 'build-stable.arboricx': [('source_build.sourceBuildGraphStable', 'sourceBuildGraph')], 178 'worker.arboricx': [('source_map.sourceMapBuild', 'sourceMapBuild'), 179 ('source_map.sourceMapBuildStable', 'sourceMapBuildStable')], 180 } 181 for filename, names in selections.items(): 182 if acceptance: 183 origin = 'rebuilt-compiler.arboricx' if filename == 'worker.arboricx' else filename 184 shutil.copy2(seed / origin, tools / filename) 185 else: 186 select_exports(fixtures / 'compiler-library.arboricx', tools / filename, names) 187 # The host independently verifies each rewritten manifest and DAG. 188 run(tricu, 'arboricx', 'import', '--file', tools / filename, 189 '--store', store / filename, '--indexed', '+RTS', '-N1', '-M512m', '-RTS') 190 runtime = tricu.parent.parent 191 assert str(runtime).startswith('/nix/store/'), 'runtime must be a Nix output' 192 closure = run('nix-store', '--query', '--requisites', runtime).splitlines() 193 (payload / 'runtime-paths.txt').write_text('\n'.join(closure) + '\n') 194 with (payload / 'runtime.export').open('wb') as f: 195 subprocess.run(['nix-store', '--export', *closure], stdout=f, check=True, timeout=600) 196 metadata = dict(format='tricu.bootstrap-recovery.v2' if io_version == 2 else 'tricu.bootstrap-recovery.v1', head=run('git', 'rev-parse', 'HEAD').strip(), 197 seed_commit=a.seed_commit, runtime=str(runtime), runtime_sha256=sha(tricu), 198 seed_inputs=inputs, selected_adapter_sha256=sha(adapter), 199 compiler_io_version=io_version, map_sha256=sha(tools / 'source-map.v1'), 200 fixture_source_scope='authenticated-reference', 201 working_source_differences=fixture_differences, 202 fixture_bundle_sha256=sha(fixtures / 'compiler-library.arboricx'), 203 accepted_stable_pair=bool(acceptance), 204 accepted_artifacts=acceptance['artifacts'] if acceptance else None, 205 limitations=(['Archive creation does not compile source.', 206 'Accepted compiler/tool artifacts copied from the reproduced second generation.'] 207 if acceptance else 208 ['No new compiler generation built.', 209 'Current tools packed from checked fixture, not a newly completed generation.']) + [ 210 'Tracked ext/zig/result build-output symlink omitted from source snapshots; retained in Git history.', 211 'Nix closure is x86_64-linux; no empty-Nix-store or cross-platform restore tested.']) 212 (payload / 'RECOVERY.json').write_text(json.dumps(metadata, indent=2) + '\n') 213 (payload / 'README.md').write_text(f'''# tricu bootstrap recovery 214 215 This is a recovery snapshot, not a new self-reproduction result. The completed 216 seed and its exact mapped source are preserved independently of current source. 217 Current tools are native-packed entrypoints from the checked compiler fixture; 218 its current source fingerprints are in source/test/fixtures/compiler-library.v1. 219 Those fingerprints alone do not prove header equivalence or a completed generation. 220 Original generation paths/provenance are deliberately not rewritten. 221 222 Prerequisites: x86_64 Linux, Nix installed at /nix, Python 3, Git, tar and sha256sum. 223 The runtime export is a runtime closure, not an offline Haskell build closure. 224 225 ## Restore and inspect 226 227 Verify the transfer directory's TRANSFER-SHA256SUMS before extracting: 228 229 ```sh 230 sha256sum -c TRANSFER-SHA256SUMS 231 tar -xzf bootstrap-recovery.tar.gz 232 cd recovery 233 sha256sum -c SHA256SUMS 234 nix-store --import < runtime.export 235 TRICU={tricu} 236 "$TRICU" --help 237 git clone history.bundle history 238 python3 seed-source/test/bootstrap_store_check.py seed --source-root "$PWD/seed-source" 239 ``` 240 241 `source/` is the captured tracked working tree (including uncommitted tracked 242 edits); `history/` is Git history. See working-tree.patch and RECOVERY.json. 243 The standard adapter in tools/ is selected explicitly; do not inherit the seed's 244 historical transition adapter for ordinary builds. 245 246 ## Current native imports / stable direct recursion 247 248 Import tools/build.arboricx or tools/build-stable.arboricx with `--indexed`, then 249 use the printed manifest as `--toolchain` and that store as `--tool-store`. 250 The stable artifact exports the same sourceBuildGraph ABI. See source/docs/tri-tools.md. 251 Checked tricu fixtures are under source/test/fixtures; historical SCP is archived. 252 SCP is an archived separate language experiment, not an execution/recovery gate. 253 254 ## Recover a full current compiler generation (expensive, explicitly opt-in) 255 256 The current worker is already packed; no legacy frontend or header bridge needed: 257 258 ```sh 259 cd source 260 python3 test/bootstrap_generations.py ../seed ../restored-generation \\ 261 --tricu "$TRICU" --source-root "$PWD" \\ 262 --worker-bundle ../tools/worker.arboricx \\ 263 --adapter ../tools/io-adapter.arboricx \\ 264 --backend stable --memory-mib 65536 --wall-seconds 7200 265 python3 test/bootstrap_store_check.py ../restored-generation --source-root "$PWD" 266 ``` 267 268 Stable lowering is selected explicitly. This generation command was NOT 269 run during archive creation. No second-generation or empty-store restore claim 270 is made. Keep the complete transfer directory together and copy it off-machine. 271 ''') 272 if acceptance: 273 (payload / 'README.md').write_text(f'''# Accepted stable compiler baseline 274 275 Frozen source commit: {a.seed_commit} 276 Both fresh-store stable generations completed and independently audited all 277 {len(acceptance['inputs'][1]['sources'])} mapped compiler modules. Generation 2 executed generation 1's rebuilt compiler 278 and reproduced executable manifests and all three packaged artifacts exactly. 279 See acceptance.json and the original inputs/status/invocation/logs in both stores. 280 No paths in that original provenance have been rewritten. 281 282 ## Restore 283 284 Prerequisites: x86_64 Linux, Nix at /nix, Python 3, Git, tar and sha256sum. 285 The included Nix closure is runtime-only, not offline Haskell build dependencies. 286 287 ```sh 288 sha256sum -c TRANSFER-SHA256SUMS 289 tar -xzf bootstrap-recovery.tar.gz 290 cd recovery 291 sha256sum -c SHA256SUMS 292 nix-store --import < runtime.export 293 TRICU={tricu} 294 git clone history.bundle history 295 python3 source/test/bootstrap_acceptance.py first-generation seed \\ 296 --tricu "$TRICU" --source-root "$PWD/seed-source" \\ 297 --output restored-acceptance.json 298 ``` 299 300 `seed-source/` is the exact frozen Git tree used for compilation; `source/` captures 301 current tracked maintenance/docs and the archive creator. Both full generation 302 stores are retained, not just bundles. `tools/` contains copies of generation 2's 303 compiler and application tools, plus its standard adapter. Use build-stable.arboricx 304 for application authoring; direct recursion and native imports are supported. 305 Application tools use IO version {io_version}. For explicit compiler selection use 306 `--toolchain` and `--tool-store`{'; also pass `--toolchain-io-version 2` (OS process authority, no v1 grants)' if io_version == 2 else '; the historical runtime default is v1'}. 307 The selected source map is retained in tools/source-map.v1; do not substitute the 308 shipped v1 map for a v2 compiler rebuild. To reproduce frozen inputs instead of 309 editing current sources, select seed-source as --source-root. 310 Fixtures under source/test/fixtures are authenticated against their retained source 311 snapshots, not asserted to match the captured working tree. RECOVERY.json lists 312 working-source differences; they are not newly published/accepted fixtures. 313 Historical SCP is archived. 314 SCP is an archived separate language experiment; its execution is not a restore gate. 315 316 ## Next compiler change 317 318 No old header bridge or historical scratch path is needed: 319 320 ```sh 321 python3 source/test/bootstrap_generations.py seed next-generation \\ 322 --tricu "$TRICU" --source-root "$PWD/source" \\ 323 --map "$PWD/tools/source-map.v1" \\ 324 --backend stable --fresh --memory-mib 65536 --wall-seconds 7200 325 ``` 326 327 Then rebuild unchanged frozen inputs with next-generation's rebuilt compiler in 328 another fresh store using --expect-identical and the strict acceptance gate. 329 See source/docs/compiler-rebuilding.md. The expensive next-generation command 330 is not executed during archive creation; relocated audits and practical execution 331 are executed. No empty-Nix-store or cross-platform restore claim is made. 332 Verify independent/off-machine backup copies before pruning historical scratch. 333 Keep the OG archive too. 334 ''') 335 # Hash and archive directly from retained generations: no staging CAS copies. 336 entries = [(f, 'recovery/' + f.relative_to(payload).as_posix()) 337 for f in sorted(payload.rglob('*')) if f.is_file() and not f.is_symlink()] 338 entries += [(f, 'recovery/seed/' + f.relative_to(seed).as_posix()) 339 for f in sorted(seed.rglob('*')) if f.is_file() and not f.is_symlink()] 340 if first: 341 entries += [(f, 'recovery/first-generation/' + f.relative_to(first).as_posix()) 342 for f in sorted(first.rglob('*')) if f.is_file() and not f.is_symlink()] 343 # Reject silently omitted links in generation evidence. 344 assert not any(f.is_symlink() for f in seed.rglob('*')), 'seed contains symlinks' 345 assert not first or not any(f.is_symlink() for f in first.rglob('*')), 'first generation contains symlinks' 346 checksums = payload / 'SHA256SUMS' 347 checksums.write_text(''.join(sha(f) + ' ' + name.removeprefix('recovery/') + '\n' for f, name in entries)) 348 archive = output / 'bootstrap-recovery.tar.gz' 349 print('Packing source, runtime, tools and completed seed...', flush=True) 350 with tarfile.open(archive, 'w:gz', compresslevel=1) as tar: 351 tar.add(payload, arcname='recovery') 352 tar.add(seed, arcname='recovery/seed') 353 if first: 354 tar.add(first, arcname='recovery/first-generation') 355 # Full isolated extraction and internal checks, not a generation build. 356 restored = work / 'restored'; restored.mkdir() 357 with tarfile.open(archive) as tar: 358 tar.extractall(restored, filter='data') 359 recovered = restored / 'recovery' 360 result = run('sha256sum', '-c', 'SHA256SUMS', cwd=recovered) 361 relocated_audit = run(sys.executable, recovered / 'seed-source/test/bootstrap_store_check.py', 362 recovered / 'seed', '--source-root', recovered / 'seed-source') 363 relocated_acceptance = None 364 if acceptance: 365 from bootstrap_acceptance import verify 366 relocated_acceptance = verify(recovered / 'first-generation', recovered / 'seed', 367 tricu, recovered / 'seed-source') 368 run('git', 'bundle', 'verify', recovered / 'history.bundle') 369 with (recovered / 'runtime.export').open('rb') as f: 370 subprocess.run(['nix-store', '--import'], stdin=f, stdout=subprocess.DEVNULL, check=True, timeout=600) 371 probe = work / 'probe'; probe.mkdir() 372 (probe / 'patterns.tri').write_text('!export first\nfirst (t left right) = left\nfirst _ = 99\n') 373 (probe / 'app.tri').write_text('!import patterns as P\n!export main\nmain = P.first (t 23 24)\n') 374 outcomes = {} 375 # Stable is the development/restore gate. Preserve the optimized artifact 376 # as historical/optional evidence, never execute it as a prerequisite. 377 for backend in ('build-stable',): 378 toolstore = work / backend 379 reply = run(tricu, 'arboricx', 'import', '--file', recovered / ('tools/' + backend + '.arboricx'), 380 '--store', toolstore, '--indexed') 381 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 382 flags = ['--toolchain', pin, *compiler_flags(io_version), '--tool-store', toolstore, '--store', work / (backend + '-apps'), 383 '--compile-timeout', '120', '+RTS', '-N1', '-M4g', '-RTS'] 384 answer = run(tricu, 'eval', probe / 'app.tri', '--format', 'number', *flags).strip() 385 assert answer == '23', (backend, answer) 386 outcomes[backend] = answer 387 if backend == 'build-stable': 388 (probe / 'recursive.tri').write_text('!export copy\ncopy [] = []\ncopy [head | tail] = t head (copy tail)\ncopy _ = []\n') 389 (probe / 'caller.tri').write_text('!import recursive as R\n!export main\nmain = R.copy "hello"\n') 390 answer = run(tricu, 'eval', probe / 'caller.tri', '--format', 'string', *flags).strip() 391 assert answer == '"hello"', answer 392 outcomes['stable-direct-recursion'] = answer 393 # SCP is an archived, separate language experiment. Historical files may 394 # remain in source/history, but no SCP execution gates tricu recovery. 395 (output / 'RESTORE-VALIDATION.json').write_text(json.dumps(dict( 396 compiler_io_version=io_version, authority='os-process' if io_version == 2 else 'v1-grants', 397 archive_sha256=sha(archive), files_checked=len(result.splitlines()), 398 internal_checksums='passed', seed_audit=relocated_audit.strip(), 399 git_bundle='verified', nix_closure='reimported into existing same-host store', 400 native_import_smokes=outcomes, scp_execution_required=False, 401 generation_built=False, accepted_stable_pair=bool(acceptance), 402 relocated_pair_audits=relocated_acceptance['audits'] if relocated_acceptance else None, 403 empty_nix_store_tested=False), indent=2) + '\n') 404 shutil.copy2(payload / 'RECOVERY.json', output / 'RECOVERY.json') 405 (output / 'TRANSFER-SHA256SUMS').write_text(''.join(sha(f) + ' ' + f.name + '\n' 406 for f in sorted(output.iterdir()) if f.name != 'TRANSFER-SHA256SUMS')) 407 print('Recovery archive:', archive, flush=True) 408 409 410 if __name__ == '__main__': 411 main()