bootstrap_archive.py (20971B)
1 #!/usr/bin/env python3 2 """Opt-in recovery archive creation; never compiles source or builds a generation. 3 4 Preserves a completed seed, matching historical source, current tracked working 5 files/history, checked compiler/support fixtures and an exact Nix runtime closure. 6 Only explicitly selected inputs are included; unrelated scratch runs are omitted. 7 """ 8 import argparse 9 import hashlib 10 import json 11 from pathlib import Path 12 import re 13 import shutil 14 import struct 15 import subprocess 16 import sys 17 import tarfile 18 import tempfile 19 20 ROOT = Path(__file__).resolve().parent.parent 21 22 23 def run(*args, cwd=ROOT): 24 return subprocess.check_output(list(map(str, args)), cwd=cwd, text=True, timeout=600) 25 26 27 def sha(path): 28 h = hashlib.sha256() 29 with path.open('rb') as f: 30 for chunk in iter(lambda: f.read(1024 * 1024), b''): 31 h.update(chunk) 32 return h.hexdigest() 33 34 35 def select_exports(bundle, destination, names): 36 """Re-label selected verified DAG roots without expanding logical trees.""" 37 from stable_bundle import assembled, sections 38 manifest, nodes = sections(bundle) 39 offset = 12 # ARBMNFST and version 40 41 def word(): 42 nonlocal offset 43 value = struct.unpack_from('>I', manifest, offset)[0]; offset += 4 44 return value 45 46 def string(): 47 nonlocal offset 48 size = word(); value = manifest[offset:offset + size]; offset += size 49 return value 50 51 def text(value): 52 value = value.encode() 53 return struct.pack('>I', len(value)) + value 54 55 for _ in range(9): 56 string() 57 for _ in range(word()): 58 string() 59 offset += 1 # closure mode 60 prefix = manifest[:offset] 61 for _ in range(word()): 62 word(); string() 63 exports = {} 64 for _ in range(word()): 65 name, index, kind, abi = string().decode(), word(), string(), string() 66 exports[name] = index 67 assert kind == b'term' and abi == b'arboricx.abi.tree.v1' 68 suffix = manifest[offset:] 69 selected = [(new, exports[old]) for old, new in names] 70 roots = struct.pack('>I', len(selected)) + b''.join( 71 struct.pack('>I', index) + text(name) for name, index in selected) 72 entries = struct.pack('>I', len(selected)) + b''.join( 73 text(name) + struct.pack('>I', index) + text('term') + text('arboricx.abi.tree.v1') 74 for name, index in selected) 75 destination.write_bytes(assembled(prefix + roots + entries + suffix, nodes)) 76 77 78 def main(): 79 p = argparse.ArgumentParser(description=__doc__) 80 p.add_argument('--generation', type=Path, required=True) 81 p.add_argument('--seed-commit', required=True, help='commit matching every recorded seed source') 82 p.add_argument('--first-generation', type=Path, help='first fresh stable generation of an accepted pair') 83 p.add_argument('--acceptance', type=Path, help='strict acceptance JSON for that pair') 84 p.add_argument('--adapter', type=Path, required=True, help='standard six-argument retained adapter') 85 p.add_argument('--tricu', type=Path, required=True, help='Nix-built Haskell executable') 86 p.add_argument('--output', type=Path, required=True, help='new transfer directory') 87 a = p.parse_args() 88 if bool(a.first_generation) != bool(a.acceptance): 89 p.error('--first-generation and --acceptance must be selected together') 90 seed, tricu, adapter = a.generation.resolve(), a.tricu.resolve(), a.adapter.resolve() 91 first = a.first_generation.resolve() if a.first_generation else None 92 acceptance = None 93 if first: 94 from bootstrap_acceptance import verify 95 recorded = json.loads(a.acceptance.read_text()) 96 acceptance = verify(first, seed, tricu) 97 assert recorded.get('accepted') and recorded['inputs'] == acceptance['inputs'] 98 assert recorded['artifacts'] == acceptance['artifacts'], 'stale acceptance evidence' 99 assert sha(adapter) == acceptance['inputs'][1]['adapter_sha256'], 'adapter differs from accepted pair' 100 status = json.loads((seed / 'status.json').read_text()) 101 assert status['complete'] and status['root'] == 'source_map', 'incomplete seed' 102 inputs = json.loads((seed / 'inputs.json').read_text()) 103 rows = [r.split('\t') for r in (seed / 'source-map.v1').read_text().splitlines()[1:]] 104 for name in ('rebuilt-compiler.arboricx', 'build.arboricx', 'build-stable.arboricx', 'io-adapter.arboricx'): 105 assert (seed / name).is_file(), name 106 output = a.output.resolve() 107 # Pair extraction needs the two full CAS stores plus compressed output. 108 required = sum(f.stat().st_size for directory in ([seed, first] if first else [seed]) 109 for f in directory.rglob('*') if f.is_file()) 110 assert shutil.disk_usage(output.parent).free > required * 2 + 1024**3, 'insufficient archive/restore space' 111 output.mkdir(parents=True, exist_ok=False) 112 with tempfile.TemporaryDirectory(prefix='tricu-recovery-') as temp: 113 work = Path(temp) 114 payload = work / 'recovery'; payload.mkdir() 115 source = payload / 'source'; source.mkdir() 116 tracked = run('git', 'ls-files', '-z').split('\0') 117 for name in filter(None, tracked): 118 # Historical tracked Zig build-output link is not a recovery input. 119 if name == 'ext/zig/result': 120 continue 121 src = ROOT / name; dst = source / name 122 dst.parent.mkdir(parents=True, exist_ok=True) 123 if src.is_symlink(): 124 dst.symlink_to(src.readlink()) 125 else: 126 shutil.copy2(src, dst) 127 # Preserve this creator even before it is committed; mounts are not copied. 128 shutil.copy2(__file__, source / 'test/bootstrap_archive.py') 129 run('git', 'bundle', 'create', payload / 'history.bundle', '--all') 130 (payload / 'working-tree.patch').write_text(run('git', 'diff', 'HEAD', '--binary')) 131 historical = payload / 'seed-source'; historical.mkdir() 132 historical_tar = work / 'seed-source.tar' 133 with historical_tar.open('wb') as f: 134 subprocess.run(['git', 'archive', a.seed_commit], cwd=ROOT, stdout=f, check=True) 135 with tarfile.open(historical_tar) as tar: 136 tar.extractall(historical, members=(m for m in tar if m.name != 'ext/zig/result'), filter='data') 137 for _, name, _, path, _ in rows: 138 data = subprocess.check_output(['git', 'show', a.seed_commit + ':' + path], cwd=ROOT) 139 assert hashlib.sha256(data).hexdigest() == inputs['sources'][name], ('seed source mismatch', path) 140 dst = historical / path; dst.parent.mkdir(parents=True, exist_ok=True); dst.write_bytes(data) 141 audit = (acceptance['audits'][1] if acceptance else 142 run(sys.executable, historical / 'test/bootstrap_store_check.py', seed, '--source-root', historical)) 143 if acceptance: 144 (payload / 'acceptance.json').write_text(json.dumps(acceptance, indent=2) + '\n') 145 (payload / 'seed-audit.txt').write_text(audit) 146 print(audit, end='', flush=True) 147 fixtures = source / 'test/fixtures' 148 for row in (fixtures / 'compiler-library.v1').read_text().splitlines()[1:]: 149 fields = row.split('\t') 150 if fields[0] == 'module': 151 data = (source / fields[3]).read_bytes() 152 assert hashlib.sha256(b'arboricx.source.v1\0' + data).hexdigest() == fields[4], fields[3] 153 elif fields[0] == 'bundle': 154 assert hashlib.sha256(b'tricu.compiler-library-fixture.v1\0' + 155 (fixtures / 'compiler-library.arboricx').read_bytes()).hexdigest() == fields[1] 156 tools = payload / 'tools'; tools.mkdir() 157 shutil.copy2(adapter, tools / 'io-adapter.arboricx') 158 store = work / 'fixture-store' 159 selections = { 160 'build.arboricx': [('source_build.sourceBuildGraph', 'sourceBuildGraph')], 161 'build-stable.arboricx': [('source_build.sourceBuildGraphStable', 'sourceBuildGraph')], 162 'worker.arboricx': [('source_map.sourceMapBuild', 'sourceMapBuild'), 163 ('source_map.sourceMapBuildStable', 'sourceMapBuildStable')], 164 } 165 for filename, names in selections.items(): 166 if acceptance: 167 origin = 'rebuilt-compiler.arboricx' if filename == 'worker.arboricx' else filename 168 shutil.copy2(seed / origin, tools / filename) 169 else: 170 select_exports(fixtures / 'compiler-library.arboricx', tools / filename, names) 171 # The host independently verifies each rewritten manifest and DAG. 172 run(tricu, 'arboricx', 'import', '--file', tools / filename, 173 '--store', store / filename, '--indexed', '+RTS', '-N1', '-M512m', '-RTS') 174 runtime = tricu.parent.parent 175 assert str(runtime).startswith('/nix/store/'), 'runtime must be a Nix output' 176 closure = run('nix-store', '--query', '--requisites', runtime).splitlines() 177 (payload / 'runtime-paths.txt').write_text('\n'.join(closure) + '\n') 178 with (payload / 'runtime.export').open('wb') as f: 179 subprocess.run(['nix-store', '--export', *closure], stdout=f, check=True, timeout=600) 180 metadata = dict(format='tricu.bootstrap-recovery.v1', head=run('git', 'rev-parse', 'HEAD').strip(), 181 seed_commit=a.seed_commit, runtime=str(runtime), runtime_sha256=sha(tricu), 182 seed_inputs=inputs, selected_adapter_sha256=sha(adapter), 183 fixture_bundle_sha256=sha(fixtures / 'compiler-library.arboricx'), 184 accepted_stable_pair=bool(acceptance), 185 accepted_artifacts=acceptance['artifacts'] if acceptance else None, 186 limitations=(['Archive creation does not compile source.', 187 'Accepted compiler/tool artifacts copied from the reproduced second generation.'] 188 if acceptance else 189 ['No new compiler generation built.', 190 'Current tools packed from checked fixture, not a newly completed generation.']) + [ 191 'Tracked ext/zig/result build-output symlink omitted from source snapshots; retained in Git history.', 192 'Nix closure is x86_64-linux; no empty-Nix-store or cross-platform restore tested.']) 193 (payload / 'RECOVERY.json').write_text(json.dumps(metadata, indent=2) + '\n') 194 (payload / 'README.md').write_text(f'''# tricu bootstrap recovery 195 196 This is a recovery snapshot, not a new self-reproduction result. The completed 197 seed and its exact mapped source are preserved independently of current source. 198 Current tools are native-packed entrypoints from the checked compiler fixture; 199 its current source fingerprints are in source/test/fixtures/compiler-library.v1. 200 Those fingerprints alone do not prove header equivalence or a completed generation. 201 Original generation paths/provenance are deliberately not rewritten. 202 203 Prerequisites: x86_64 Linux, Nix installed at /nix, Python 3, Git, tar and sha256sum. 204 The runtime export is a runtime closure, not an offline Haskell build closure. 205 206 ## Restore and inspect 207 208 Verify the transfer directory's TRANSFER-SHA256SUMS before extracting: 209 210 ```sh 211 sha256sum -c TRANSFER-SHA256SUMS 212 tar -xzf bootstrap-recovery.tar.gz 213 cd recovery 214 sha256sum -c SHA256SUMS 215 nix-store --import < runtime.export 216 TRICU={tricu} 217 "$TRICU" --help 218 git clone history.bundle history 219 python3 seed-source/test/bootstrap_store_check.py seed --source-root "$PWD/seed-source" 220 ``` 221 222 `source/` is the captured tracked working tree (including uncommitted tracked 223 edits); `history/` is Git history. See working-tree.patch and RECOVERY.json. 224 The standard adapter in tools/ is selected explicitly; do not inherit the seed's 225 historical transition adapter for ordinary builds. 226 227 ## Current native imports / stable direct recursion 228 229 Import tools/build.arboricx or tools/build-stable.arboricx with `--indexed`, then 230 use the printed manifest as `--toolchain` and that store as `--tool-store`. 231 The stable artifact exports the same sourceBuildGraph ABI. See source/docs/tri-tools.md. 232 Checked tricu fixtures are under source/test/fixtures; historical SCP is archived. 233 SCP is an archived separate language experiment, not an execution/recovery gate. 234 235 ## Recover a full current compiler generation (expensive, explicitly opt-in) 236 237 The current worker is already packed; no legacy frontend or header bridge needed: 238 239 ```sh 240 cd source 241 python3 test/bootstrap_generations.py ../seed ../restored-generation \\ 242 --tricu "$TRICU" --source-root "$PWD" \\ 243 --worker-bundle ../tools/worker.arboricx \\ 244 --adapter ../tools/io-adapter.arboricx \\ 245 --backend stable --memory-mib 65536 --wall-seconds 7200 246 python3 test/bootstrap_store_check.py ../restored-generation --source-root "$PWD" 247 ``` 248 249 Stable lowering is selected explicitly. This generation command was NOT 250 run during archive creation. No second-generation or empty-store restore claim 251 is made. Keep the complete transfer directory together and copy it off-machine. 252 ''') 253 if acceptance: 254 (payload / 'README.md').write_text(f'''# Accepted stable compiler baseline 255 256 Frozen source commit: {a.seed_commit} 257 Both fresh-store stable generations completed and independently audited all 258 {len(acceptance['inputs'][1]['sources'])} mapped compiler modules. Generation 2 executed generation 1's rebuilt compiler 259 and reproduced executable manifests and all three packaged artifacts exactly. 260 See acceptance.json and the original inputs/status/invocation/logs in both stores. 261 No paths in that original provenance have been rewritten. 262 263 ## Restore 264 265 Prerequisites: x86_64 Linux, Nix at /nix, Python 3, Git, tar and sha256sum. 266 The included Nix closure is runtime-only, not offline Haskell build dependencies. 267 268 ```sh 269 sha256sum -c TRANSFER-SHA256SUMS 270 tar -xzf bootstrap-recovery.tar.gz 271 cd recovery 272 sha256sum -c SHA256SUMS 273 nix-store --import < runtime.export 274 TRICU={tricu} 275 git clone history.bundle history 276 python3 source/test/bootstrap_acceptance.py first-generation seed \\ 277 --tricu "$TRICU" --source-root "$PWD/seed-source" \\ 278 --output restored-acceptance.json 279 ``` 280 281 `seed-source/` is the exact frozen Git tree used for compilation; `source/` captures 282 current tracked maintenance/docs and the archive creator. Both full generation 283 stores are retained, not just bundles. `tools/` contains copies of generation 2's 284 compiler and application tools, plus its standard adapter. Use build-stable.arboricx 285 for application authoring; direct recursion and native imports are supported. 286 Checked tricu fixtures remain under source/test/fixtures; historical SCP is archived. 287 SCP is an archived separate language experiment; its execution is not a restore gate. 288 289 ## Next compiler change 290 291 No old header bridge or historical scratch path is needed: 292 293 ```sh 294 python3 source/test/bootstrap_generations.py seed next-generation \\ 295 --tricu "$TRICU" --source-root "$PWD/source" \\ 296 --map "$PWD/source/bootstrap/compiler-source-map.v1" \\ 297 --backend stable --fresh --memory-mib 65536 --wall-seconds 7200 298 ``` 299 300 Then rebuild unchanged frozen inputs with next-generation's rebuilt compiler in 301 another fresh store using --expect-identical and the strict acceptance gate. 302 See source/docs/compiler-rebuilding.md. The expensive next-generation command 303 is not executed during archive creation; relocated audits and practical execution 304 are executed. No empty-Nix-store or cross-platform restore claim is made. 305 Verify independent/off-machine backup copies before pruning historical scratch. 306 Keep the OG archive too. 307 ''') 308 # Hash and archive directly from retained generations: no staging CAS copies. 309 entries = [(f, 'recovery/' + f.relative_to(payload).as_posix()) 310 for f in sorted(payload.rglob('*')) if f.is_file() and not f.is_symlink()] 311 entries += [(f, 'recovery/seed/' + f.relative_to(seed).as_posix()) 312 for f in sorted(seed.rglob('*')) if f.is_file() and not f.is_symlink()] 313 if first: 314 entries += [(f, 'recovery/first-generation/' + f.relative_to(first).as_posix()) 315 for f in sorted(first.rglob('*')) if f.is_file() and not f.is_symlink()] 316 # Reject silently omitted links in generation evidence. 317 assert not any(f.is_symlink() for f in seed.rglob('*')), 'seed contains symlinks' 318 assert not first or not any(f.is_symlink() for f in first.rglob('*')), 'first generation contains symlinks' 319 checksums = payload / 'SHA256SUMS' 320 checksums.write_text(''.join(sha(f) + ' ' + name.removeprefix('recovery/') + '\n' for f, name in entries)) 321 archive = output / 'bootstrap-recovery.tar.gz' 322 print('Packing source, runtime, tools and completed seed...', flush=True) 323 with tarfile.open(archive, 'w:gz', compresslevel=1) as tar: 324 tar.add(payload, arcname='recovery') 325 tar.add(seed, arcname='recovery/seed') 326 if first: 327 tar.add(first, arcname='recovery/first-generation') 328 # Full isolated extraction and internal checks, not a generation build. 329 restored = work / 'restored'; restored.mkdir() 330 with tarfile.open(archive) as tar: 331 tar.extractall(restored, filter='data') 332 recovered = restored / 'recovery' 333 result = run('sha256sum', '-c', 'SHA256SUMS', cwd=recovered) 334 relocated_audit = run(sys.executable, recovered / 'seed-source/test/bootstrap_store_check.py', 335 recovered / 'seed', '--source-root', recovered / 'seed-source') 336 relocated_acceptance = None 337 if acceptance: 338 from bootstrap_acceptance import verify 339 relocated_acceptance = verify(recovered / 'first-generation', recovered / 'seed', 340 tricu, recovered / 'seed-source') 341 run('git', 'bundle', 'verify', recovered / 'history.bundle') 342 with (recovered / 'runtime.export').open('rb') as f: 343 subprocess.run(['nix-store', '--import'], stdin=f, stdout=subprocess.DEVNULL, check=True, timeout=600) 344 probe = work / 'probe'; probe.mkdir() 345 (probe / 'patterns.tri').write_text('!export first\nfirst (t left right) = left\nfirst _ = 99\n') 346 (probe / 'app.tri').write_text('!import patterns as P\n!export main\nmain = P.first (t 23 24)\n') 347 outcomes = {} 348 # Stable is the development/restore gate. Preserve the optimized artifact 349 # as historical/optional evidence, never execute it as a prerequisite. 350 for backend in ('build-stable',): 351 toolstore = work / backend 352 reply = run(tricu, 'arboricx', 'import', '--file', recovered / ('tools/' + backend + '.arboricx'), 353 '--store', toolstore, '--indexed') 354 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 355 flags = ['--toolchain', pin, '--tool-store', toolstore, '--store', work / (backend + '-apps'), 356 '--compile-timeout', '120', '+RTS', '-N1', '-M4g', '-RTS'] 357 answer = run(tricu, 'eval', probe / 'app.tri', '--format', 'number', *flags).strip() 358 assert answer == '23', (backend, answer) 359 outcomes[backend] = answer 360 if backend == 'build-stable': 361 (probe / 'recursive.tri').write_text('!export copy\ncopy [] = []\ncopy [head | tail] = t head (copy tail)\ncopy _ = []\n') 362 (probe / 'caller.tri').write_text('!import recursive as R\n!export main\nmain = R.copy "hello"\n') 363 answer = run(tricu, 'eval', probe / 'caller.tri', '--format', 'string', *flags).strip() 364 assert answer == '"hello"', answer 365 outcomes['stable-direct-recursion'] = answer 366 # SCP is an archived, separate language experiment. Historical files may 367 # remain in source/history, but no SCP execution gates tricu recovery. 368 (output / 'RESTORE-VALIDATION.json').write_text(json.dumps(dict( 369 archive_sha256=sha(archive), files_checked=len(result.splitlines()), 370 internal_checksums='passed', seed_audit=relocated_audit.strip(), 371 git_bundle='verified', nix_closure='reimported into existing same-host store', 372 native_import_smokes=outcomes, scp_execution_required=False, 373 generation_built=False, accepted_stable_pair=bool(acceptance), 374 relocated_pair_audits=relocated_acceptance['audits'] if relocated_acceptance else None, 375 empty_nix_store_tested=False), indent=2) + '\n') 376 shutil.copy2(payload / 'RECOVERY.json', output / 'RECOVERY.json') 377 (output / 'TRANSFER-SHA256SUMS').write_text(''.join(sha(f) + ' ' + f.name + '\n' 378 for f in sorted(output.iterdir()) if f.name != 'TRANSFER-SHA256SUMS')) 379 print('Recovery archive:', archive, flush=True) 380 381 382 if __name__ == '__main__': 383 main()