bootstrap_cross_host.py (13376B)
1 #!/usr/bin/env python3 2 """Opt-in isolated H1 -> H2 -> {H3,Z3} reproduction, or bounded native_math probe. 3 4 Every job sees only /source, /kit, /work, /nix/store and ephemeral /tmp,/proc,/dev. 5 The generation supervisor runs inside isolation and monitors the execution 6 child. Haskell supplies common artifact transport and packaging. 7 """ 8 import argparse 9 import json 10 from pathlib import Path 11 import shutil 12 import subprocess 13 import sys 14 import tarfile 15 import time 16 17 from bootstrap_acceptance import ARTIFACTS, sha, verify, verify_generation 18 from bootstrap_generations import source_map, snapshot 19 from bootstrap_inputs import load_kit 20 from bootstrap_retention_lock import protect_generations 21 from process_rss import run_guarded 22 23 ROOT = Path(__file__).resolve().parent.parent 24 25 26 def save(path, value): 27 path.write_text(json.dumps(value, indent=2)+'\n') 28 29 30 def inventory(run): 31 return {p.name:sha(p) for p in (run/'cas/objects').glob('*/*')} 32 33 34 def verify_cross(first, second, control, zig, runtime, library, source): 35 if len({p.resolve() for p in (first,second,control,zig)}) != 4: 36 raise ValueError('four distinct fresh generations required') 37 pair = verify(first, second, runtime, source) 38 control_pair = verify(second, control, runtime, source) 39 zi, zs, audit = verify_generation(zig, library, source, 'zig') 40 hi = control_pair['inputs'][1] 41 for key in ('sources','map_sha256','backend','root','adapter_sha256', 42 'compiler_bundle_sha256','compiler_tree','compiler_tree_kind', 43 'worker_export','backend_options'): 44 if zi[key] != hi[key]: 45 raise ValueError('cross-host input mismatch: '+key) 46 if zi.get('transport_runtime_sha256') != sha(runtime) or zi.get('transport_host') != 'haskell': 47 raise ValueError('cross-host transport identity mismatch') 48 if zi.get('executor_sha256') != sha(source/'test/zig_generation_executor.py'): 49 raise ValueError('execution launcher identity mismatch') 50 request = json.loads((zig/'execution-request.json').read_text()) 51 if request != dict(worker_export=zi['worker_export'], 52 compiler_bundle_sha256=zi['compiler_bundle_sha256'], 53 adapter_sha256=zi['adapter_sha256'], 54 arguments=['/source','/work/generation/overlay','/work/generation/cas', 55 zi['compiler_bundle_sha256'],(zig/'source-map.v1').read_text(),'source_map']): 56 raise ValueError('isolated Zig execution request mismatch') 57 for name in ARTIFACTS: 58 if sha(second/name) != sha(zig/name): 59 raise ValueError('cross-host packaged artifact mismatch: '+name) 60 # Byte-identical full inventories include manifests, receipts and every 61 # published indexed object, not just a subset of selected export hashes. 62 expected = inventory(second) 63 for run in (control,zig): 64 if inventory(run) != expected: 65 raise ValueError('complete canonical object/receipt inventory differs: '+str(run)) 66 for alias in hi['sources']: 67 if (second/'cas/cache/source-build'/alias).read_bytes() != (run/'cas/cache/source-build'/alias).read_bytes(): 68 raise ValueError('complete checkpoint mismatch: '+alias) 69 return dict(accepted=True, scope='cross-host-fixed-point-not-retention', 70 generations=list(map(str,(first,second,control,zig))), 71 haskell_pair=pair, haskell_control=control_pair, 72 zig_inputs=zi, zig_audit=audit, objects=expected, 73 artifacts={name:sha(zig/name) for name in ARTIFACTS}) 74 75 76 def sandbox_command(source, kit, stage, runtime, library, host, root, memory, seconds): 77 command = ['unshare','--user','--map-root-user','--mount','--pid','--fork', 78 '--kill-child','--mount-proc','--','bwrap','--die-with-parent','--unshare-all', 79 '--ro-bind','/nix/store','/nix/store','--proc','/proc','--dev','/dev', 80 '--tmpfs','/tmp','--ro-bind',str(source),'/source', 81 '--ro-bind',str(kit),'/kit','--bind',str(stage),'/work', 82 '--chdir','/source','--setenv','HOME','/tmp','--setenv','TMPDIR','/tmp', 83 '--setenv','PYTHONDONTWRITEBYTECODE','1', 84 sys.executable,'/source/test/bootstrap_generations.py','/work/generation', 85 '--bootstrap-kit','/kit','--map','/source/bootstrap/compiler-source-map.v1', 86 '--source-root','/source','--root',root,'--backend','stable','--fresh', 87 '--tricu',str(runtime),'--execution-host',host,'--capabilities','8', 88 '--memory-mib',str(memory),'--wall-seconds',str(seconds), 89 '--initial-seconds',str(min(600,seconds)), 90 '--extended-seconds',str(min(1800,seconds))] 91 if host=='zig': command += ['--zig-library',str(library)] 92 return command 93 94 95 def main(): 96 p = argparse.ArgumentParser(description=__doc__) 97 p.add_argument('--directory',type=Path,required=True) 98 p.add_argument('--source-commit',required=True) 99 p.add_argument('--tricu',type=Path,required=True) 100 p.add_argument('--zig-library',type=Path,required=True) 101 p.add_argument('--probe',action='store_true',help='native_math on both hosts, 1 GiB/120s per job') 102 p.add_argument('--verify-only',action='store_true') 103 p.add_argument('--continue-experiment',action='store_true',help='re-audit completed jobs and retry pre-execution launch failures') 104 args=p.parse_args() 105 directory=args.directory.resolve() 106 runtime,library=args.tricu.resolve(),args.zig_library.resolve() 107 source=directory/'source' 108 locks=protect_generations([directory]) 109 if args.verify_only: 110 evidence=verify_cross(*(directory/label/'generation' for label in ('H1','H2','H3','Z3')), 111 runtime,library,source) 112 print(json.dumps(evidence,indent=2)) 113 return 114 commit=subprocess.check_output(['git','rev-parse',args.source_commit+'^{commit}'],cwd=ROOT,text=True).strip() 115 archive=directory/'source.tar' 116 if args.continue_experiment: 117 original=json.loads((directory/'experiment.json').read_text()) 118 if (args.probe or original['source_commit']!=commit or original['source_archive_sha256']!=sha(archive) 119 or original['runtime']!=dict(path=str(runtime),sha256=sha(runtime)) 120 or original['zig_library']!=dict(path=str(library),sha256=sha(library))): 121 raise ValueError('continuation requires identical frozen sources and runtimes') 122 with tarfile.open(archive) as tar: 123 for member in tar: 124 if member.isfile() and (source/member.name).read_bytes()!=tar.extractfile(member).read(): 125 raise ValueError('frozen source changed: '+member.name) 126 # Preserve the failed attempt, including its exact original tooling. 127 shutil.copyfile(directory/'report.json',directory/('report-before-'+str(time.time_ns())+'.json')) 128 else: 129 directory.mkdir(parents=True,exist_ok=False) 130 with archive.open('wb') as out: 131 subprocess.run(['git','archive',commit],cwd=ROOT,stdout=out,check=True) 132 source.mkdir() 133 with tarfile.open(archive) as tar: 134 # Historical tracked build-output symlink, excluded by recovery 135 # archives too. Keep all other extraction safety checks. 136 tar.extractall(source,members=(m for m in tar if m.name!='ext/zig/result'),filter='data') 137 # The sandbox also mounts this snapshot read-only; never use live sources. 138 map_text,entries,required=source_map(source/'bootstrap/compiler-source-map.v1',source,'source_map') 139 attempt='continuation-'+str(time.time_ns()) if args.continue_experiment else 'harness' 140 harness=directory/attempt/'test' 141 harness.mkdir(parents=True) 142 for path in (ROOT/'test').glob('*.py'): shutil.copyfile(path,harness/path.name) 143 tooling={path.name:sha(path) for path in harness.glob('*.py')} 144 memory,seconds=(1024,120) if args.probe else (32768,3600) 145 definition=dict(schema='tricu.cross-host-experiment.v1',source_commit=commit, 146 source_archive_sha256=sha(archive), excluded_source_paths=['ext/zig/result'], 147 runtime=dict(path=str(runtime),sha256=sha(runtime)), 148 zig_library=dict(path=str(library),sha256=sha(library)), 149 python=dict(path=sys.executable,sha256=sha(Path(sys.executable))), 150 harness_sha256=sha(Path(__file__)), tooling_sha256=tooling, 151 isolation_tools={name:dict(path=str(Path(shutil.which(name)).resolve()),sha256=sha(Path(shutil.which(name)))) for name in ('unshare','bwrap')}, 152 limits=dict(memory_mib=memory,wall_seconds=seconds,capabilities=8), 153 expected='H2 = H3 = Z3: complete objects, receipts, checkpoints and three bundles', 154 comparison_exclusions=['runtime/transport provenance','timings','logs','filesystem locations','invocation records'], 155 **snapshot(map_text,entries,source)) 156 save(directory/(attempt+'-definition.json' if args.continue_experiment else 'experiment.json'),definition) 157 report=dict(complete=False,scope='probe' if args.probe else 'full',stages=[]) 158 save(directory/'report.json',report) 159 def job(label,kit,host): 160 # Verify the kit before crossing the sandbox boundary; the shared runner 161 # independently captures/authenticates it again inside. 162 load_kit(kit,'stable') 163 stage=directory/label 164 if args.continue_experiment and stage.exists(): 165 if (stage/'generation/status.json').is_file(): 166 inputs,_,_=verify_generation(stage/'generation',runtime if host=='haskell' else library,source,host) 167 if inputs['compiler_bundle_sha256']!=sha(kit/'rebuilt-compiler.arboricx') or inputs['adapter_sha256']!=sha(kit/'io-adapter.arboricx'): 168 raise ValueError('completed job used a different kit') 169 report['stages'].append(label) 170 save(directory/'report.json',report) 171 print('Re-audited completed '+label,flush=True) 172 return stage/'generation' 173 if (stage/'generation').exists(): 174 raise ValueError('cannot resume a partially executed generation; select a fresh experiment') 175 stage.rename(directory/(label+'-launch-failed-'+str(time.time_ns()))) 176 stage.mkdir() 177 command=sandbox_command(source,kit,stage,runtime,library,host, 178 'native_math' if args.probe else 'source_map',memory,seconds) 179 save(stage/'sandbox-command.json',command) 180 try: 181 reply,_=run_guarded(command,memory_mib=memory,seconds=seconds+30) 182 except Exception as error: 183 (stage/'supervisor.log').write_text(str(getattr(error,'output',''))+'\n'+str(error)) 184 report['failure']=label 185 save(directory/'report.json',report) 186 raise 187 (stage/'supervisor.log').write_text(reply) 188 report['stages'].append(label) 189 save(directory/'report.json',report) 190 print('Completed isolated '+label,flush=True) 191 return stage/'generation' 192 def kit_from(run,label): 193 kit=directory/(label+'-kit') 194 if kit.exists(): 195 load_kit(kit,'stable') 196 for name in ('rebuilt-compiler.arboricx','io-adapter.arboricx'): 197 if (kit/name).read_bytes()!=(run/name).read_bytes(): 198 raise ValueError('retained kit differs from parent generation') 199 return kit 200 kit.mkdir() 201 artifacts={} 202 for role,name,export in [('compiler','rebuilt-compiler.arboricx','sourceMapBuildStable'), 203 ('adapter','io-adapter.arboricx','main')]: 204 shutil.copyfile(run/name,kit/name) 205 artifacts[role]=dict(file=name,export=export,sha256=sha(kit/name)) 206 save(kit/'bootstrap-kit.json',dict(schema='tricu.bootstrap-kit.v1',backend='stable', 207 parent_generation=str(run),artifacts=artifacts)) 208 return kit 209 first=job('H1',source/'bootstrap/rebuild','haskell') 210 if args.probe: 211 zig=job('Z3',source/'bootstrap/rebuild','zig') 212 if inventory(first)!=inventory(zig): raise ValueError('probe complete object inventories differ') 213 else: 214 second=job('H2',kit_from(first,'H1'),'haskell') 215 accepted=verify(first,second,runtime,source) 216 if (directory/'accepted-pair.json').exists(): 217 if json.loads((directory/'accepted-pair.json').read_text())!=accepted: 218 raise ValueError('existing pair acceptance changed') 219 else: 220 save(directory/'accepted-pair.json',accepted) 221 kit=kit_from(second,'H2') 222 control=job('H3',kit,'haskell') 223 zig=job('Z3',kit,'zig') 224 save(directory/'accepted-cross-host.json',verify_cross(first,second,control,zig,runtime,library,source)) 225 if tooling != {name:sha(ROOT/'test'/name) for name in tooling}: 226 raise ValueError('experiment tooling changed during execution') 227 report['complete']=True 228 save(directory/'report.json',report) 229 label='Isolated mapped-library probe' if args.probe else 'Cross-runtime compiler reproduction' 230 print(label+' verified: '+str(directory)) 231 232 233 if __name__=='__main__': 234 main()