bootstrap_cross_host.py (16144B)
1 #!/usr/bin/env python3 2 """Opt-in isolated H1 -> H2 -> {H3,Z3} reproduction, or bounded native_math probe. 3 4 Every job sees only /source, /kit, /work, /nix/store and ephemeral /tmp,/proc,/dev. 5 The generation supervisor runs inside isolation and monitors the execution 6 child. Haskell supplies common artifact transport and packaging. 7 """ 8 import argparse 9 import json 10 from pathlib import Path 11 import shutil 12 import subprocess 13 import sys 14 import tarfile 15 import time 16 17 from bootstrap_acceptance import ARTIFACTS, sha, verify, verify_generation 18 from bootstrap_generations import source_map, snapshot 19 from bootstrap_inputs import load_kit, kit_io_version 20 from generation_io import versions, produced_adapter, cli_version, version 21 from bootstrap_retention_lock import protect_generations 22 from process_rss import run_guarded 23 24 ROOT = Path(__file__).resolve().parent.parent 25 26 27 def save(path, value): 28 path.write_text(json.dumps(value, indent=2)+'\n') 29 30 31 def inventory(run): 32 return {p.name:sha(p) for p in (run/'cas/objects').glob('*/*')} 33 34 35 def verify_cross(first, second, control, zig, runtime, library, source): 36 if len({p.resolve() for p in (first,second,control,zig)}) != 4: 37 raise ValueError('four distinct fresh generations required') 38 pair = verify(first, second, runtime, source) 39 control_pair = verify(second, control, runtime, source) 40 zi, zs, audit = verify_generation(zig, library, source, 'zig') 41 hi = control_pair['inputs'][1] 42 if versions(zi) != versions(hi) or produced_adapter(zig,zi) != produced_adapter(control,hi): 43 raise ValueError('cross-host IO framing or produced adapter mismatch') 44 for key in ('sources','map_sha256','backend','root','adapter_sha256', 45 'compiler_bundle_sha256','compiler_tree','compiler_tree_kind', 46 'worker_export','backend_options'): 47 if zi[key] != hi[key]: 48 raise ValueError('cross-host input mismatch: '+key) 49 if zi.get('transport_runtime_sha256') != sha(runtime) or zi.get('transport_host') != 'haskell': 50 raise ValueError('cross-host transport identity mismatch') 51 if zi.get('executor_sha256') != sha(source/'test/zig_generation_executor.py'): 52 raise ValueError('execution launcher identity mismatch') 53 request = json.loads((zig/'execution-request.json').read_text()) 54 expected_request = dict(worker_export=zi['worker_export'], 55 compiler_bundle_sha256=zi['compiler_bundle_sha256'], 56 adapter_sha256=zi['adapter_sha256'], 57 arguments=['/source','/work/generation/overlay','/work/generation/cas', 58 zi['compiler_bundle_sha256'],(zig/'source-map.v1').read_text(),'source_map']) 59 if 'execution_io_version' in request: 60 expected_request['execution_io_version'] = versions(zi)[0] 61 if request != expected_request: 62 raise ValueError('isolated Zig execution request mismatch') 63 for name in ARTIFACTS: 64 if sha(second/name) != sha(zig/name): 65 raise ValueError('cross-host packaged artifact mismatch: '+name) 66 # Byte-identical full inventories include manifests, receipts and every 67 # published indexed object, not just a subset of selected export hashes. 68 expected = inventory(second) 69 for run in (control,zig): 70 if inventory(run) != expected: 71 raise ValueError('complete canonical object/receipt inventory differs: '+str(run)) 72 for alias in hi['sources']: 73 if (second/'cas/cache/source-build'/alias).read_bytes() != (run/'cas/cache/source-build'/alias).read_bytes(): 74 raise ValueError('complete checkpoint mismatch: '+alias) 75 return dict(accepted=True, scope='cross-host-fixed-point-not-retention', 76 generations=list(map(str,(first,second,control,zig))), 77 haskell_pair=pair, haskell_control=control_pair, 78 zig_inputs=zi, zig_audit=audit, objects=expected, 79 artifacts={name:sha(zig/name) for name in ARTIFACTS}) 80 81 82 def map_name(io_version): 83 return 'bootstrap/compiler-source-map' + ('.io-v2' if version(io_version)==2 else '') + '.v1' 84 85 86 def freeze_kit(selected, destination, io_version): 87 raw,payloads,_=load_kit(selected,'stable') 88 if kit_io_version(json.loads(raw)) != version(io_version): 89 raise ValueError('selected kit IO version mismatch') 90 if destination.exists(): 91 retained,contents,_=load_kit(destination,'stable') 92 if retained!=raw or contents!=payloads: 93 raise ValueError('frozen seed kit changed') 94 else: 95 destination.mkdir() 96 (destination/'bootstrap-kit.json').write_bytes(raw) 97 for role,name in [('compiler','rebuilt-compiler.arboricx'),('adapter','io-adapter.arboricx')]: 98 (destination/name).write_bytes(payloads[role]) 99 return sha(destination/'bootstrap-kit.json') 100 101 102 def kit_from_generation(run,kit): 103 record=json.loads((run/'inputs.json').read_text()) 104 output=versions(record)[1] 105 adapter=produced_adapter(run,record) 106 compiler=(run/'rebuilt-compiler.arboricx').read_bytes() 107 if kit.exists(): 108 raw,payloads,_=load_kit(kit,'stable') 109 if kit_io_version(json.loads(raw))!=output or payloads!={'compiler':compiler,'adapter':adapter}: 110 raise ValueError('retained kit differs from parent generation') 111 return kit 112 kit.mkdir() 113 artifacts={} 114 for role,name,export,payload in [('compiler','rebuilt-compiler.arboricx','sourceMapBuildStable',compiler), 115 ('adapter','io-adapter.arboricx','main',adapter)]: 116 (kit/name).write_bytes(payload) 117 artifacts[role]=dict(file=name,export=export,sha256=sha(kit/name)) 118 save(kit/'bootstrap-kit.json',dict(schema='tricu.bootstrap-kit.v2',backend='stable',io_version=output, 119 parent_generation=str(run),artifacts=artifacts)) 120 return kit 121 122 123 def sandbox_command(source, kit, stage, runtime, library, host, root, memory, seconds, io_version=1): 124 command = ['unshare','--user','--map-root-user','--mount','--pid','--fork', 125 '--kill-child','--mount-proc','--','bwrap','--die-with-parent','--unshare-all', 126 '--ro-bind','/nix/store','/nix/store','--proc','/proc','--dev','/dev', 127 '--tmpfs','/tmp','--ro-bind',str(source),'/source', 128 '--ro-bind',str(kit),'/kit','--bind',str(stage),'/work', 129 '--chdir','/source','--setenv','HOME','/tmp','--setenv','TMPDIR','/tmp', 130 '--setenv','PYTHONDONTWRITEBYTECODE','1', 131 sys.executable,'/source/test/bootstrap_generations.py','/work/generation', 132 '--bootstrap-kit','/kit','--map','/source/'+map_name(io_version), 133 '--source-root','/source','--root',root,'--backend','stable','--fresh', 134 '--tricu',str(runtime),'--execution-host',host,'--capabilities','8', 135 '--memory-mib',str(memory),'--wall-seconds',str(seconds), 136 '--initial-seconds',str(min(600,seconds)), 137 '--extended-seconds',str(min(1800,seconds))] 138 if host=='zig': command += ['--zig-library',str(library)] 139 return command 140 141 142 def main(): 143 p = argparse.ArgumentParser(description=__doc__) 144 p.add_argument('--directory',type=Path,required=True) 145 p.add_argument('--source-commit',required=True) 146 p.add_argument('--tricu',type=Path,required=True) 147 p.add_argument('--zig-library',type=Path,required=True) 148 p.add_argument('--probe',action='store_true',help='native_math on both hosts, 1 GiB/120s per job') 149 p.add_argument('--verify-only',action='store_true') 150 p.add_argument('--continue-experiment',action='store_true',help='re-audit completed jobs and retry pre-execution launch failures') 151 p.add_argument('--io-version',type=cli_version,default=1,help='explicit cohort framing; default historical v1') 152 p.add_argument('--bootstrap-kit',type=Path,help='explicit trusted seed; required for a new v2 experiment') 153 args=p.parse_args() 154 directory=args.directory.resolve() 155 runtime,library=args.tricu.resolve(),args.zig_library.resolve() 156 source=directory/'source' 157 locks=protect_generations([directory]) 158 if args.verify_only: 159 evidence=verify_cross(*(directory/label/'generation' for label in ('H1','H2','H3','Z3')), 160 runtime,library,source) 161 print(json.dumps(evidence,indent=2)) 162 return 163 commit=subprocess.check_output(['git','rev-parse',args.source_commit+'^{commit}'],cwd=ROOT,text=True).strip() 164 archive=directory/'source.tar' 165 if args.io_version==2 and not args.bootstrap_kit and not args.continue_experiment: 166 p.error('new v2 experiments require --bootstrap-kit') 167 if args.bootstrap_kit: 168 raw,_,_=load_kit(args.bootstrap_kit,'stable') 169 if kit_io_version(json.loads(raw)) != args.io_version: 170 raise ValueError('selected kit IO version mismatch') 171 if args.continue_experiment: 172 original=json.loads((directory/'experiment.json').read_text()) 173 if (args.probe or original.get('io_version',1)!=args.io_version or original['source_commit']!=commit or original['source_archive_sha256']!=sha(archive) 174 or original['runtime']!=dict(path=str(runtime),sha256=sha(runtime)) 175 or original['zig_library']!=dict(path=str(library),sha256=sha(library))): 176 raise ValueError('continuation requires identical frozen sources and runtimes') 177 with tarfile.open(archive) as tar: 178 for member in tar: 179 if member.isfile() and (source/member.name).read_bytes()!=tar.extractfile(member).read(): 180 raise ValueError('frozen source changed: '+member.name) 181 # Preserve the failed attempt, including its exact original tooling. 182 shutil.copyfile(directory/'report.json',directory/('report-before-'+str(time.time_ns())+'.json')) 183 else: 184 directory.mkdir(parents=True,exist_ok=False) 185 with archive.open('wb') as out: 186 subprocess.run(['git','archive',commit],cwd=ROOT,stdout=out,check=True) 187 source.mkdir() 188 with tarfile.open(archive) as tar: 189 # Historical tracked build-output symlink, excluded by recovery 190 # archives too. Keep all other extraction safety checks. 191 tar.extractall(source,members=(m for m in tar if m.name!='ext/zig/result'),filter='data') 192 # The sandbox also mounts this snapshot read-only; never use live sources. 193 selected_seed=args.bootstrap_kit or (directory/'seed-kit' if args.continue_experiment and 'seed_manifest_sha256' in original else source/'bootstrap/rebuild') 194 seed_pin=freeze_kit(selected_seed,directory/'seed-kit',args.io_version) 195 if args.continue_experiment and seed_pin!=original.get('seed_manifest_sha256',seed_pin): 196 raise ValueError('continuation seed identity mismatch') 197 map_text,entries,required=source_map(source/map_name(args.io_version),source,'source_map') 198 attempt='continuation-'+str(time.time_ns()) if args.continue_experiment else 'harness' 199 harness=directory/attempt/'test' 200 harness.mkdir(parents=True) 201 for path in (ROOT/'test').glob('*.py'): shutil.copyfile(path,harness/path.name) 202 tooling={path.name:sha(path) for path in harness.glob('*.py')} 203 memory,seconds=(1024,120) if args.probe else (32768,3600) 204 definition=dict(schema='tricu.cross-host-experiment.v1',source_commit=commit, 205 io_version=args.io_version,source_map=map_name(args.io_version),seed_manifest_sha256=seed_pin, 206 source_archive_sha256=sha(archive), excluded_source_paths=['ext/zig/result'], 207 runtime=dict(path=str(runtime),sha256=sha(runtime)), 208 zig_library=dict(path=str(library),sha256=sha(library)), 209 python=dict(path=sys.executable,sha256=sha(Path(sys.executable))), 210 harness_sha256=sha(Path(__file__)), tooling_sha256=tooling, 211 isolation_tools={name:dict(path=str(Path(shutil.which(name)).resolve()),sha256=sha(Path(shutil.which(name)))) for name in ('unshare','bwrap')}, 212 limits=dict(memory_mib=memory,wall_seconds=seconds,capabilities=8), 213 expected='H2 = H3 = Z3: complete objects, receipts, checkpoints and three bundles', 214 comparison_exclusions=['runtime/transport provenance','timings','logs','filesystem locations','invocation records'], 215 **snapshot(map_text,entries,source)) 216 save(directory/(attempt+'-definition.json' if args.continue_experiment else 'experiment.json'),definition) 217 report=dict(complete=False,scope='probe' if args.probe else 'full',stages=[]) 218 save(directory/'report.json',report) 219 def job(label,kit,host): 220 # Verify the kit before crossing the sandbox boundary; the shared runner 221 # independently captures/authenticates it again inside. 222 raw, _, _ = load_kit(kit,'stable') 223 if kit_io_version(json.loads(raw)) != args.io_version: 224 raise ValueError('cross-host kit IO version mismatch') 225 stage=directory/label 226 if args.continue_experiment and stage.exists(): 227 if (stage/'generation/status.json').is_file(): 228 inputs,_,_=verify_generation(stage/'generation',runtime if host=='haskell' else library,source,host) 229 if inputs['compiler_bundle_sha256']!=sha(kit/'rebuilt-compiler.arboricx') or inputs['adapter_sha256']!=sha(kit/'io-adapter.arboricx'): 230 raise ValueError('completed job used a different kit') 231 report['stages'].append(label) 232 save(directory/'report.json',report) 233 print('Re-audited completed '+label,flush=True) 234 return stage/'generation' 235 if (stage/'generation').exists(): 236 raise ValueError('cannot resume a partially executed generation; select a fresh experiment') 237 stage.rename(directory/(label+'-launch-failed-'+str(time.time_ns()))) 238 stage.mkdir() 239 command=sandbox_command(source,kit,stage,runtime,library,host, 240 'native_math' if args.probe else 'source_map',memory,seconds,args.io_version) 241 save(stage/'sandbox-command.json',command) 242 try: 243 reply,_=run_guarded(command,memory_mib=memory,seconds=seconds+30) 244 except Exception as error: 245 (stage/'supervisor.log').write_text(str(getattr(error,'output',''))+'\n'+str(error)) 246 report['failure']=label 247 save(directory/'report.json',report) 248 raise 249 (stage/'supervisor.log').write_text(reply) 250 report['stages'].append(label) 251 save(directory/'report.json',report) 252 print('Completed isolated '+label,flush=True) 253 return stage/'generation' 254 def kit_from(run,label): 255 return kit_from_generation(run,directory/(label+'-kit')) 256 first=job('H1',directory/'seed-kit','haskell') 257 if args.probe: 258 zig=job('Z3',directory/'seed-kit','zig') 259 if inventory(first)!=inventory(zig): raise ValueError('probe complete object inventories differ') 260 else: 261 second=job('H2',kit_from(first,'H1'),'haskell') 262 accepted=verify(first,second,runtime,source) 263 if (directory/'accepted-pair.json').exists(): 264 if json.loads((directory/'accepted-pair.json').read_text())!=accepted: 265 raise ValueError('existing pair acceptance changed') 266 else: 267 save(directory/'accepted-pair.json',accepted) 268 kit=kit_from(second,'H2') 269 control=job('H3',kit,'haskell') 270 zig=job('Z3',kit,'zig') 271 save(directory/'accepted-cross-host.json',verify_cross(first,second,control,zig,runtime,library,source)) 272 if tooling != {name:sha(ROOT/'test'/name) for name in tooling}: 273 raise ValueError('experiment tooling changed during execution') 274 report['complete']=True 275 save(directory/'report.json',report) 276 label='Isolated mapped-library probe' if args.probe else 'Cross-runtime compiler reproduction' 277 print(label+' verified: '+str(directory)) 278 279 280 if __name__=='__main__': 281 main()