io_generation.py (12644B)
1 #!/usr/bin/env python3 2 """Prepare an explicit v2 map-builder kit and exercise a scoped generation. 3 4 The current compiler prepares v2 actions. The real generation runner 5 then stages all mapped sources and builds native_math over v2. This is deliberately 6 NOT a full compiler generation pair or deployment/retention acceptance. 7 """ 8 import argparse 9 import hashlib 10 import json 11 from pathlib import Path 12 import subprocess 13 import sys 14 from source_order import Runner 15 from bootstrap_generations import source_map 16 from bootstrap_inputs import load_kit 17 from generation_io import produced_adapter, versions 18 19 ROOT = Path(__file__).resolve().parent.parent 20 21 22 def main(): 23 p = argparse.ArgumentParser(description=__doc__) 24 p.add_argument('tricu', type=Path) 25 p.add_argument('--kit', type=Path, help='validate an explicitly prepared v2 kit instead of preparing a current candidate') 26 p.add_argument('--directory', type=Path, required=True) 27 p.add_argument('--haskell-library',type=Path,help='also run the explicit v2 executor and compare complete inventories') 28 args = p.parse_args() 29 root = args.directory.resolve(); root.mkdir(parents=True, exist_ok=False) 30 mapping = ROOT/'bootstrap/compiler-source-map.v1' 31 runner = Runner(args.tricu, root) 32 if args.kit: 33 kit = args.kit.resolve() 34 raw, _, _ = load_kit(kit, 'stable') 35 metadata = json.loads(raw) 36 from bootstrap_inputs import kit_io_version 37 if kit_io_version(metadata) != 2: 38 raise ValueError('this probe requires an explicit v2 kit') 39 else: 40 projection = root/'projection'; projection.mkdir() 41 (projection/'inputs').mkdir() 42 _, entries, _ = source_map(mapping, ROOT, 'source_map') 43 workspace = ['tricu.workspace.v1'] 44 for alias, (_, relative, _) in entries.items(): 45 target = 'inputs/'+alias+'.tri' 46 (projection/target).write_bytes((ROOT/relative).read_bytes()) 47 workspace.append(alias+'\t'+target) 48 (projection/'tricu.workspace').write_text('\n'.join(workspace)+'\n') 49 worker = projection/'worker.tri' 50 worker.write_text('!import source_map as M\n!export sourceMapBuildStable\nsourceMapBuildStable = M.sourceMapBuildStable\n') 51 adapter = projection/'adapter.tri' 52 adapter.write_text('!import io as IO\n!export main\nmain build source overlay store compiler mapping root = IO.io (build source overlay store compiler mapping root)\n') 53 kit = root/'kit'; kit.mkdir() 54 for label, source, destination in [('worker',worker,'rebuilt-compiler.arboricx'),('adapter',adapter,'io-adapter.arboricx')]: 55 runner.run('compile-'+label,['compile','--file',source,'--store',runner.store, 56 '--output',kit/destination,'--compile-timeout','600'],seconds=660) 57 metadata = dict(schema='tricu.bootstrap-kit.v2', backend='stable', io_version=2, artifacts={}) 58 for role, filename, export in [('compiler','rebuilt-compiler.arboricx','sourceMapBuildStable'),('adapter','io-adapter.arboricx','main')]: 59 metadata['artifacts'][role]=dict(file=filename,export=export,sha256=hashlib.sha256((kit/filename).read_bytes()).hexdigest()) 60 (kit/'bootstrap-kit.json').write_text(json.dumps(metadata,indent=2)+'\n') 61 load_kit(kit,'stable') 62 checks=[] 63 def generation(label, extra=(), success=True, selected_kit=kit, preparation_failure=False): 64 output=root/label 65 cmd=[sys.executable,str(ROOT/'test/bootstrap_generations.py'),str(output), 66 '--bootstrap-kit',str(selected_kit),'--tricu',str(args.tricu.resolve()), 67 '--map',str(mapping),'--source-root',str(ROOT),'--root','native_math', 68 '--backend','stable','--memory-mib','8192','--wall-seconds','600',*extra] 69 reply=subprocess.run(cmd,capture_output=True,text=True,timeout=660) 70 (root/(label+'.log')).write_text(reply.stdout+reply.stderr) 71 assert (reply.returncode==0)==success,(label,reply.stdout[-2000:],reply.stderr) 72 if success: 73 status=json.loads((output/'status.json').read_text()) 74 assert status['complete'] and versions(status)==(2,2) 75 assert produced_adapter(output,status)==(kit/'io-adapter.arboricx').read_bytes() 76 invocation=json.loads((output/'invocation.json').read_text()) 77 assert '--unsafe-io' not in invocation 78 assert not list(output.rglob('.tricu-*')) 79 elif preparation_failure: 80 assert not json.loads((output/'status.json').read_text())['complete'] 81 assert not (output/'cas').exists(), 'invalid adapter started compilation' 82 else: 83 assert not output.exists(), 'selection failure created output' 84 checks.append(label) 85 return output 86 generation('wrong-version',('--execution-io-version','1'),False) 87 generation('missing-produced-adapter',('--produced-io-version','1'),False) 88 generation('noncanonical-version',('--execution-io-version','02'),False) 89 fake_library=root/'not-an-embedding-library'; fake_library.write_bytes(b'not loaded') 90 generation('invalid-embedding-runtime',('--execution-host','haskell-ffi','--haskell-library',str(fake_library)),False) 91 generation('invalid-produced-adapter',('--produced-io-version','1','--produced-adapter',str(fake_library)),False,preparation_failure=True) 92 output=generation('scoped') 93 # Independent existing store/receipt audit runs inside the generation runner. 94 # Compare actual complete native_math manifest with the authenticated packaged v2 kit. 95 control=root/'packaged-control' 96 command=[sys.executable,str(ROOT/'test/bootstrap_generations.py'),str(control), 97 '--bootstrap-kit',str(ROOT/'bootstrap/rebuild'),'--tricu',str(args.tricu.resolve()), 98 '--map',str(mapping),'--source-root',str(ROOT),'--root','native_math', 99 '--backend','stable','--memory-mib','8192','--wall-seconds','600', 100 '--produced-io-version','2','--produced-adapter',str(kit/'io-adapter.arboricx')] 101 reply=subprocess.run(command,capture_output=True,text=True,timeout=660) 102 (root/'packaged-control.log').write_text(reply.stdout+reply.stderr) 103 assert reply.returncode==0,reply.stderr 104 def manifest(directory): 105 return (directory/'cas/cache/source-build/native_math').read_text().split('\t')[2] 106 assert manifest(output)==manifest(control) 107 assert versions(json.loads((control/'status.json').read_text()))==(2,2) 108 checks.append('selected-packaged-complete-manifest-equality') 109 # Exercise both unchanged native bytes and rewritten unit/reexport overlays. 110 # The independent generation audit checks both source-domain digests in every 111 # map receipt; exact overlay expectations also detect accidentally reused raw 112 # digests on the rewritten branch. 113 fixture=root/'mixed-source'; fixture.mkdir() 114 originals={'dep':b'answer = 42\n', 115 'native_math':b'!import "dependency" !Local\nlocal = answer\n', 116 'native':b'!export value\n-- UTF-8: \xce\xbb\nvalue = 7\n'} 117 for name,contents in originals.items(): 118 (fixture/(name+'.tri')).write_bytes(contents) 119 mixed_map=fixture/'map.v1' 120 mixed_map.write_text('arboricx.source-map.v1\n' 121 'unit\tdep\tdependency\tdep.tri\t-\n' 122 'reexport\tnative_math\tmain\tnative_math.tri\tdep\n' 123 'native\tnative\tnative\tnative.tri\t-\n') 124 mixed_manifests=[] 125 selections=[('candidate',kit,[]),('packaged',ROOT/'bootstrap/rebuild',[])] 126 if args.haskell_library: 127 selections.append(('haskell-ffi',kit,['--execution-host','haskell-ffi','--haskell-library',str(args.haskell_library.resolve())])) 128 for label,selected,extra in selections: 129 destination=root/('mixed-'+label) 130 command=[sys.executable,str(ROOT/'test/bootstrap_generations.py'),str(destination), 131 '--bootstrap-kit',str(selected),'--tricu',str(args.tricu.resolve()), 132 '--map',str(mixed_map),'--source-root',str(fixture),'--root','native_math', 133 '--backend','stable','--capabilities','1','--memory-mib','8192','--wall-seconds','180',*extra] 134 reply=subprocess.run(command,capture_output=True,text=True,timeout=210) 135 (root/('mixed-'+label+'.log')).write_text(reply.stdout+reply.stderr) 136 assert reply.returncode==0,(label,reply.stdout[-2000:],reply.stderr) 137 # Legacy lines/unlines emits the trailing empty line as a newline. 138 expected={'dep':b'!export *\nanswer = 42\n\n', 139 'native_math':b'!import dep *\n!export *\n!export * from dep\nlocal = answer\n\n', 140 'native':originals['native']} 141 for name,contents in expected.items(): 142 assert (destination/'overlay'/(name+'.tri')).read_bytes()==contents,(label,name) 143 mixed_manifests.append(manifest(destination)) 144 checks.append('mixed-staging-digests-and-overlays-'+label) 145 assert len(set(mixed_manifests))==1,mixed_manifests 146 ffi_output = None 147 if args.haskell_library: 148 ffi_output = generation('ffi-scoped',('--execution-host','haskell-ffi','--haskell-library',str(args.haskell_library.resolve()))) 149 def inventory(directory): 150 return {p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (directory/'cas/objects').glob('*/*')} 151 assert inventory(ffi_output)==inventory(output) 152 assert manifest(ffi_output)==manifest(output) 153 checks.append('cli-ffi-complete-inventory-equality') 154 from haskell_generation_executor import validate_library 155 try: 156 validate_library(args.haskell_library.resolve(), 1) 157 except (ValueError, AttributeError): 158 checks.append('current-embedding-rejects-v1-before-preparation') 159 else: 160 raise AssertionError('current runtime still exposes v1 execution') 161 # Captured request tampering must fail before any IO execution, not select 162 # another interpreter or silently reinterpret absent/Boolean metadata. 163 request_path=ffi_output/'execution-request.json' 164 original=request_path.read_bytes() 165 map_path=ffi_output/'source-map.v1' 166 original_map=map_path.read_bytes() 167 before=inventory(ffi_output) 168 command=json.loads((ffi_output/'invocation.json').read_text()) 169 try: 170 for label,value in [('wrong',1),('boolean',True),('future',3),('missing',None)]: 171 request=json.loads(original) 172 if value is None: request.pop('execution_io_version') 173 else: request['execution_io_version']=value 174 request_path.write_text(json.dumps(request)) 175 reply=subprocess.run(command,capture_output=True,text=True,timeout=30) 176 assert reply.returncode!=0,(label,reply.stdout) 177 assert 'IO version' in reply.stderr,(label,reply.stderr) 178 assert inventory(ffi_output)==before 179 checks.append('ffi-request-'+label) 180 for key in ['worker_export', 'compiler_bundle_sha256', 'adapter_sha256', 'arguments', 'extra']: 181 request=json.loads(original) 182 if key=='arguments': 183 request[key][2]=str(root/'undeclared-store') 184 elif key=='extra': 185 request[key]=True 186 else: 187 request[key]='0'*64 188 request_path.write_text(json.dumps(request)) 189 reply=subprocess.run(command,capture_output=True,text=True,timeout=30) 190 assert reply.returncode!=0 and 'identity/arguments mismatch' in reply.stderr,(key,reply.stderr) 191 assert inventory(ffi_output)==before and not (root/'undeclared-store').exists() 192 checks.append('ffi-request-'+key) 193 request=json.loads(original) 194 request['arguments'][4]+='\n' 195 map_path.write_bytes(original_map+b'\n') 196 request_path.write_text(json.dumps(request)) 197 reply=subprocess.run(command,capture_output=True,text=True,timeout=30) 198 assert reply.returncode!=0 and 'execution map identity mismatch' in reply.stderr 199 assert inventory(ffi_output)==before 200 checks.append('ffi-captured-map-tampering') 201 finally: 202 map_path.write_bytes(original_map) 203 request_path.write_bytes(original) 204 (root/'report.json').write_text(json.dumps(dict(checks=checks,records=runner.records, 205 kit=metadata,scoped_generation=str(output),ffi_generation=str(ffi_output) if ffi_output else None,generation_pair=False),indent=2)+'\n') 206 print(f'{len(checks)} generation boundary controls passed; no full compiler pair built') 207 208 209 if __name__=='__main__': main()