cross_host_generation.py (11307B)
1 #!/usr/bin/env python3 2 """Bounded compiler compatibility checks through the shared generation runner. 3 Haskell supplies transport and result execution; each selected execution host 4 compiles and publishes the test modules. Independent audits verify the stores. 5 """ 6 import argparse 7 import hashlib 8 import json 9 from pathlib import Path 10 import sys 11 import subprocess 12 from process_rss import run_guarded 13 14 ROOT = Path(__file__).resolve().parent.parent 15 16 17 def main(): 18 p = argparse.ArgumentParser(description=__doc__) 19 p.add_argument('--tricu', type=Path, required=True) 20 p.add_argument('--library', type=Path, required=True) 21 p.add_argument('--directory', type=Path, required=True) 22 p.add_argument('--zig-gc-policy', type=int, nargs=2, metavar=('MIN_ALLOCATIONS', 'LIVE_MULTIPLIER')) 23 p.add_argument('--compatibility', action='store_true', help='also check broader syntax, invalidation and negative controls') 24 args = p.parse_args() 25 if args.zig_gc_policy and any(not 0 < n <= 0xffffffff for n in args.zig_gc_policy): 26 p.error('GC policy requires positive u32 values') 27 directory = args.directory.resolve() 28 directory.mkdir(parents=True, exist_ok=False) 29 source = directory / 'source' 30 source.mkdir() 31 (source / 'dep.tri').write_text('!export answer\nanswer = 41\n') 32 (source / 'app.tri').write_text('!import dep as D\n!export main\nmain = D.answer\n') 33 source_map = directory / 'source-map.v1' 34 source_map.write_text('arboricx.source-map.v1\nnative\tdep\tdep\tdep.tri\t-\nnative\tapp\tapp\tapp.tri\tdep\n') 35 report = dict(complete=False, stages=[]) 36 def save(): 37 (directory / 'report.json').write_text(json.dumps(report, indent=2)+'\n') 38 def run(label, command, success=True): 39 try: 40 output, peak = run_guarded(list(map(str,command)), memory_mib=1024, seconds=150) 41 if not success: 42 raise AssertionError('unexpected success: '+label) 43 except subprocess.CalledProcessError as error: 44 (directory / (label+'.log')).write_text(error.output) 45 if success: 46 report['failure'] = label 47 save() 48 raise 49 report['stages'].append(dict(label=label, expected_failure=True)) 50 save() 51 return error.output 52 except Exception as error: 53 (directory / (label+'.log')).write_text(str(getattr(error,'output',''))+'\n'+str(error)) 54 report['failure'] = label 55 save() 56 raise 57 (directory / (label+'.log')).write_text(output) 58 report['stages'].append(dict(label=label,peak_kib=peak)) 59 save() 60 return output 61 def checkpoints(host): 62 return {name:(directory/host/'cas/cache/source-build'/name).read_bytes() for name in ('dep','app')} 63 save() 64 commands = {} 65 for host in ('haskell','zig'): 66 output = directory / host 67 command = [sys.executable, ROOT/'test/bootstrap_generations.py', output, 68 '--bootstrap-kit', ROOT/'bootstrap/rebuild', '--tricu',args.tricu.resolve(), 69 '--execution-host',host,'--source-root',source,'--map',source_map,'--root','app', 70 '--backend','stable','--memory-mib','1024','--wall-seconds','120', 71 '--initial-seconds','60','--extended-seconds','100'] 72 if host == 'zig': 73 command += ['--zig-library',args.library.resolve()] 74 if args.zig_gc_policy: 75 command += ['--zig-gc-policy', *map(str, args.zig_gc_policy)] 76 commands[host] = command 77 run(host+'-cold',command+['--fresh']) 78 status = json.loads((output/'status.json').read_text()) 79 assert status['complete'] and status['execution_host']==host and status['fresh_store'] 80 assert status['transport_host']=='haskell' 81 if host == 'zig' and args.zig_gc_policy: 82 assert status['zig_gc_policy'] == args.zig_gc_policy 83 assert json.loads((output/'execution.json').read_text())['gc_policy'] == args.zig_gc_policy 84 runtime = args.tricu if host == 'haskell' else args.library 85 assert status['runtime_sha256']==hashlib.sha256(runtime.read_bytes()).hexdigest() 86 invocation = json.loads((output/'invocation.json').read_text()) 87 assert ('+RTS' in invocation) == (host == 'haskell') 88 cold = checkpoints(host) 89 run(host+'-warm',command) 90 assert cold == checkpoints(host) 91 pin = cold['app'].decode().split('\t')[2] 92 answer = run(host+'-result',[args.tricu,'arboricx','run','--store',output/'cas', 93 '--module',pin,'--name','main','--format','number', 94 '+RTS','-N1','-M1024m','-RTS']) 95 assert answer.strip()=='41', answer 96 assert checkpoints('haskell') == checkpoints('zig'), 'complete module receipts differ' 97 # Exact independently audited object sets, not just root result equality. 98 objects = lambda host: {p.name:p.read_bytes() for p in (directory/host/'cas/objects').glob('*/*')} 99 assert objects('haskell') == objects('zig'), 'published object sets differ' 100 if args.compatibility: 101 def same_objects(): 102 assert objects('haskell') == objects('zig'), 'published object sets differ' 103 def stage(label, success=True, diagnostic=None): 104 for host, command in commands.items(): 105 reply = run(host+'-'+label, command, success=success) 106 status = json.loads((directory/host/'status.json').read_text()) 107 assert status['complete'] == success 108 if diagnostic: 109 assert diagnostic in reply, (host,label,reply[-3000:]) 110 same_objects() 111 def result(host, name, expected): 112 pin = (directory/host/'cas/cache/source-build/app').read_text().split('\t')[2] 113 reply = run(host+'-result-'+name,[args.tricu,'arboricx','run','--store',directory/host/'cas', 114 '--module',pin,'--name',name,'--format','number', 115 '+RTS','-N1','-M1024m','-RTS']) 116 assert reply.strip()==str(expected), reply 117 dep = '!export answer\n!export extra\nanswer = 42\nextra = 99\n-- preserved UTF-8: λ 😀\n' 118 (source/'dep.tri').write_text(dep) 119 (source/'bridge.tri').write_text('!export * from dep\n') 120 source_map.write_text('arboricx.source-map.v1\nnative\tdep\tdep\tdep.tri\t-\n' 121 'native\tbridge\tbridge\tbridge.tri\tdep\n' 122 'native\tapp\tapp\tapp.tri\tbridge\n') 123 app = '!import bridge as D\n!export main\n!export local\n!export pinned\n!export sequenced\n!export lazy\n!export partial\nwalk t = D.answer\nwalk (t x) = walk x\nwalk _ = 0\nmain = walk (t (t t))\nlocal = (let rec\n copy [] = []\n copy [x | xs] = t x (copy xs)\n copy _ = []\n in (x : case x of\n [head | _] = head\n _ = 0) (copy [42 8]))\nchoose ^D.extra = 42\nchoose _ = 0\npinned = choose 99\npass value next = next value\nsequenced = (do pass\n value <- 42\n (x : x) value)\nlazy = (x : 42) ((x : x x) (x : x x))\nfirst x y = x\npartial = first 42 0\n' 124 (source/'app.tri').write_text(app) 125 stage('features-and-change') 126 for host in commands: 127 for name in ('main','local','pinned','sequenced','lazy','partial'): 128 result(host,name,42) 129 before = {host:checkpoints(host) for host in commands} 130 stage('features-warm') 131 assert before == {host:checkpoints(host) for host in commands} 132 # Invalid source, not merely a supervisor preflight error. 133 (source/'app.tri').write_text('!import bridge as D\n!export main\nmain = missingName\n') 134 stage('undefined-name',success=False) 135 assert json.loads((directory/'zig/execution.json').read_text())['diagnostic']=={'first_number':20} 136 assert 'error: [20, "missingName"' in (directory/'haskell-undefined-name.log').read_text() 137 (source/'app.tri').write_text(app) 138 stage('restore-source') 139 assert before == {host:checkpoints(host) for host in commands} 140 # A matching corrupt local checkpoint must fail closed, not recompile. 141 saved = {} 142 for host in commands: 143 path = directory/host/'cas/cache/source-build/app' 144 saved[host] = path.read_bytes() 145 path.write_text('corrupt') 146 stage('corrupt-checkpoint',success=False) 147 for host in commands: 148 (directory/host/'cas/cache/source-build/app').write_bytes(saved[host]) 149 # Corrupt a referenced executable while keeping its digest/path. 150 for host in commands: 151 pin = saved[host].decode().split('\t')[2] 152 manifest = (directory/host/'cas/objects'/pin[:3]/pin).read_text() 153 digest = next(row.split('\t')[3] for row in manifest.splitlines()[1:] if row.split('\t')[1]=='main') 154 path = directory/host/'cas/objects'/digest[:3]/digest 155 saved[host] = (path,path.read_bytes()) 156 path.write_bytes(b'corrupt') 157 stage('corrupt-executable',success=False) 158 for path,payload in saved.values(): path.write_bytes(payload) 159 stage('restore-object') 160 # An interrupted preparation must invalidate success; completed dependency 161 # publications survive and the next invocation reconstructs the root. 162 for host,command in commands.items(): 163 (directory/host/'cas/cache/source-build/app').unlink() 164 run(host+'-resource-stop',command+['--memory-mib','1'],success=False) 165 assert not json.loads((directory/host/'status.json').read_text())['complete'] 166 # Also stop a running Zig invocation, not only Haskell transport setup. 167 run('zig-execution-deadline',commands['zig']+['--wall-seconds','0.5'],success=False) 168 stopped = json.loads((directory/'zig/status.json').read_text()) 169 assert not stopped['complete'] and stopped['reason']=='whole-job wall-time guard', stopped 170 assert 'code' in stopped, 'guard must reach the execution subprocess' 171 stage('restart') 172 assert before == {host:checkpoints(host) for host in commands} 173 # Header preflight rejects initial BOM before an invocation starts; 174 # existing status remains historical, not evidence of this failed attempt. 175 (source/'app.tri').write_bytes(b'\xef\xbb\xbf'+app.encode()) 176 for host, command in commands.items(): 177 reply = run(host+'-source-bom',command,success=False) 178 assert 'header mismatch' in reply 179 (source/'app.tri').write_text(app) 180 stage('final-restored') 181 for host,command in commands.items(): 182 reply = run(host+'-missing-dependency',command+['--root','absent'],success=False) 183 assert 'undeclared map dependency' in reply 184 reply = run('zig-missing-library',commands['zig']+['--zig-library',str(directory/'missing.so')],success=False) 185 assert 'no fallback' in reply 186 report['compatibility'] = True 187 report['complete'] = True 188 save() 189 print('Shared-runner cold/warm parity and exact receipts/objects passed; compatibility=' 190 +str(args.compatibility)+': '+str(directory)) 191 192 193 if __name__ == '__main__': 194 main()