shared_library_generation.py (11616B)
1 #!/usr/bin/env python3 2 """Bounded compiler compatibility checks through the shared generation runner. 3 Haskell supplies transport and result execution; each selected Haskell execution interface 4 compiles and publishes the test modules. Independent audits verify the stores. 5 """ 6 import argparse 7 import hashlib 8 import json 9 from pathlib import Path 10 import sys 11 import subprocess 12 from process_rss import run_guarded 13 14 ROOT = Path(__file__).resolve().parent.parent 15 16 17 def published_objects(directory): 18 """Committed names only; SIGKILL can leave private IO stages behind.""" 19 objects = {} 20 for path in (directory / 'objects').glob('*/*'): 21 if '.tricu-stage-' in path.name: 22 continue # Reserved by IO.Staging; never a publishable object name. 23 assert len(path.name) == 64 and all(c in '0123456789abcdef' for c in path.name), path 24 assert path.parent.name == path.name[:3], path 25 objects[path.name] = path.read_bytes() 26 return objects 27 28 29 def main(): 30 p = argparse.ArgumentParser(description=__doc__) 31 p.add_argument('--tricu', type=Path, required=True) 32 p.add_argument('--library', type=Path, required=True) 33 p.add_argument('--directory', type=Path, required=True) 34 p.add_argument('--compatibility', action='store_true', help='also check broader syntax, invalidation and negative controls') 35 args = p.parse_args() 36 directory = args.directory.resolve() 37 directory.mkdir(parents=True, exist_ok=False) 38 source = directory / 'source' 39 source.mkdir() 40 (source / 'dep.tri').write_text('!export answer\nanswer = 41\n') 41 (source / 'app.tri').write_text('!import dep as D\n!export main\nmain = D.answer\n') 42 source_map = directory / 'source-map.v1' 43 source_map.write_text('arboricx.source-map.v1\nnative\tdep\tdep\tdep.tri\t-\nnative\tapp\tapp\tapp.tri\tdep\n') 44 report = dict(complete=False, stages=[]) 45 def save(): 46 (directory / 'report.json').write_text(json.dumps(report, indent=2)+'\n') 47 def run(label, command, success=True): 48 try: 49 output, peak = run_guarded(list(map(str,command)), memory_mib=1024, seconds=150) 50 if not success: 51 raise AssertionError('unexpected success: '+label) 52 except subprocess.CalledProcessError as error: 53 (directory / (label+'.log')).write_text(error.output) 54 if success: 55 report['failure'] = label 56 save() 57 raise 58 report['stages'].append(dict(label=label, expected_failure=True)) 59 save() 60 return error.output 61 except Exception as error: 62 (directory / (label+'.log')).write_text(str(getattr(error,'output',''))+'\n'+str(error)) 63 report['failure'] = label 64 save() 65 raise 66 (directory / (label+'.log')).write_text(output) 67 report['stages'].append(dict(label=label,peak_kib=peak)) 68 save() 69 return output 70 def checkpoints(host): 71 return {name:(directory/host/'cas/cache/source-build'/name).read_bytes() for name in ('dep','app')} 72 save() 73 commands = {} 74 for host in ('haskell','haskell-ffi'): 75 output = directory / host 76 command = [sys.executable, ROOT/'test/bootstrap_generations.py', output, 77 '--bootstrap-kit', ROOT/'bootstrap/rebuild', '--tricu',args.tricu.resolve(), 78 '--execution-host',host,'--source-root',source,'--map',source_map,'--root','app', 79 '--backend','stable','--memory-mib','1024','--wall-seconds','120', 80 '--initial-seconds','60','--extended-seconds','100'] 81 if host == 'haskell-ffi': 82 command += ['--haskell-library',args.library.resolve()] 83 commands[host] = command 84 run(host+'-cold',command+['--fresh']) 85 status = json.loads((output/'status.json').read_text()) 86 assert status['complete'] and status['execution_host']==host and status['fresh_store'] 87 assert status['transport_host']=='haskell' 88 runtime = args.tricu if host == 'haskell' else args.library 89 assert status['runtime_sha256']==hashlib.sha256(runtime.read_bytes()).hexdigest() 90 invocation = json.loads((output/'invocation.json').read_text()) 91 assert ('+RTS' in invocation) == (host == 'haskell') 92 cold = checkpoints(host) 93 run(host+'-warm',command) 94 assert cold == checkpoints(host) 95 pin = cold['app'].decode().split('\t')[2] 96 answer = run(host+'-result',[args.tricu,'arboricx','run','--store',output/'cas', 97 '--module',pin,'--name','main','--format','number', 98 '+RTS','-N1','-M1024m','-RTS']) 99 assert answer.strip()=='41', answer 100 assert checkpoints('haskell') == checkpoints('haskell-ffi'), 'complete module receipts differ' 101 # Exact independently audited object sets, not just root result equality. 102 objects = lambda host: published_objects(directory/host/'cas') 103 assert objects('haskell') == objects('haskell-ffi'), 'published object sets differ' 104 if args.compatibility: 105 def same_objects(): 106 assert objects('haskell') == objects('haskell-ffi'), 'published object sets differ' 107 def stage(label, success=True, diagnostic=None): 108 for host, command in commands.items(): 109 reply = run(host+'-'+label, command, success=success) 110 status = json.loads((directory/host/'status.json').read_text()) 111 assert status['complete'] == success 112 if diagnostic: 113 assert diagnostic in reply, (host,label,reply[-3000:]) 114 same_objects() 115 def result(host, name, expected): 116 pin = (directory/host/'cas/cache/source-build/app').read_text().split('\t')[2] 117 reply = run(host+'-result-'+name,[args.tricu,'arboricx','run','--store',directory/host/'cas', 118 '--module',pin,'--name',name,'--format','number', 119 '+RTS','-N1','-M1024m','-RTS']) 120 assert reply.strip()==str(expected), reply 121 dep = '!export answer\n!export extra\nanswer = 42\nextra = 99\n-- preserved UTF-8: λ 😀\n' 122 (source/'dep.tri').write_text(dep) 123 (source/'bridge.tri').write_text('!export * from dep\n') 124 source_map.write_text('arboricx.source-map.v1\nnative\tdep\tdep\tdep.tri\t-\n' 125 'native\tbridge\tbridge\tbridge.tri\tdep\n' 126 'native\tapp\tapp\tapp.tri\tbridge\n') 127 app = '!import bridge as D\n!export main\n!export local\n!export pinned\n!export sequenced\n!export lazy\n!export partial\nwalk t = D.answer\nwalk (t x) = walk x\nwalk _ = 0\nmain = walk (t (t t))\nlocal = (let rec\n copy [] = []\n copy [x | xs] = t x (copy xs)\n copy _ = []\n in (x : case x of\n [head | _] = head\n _ = 0) (copy [42 8]))\nchoose ^D.extra = 42\nchoose _ = 0\npinned = choose 99\npass value next = next value\nsequenced = (do pass\n value <- 42\n (x : x) value)\nlazy = (x : 42) ((x : x x) (x : x x))\nfirst x y = x\npartial = first 42 0\n' 128 (source/'app.tri').write_text(app) 129 stage('features-and-change') 130 for host in commands: 131 for name in ('main','local','pinned','sequenced','lazy','partial'): 132 result(host,name,42) 133 before = {host:checkpoints(host) for host in commands} 134 stage('features-warm') 135 assert before == {host:checkpoints(host) for host in commands} 136 # Invalid source, not merely a supervisor preflight error. 137 (source/'app.tri').write_text('!import bridge as D\n!export main\nmain = missingName\n') 138 stage('undefined-name',success=False) 139 assert json.loads((directory/'haskell-ffi/execution.json').read_text())['diagnostic']=={'first_number':20} 140 assert 'error: [20, "missingName"' in (directory/'haskell-undefined-name.log').read_text() 141 (source/'app.tri').write_text(app) 142 stage('restore-source') 143 assert before == {host:checkpoints(host) for host in commands} 144 # A matching corrupt local checkpoint must fail closed, not recompile. 145 saved = {} 146 for host in commands: 147 path = directory/host/'cas/cache/source-build/app' 148 saved[host] = path.read_bytes() 149 path.write_text('corrupt') 150 stage('corrupt-checkpoint',success=False) 151 for host in commands: 152 (directory/host/'cas/cache/source-build/app').write_bytes(saved[host]) 153 # Corrupt a referenced executable while keeping its digest/path. 154 for host in commands: 155 pin = saved[host].decode().split('\t')[2] 156 manifest = (directory/host/'cas/objects'/pin[:3]/pin).read_text() 157 digest = next(row.split('\t')[3] for row in manifest.splitlines()[1:] if row.split('\t')[1]=='main') 158 path = directory/host/'cas/objects'/digest[:3]/digest 159 saved[host] = (path,path.read_bytes()) 160 path.write_bytes(b'corrupt') 161 stage('corrupt-executable',success=False) 162 for path,payload in saved.values(): path.write_bytes(payload) 163 stage('restore-object') 164 # An interrupted preparation must invalidate success; completed dependency 165 # publications survive and the next invocation reconstructs the root. 166 for host,command in commands.items(): 167 (directory/host/'cas/cache/source-build/app').unlink() 168 run(host+'-resource-stop',command+['--memory-mib','1'],success=False) 169 assert not json.loads((directory/host/'status.json').read_text())['complete'] 170 # Also stop a running shared-library invocation, not only Haskell transport setup. 171 run('ffi-execution-deadline',commands['haskell-ffi']+['--wall-seconds','0.5'],success=False) 172 stopped = json.loads((directory/'haskell-ffi/status.json').read_text()) 173 assert not stopped['complete'] and stopped['reason']=='whole-job wall-time guard', stopped 174 assert 'code' in stopped, 'guard must reach the execution subprocess' 175 report['interrupted_private_stages'] = { 176 host: [dict(path=str(path.relative_to(directory/host)), bytes=path.stat().st_size) 177 for path in (directory/host/'cas/objects').glob('*/*') 178 if '.tricu-stage-' in path.name] 179 for host in commands} 180 stage('restart') 181 assert before == {host:checkpoints(host) for host in commands} 182 # Header preflight rejects initial BOM before an invocation starts; 183 # existing status remains historical, not evidence of this failed attempt. 184 (source/'app.tri').write_bytes(b'\xef\xbb\xbf'+app.encode()) 185 for host, command in commands.items(): 186 reply = run(host+'-source-bom',command,success=False) 187 assert 'header mismatch' in reply 188 (source/'app.tri').write_text(app) 189 stage('final-restored') 190 for host,command in commands.items(): 191 reply = run(host+'-missing-dependency',command+['--root','absent'],success=False) 192 assert 'undeclared map dependency' in reply 193 reply = run('ffi-missing-library',commands['haskell-ffi']+['--haskell-library',str(directory/'missing.so')],success=False) 194 assert 'no fallback' in reply 195 report['compatibility'] = True 196 report['complete'] = True 197 save() 198 print('Shared-runner cold/warm parity and exact receipts/objects passed; compatibility=' 199 +str(args.compatibility)+': '+str(directory)) 200 201 202 if __name__ == '__main__': 203 main()