source_building.py (8993B)
1 #!/usr/bin/env python3 2 """Opt-in build/lock/map/bundle candidates: complete canonical bytes and demand. 3 4 Produces authenticated selected map workers for separate live build/cache tests. 5 """ 6 from pathlib import Path 7 import json 8 import struct 9 from compiler_candidate import Candidate,options,ROOT 10 from source_resolution_models import n,LEAF,TRUE,pair,tree_list,result,expect,chars 11 from source_modules import tree_cases 12 from source_order import field 13 from stable_bundle import assembled,byte_tree 14 from http_pipeline import finished 15 DRIVER='''!import prelude as B 16 !import source_bundle as W 17 !import source_lock as L 18 !import source_map as M 19 !import source_build as G 20 !import effects as E 21 !export bundle 22 !export locks 23 !export maps 24 !export receipts 25 !export integers 26 !export demand 27 !export wrong 28 bundle rows = B.map W.sourceBundleEncode rows 29 locks rows = B.map (groups : B.pair (L.sourceLockEncode groups) (L.sourceLockDecode (L.sourceLockEncode groups))) rows 30 maps rows = B.map M.sourceMapDecode rows 31 receipts rows = B.map (row : G.sourceBuildReceiptBytes "source" "compiler" "options" "manifest" (B.fst row) (B.snd row)) rows 32 integers rows = B.map (value : B.pair (W.sourceBundleU32 value) (W.sourceBundleU64 value)) rows 33 spin x = spin x 34 demand x = [ 35 (E.run (M.sourceMapStageRows [] (spin x) (spin x) (spin x) []) []) 36 (E.run (L.sourceLockVerifyRefs [] (spin x) (spin x) []) []) 37 (E.run (W.sourceBundleHydrate [] (spin x) []) []) 38 (E.run (G.sourceBuildDependencies [] (spin x) (spin x) (spin x) (spin x) (spin x) 23 (spin x) []) []) 39 (E.run (G.sourceBuildWalk "../bad" (spin x) (spin x) (spin x) (spin x) (spin x) (spin x)) []) 40 (E.run (G.sourceBuildWalk "app" (spin x) (spin x) (spin x) [(B.pair "app" t)] (B.pair [] []) (spin x)) []) 41 (E.run (G.sourceBuildWalk "app" (spin x) (spin x) (spin x) (spin x) (B.pair [(B.pair "app" 23)] []) (spin x)) []) 42 (E.run (G.sourceBuildGetManifest (spin x) "hash" (B.pair [] [(B.pair "hash" 23)])) [])] 43 wrong _ = t 44 ''' 45 46 def bundle_bytes(named): 47 nodes=[];seen={} 48 def visit(tree): 49 tag=tree[0] 50 key=(0,) if tag==0 else (1,visit(tree[1:])) if tag==1 else (2,visit(field(tree,0)),visit(field(tree,1))) 51 if key not in seen:seen[key]=len(nodes);nodes.append(key) 52 return seen[key] 53 roots=[(name,visit(tree)) for name,tree in named] 54 u32=lambda v:struct.pack('>I',v) 55 text=lambda v:u32(len(v.encode()))+v.encode() 56 strings=['arboricx.bundle.manifest.v1','tree-calculus-executable-object','tree-calculus.v1','indexed','arboricx.indexed.node.v1','arboricx.indexed.payload.v1','tree-calculus.v1','normal-order','arboricx.abi.tree.v1'] 57 manifest=b'ARBMNFST'+struct.pack('>HH',1,1)+b''.join(text(s) for s in strings)+u32(0)+b'\0' 58 manifest+=u32(len(roots))+b''.join(u32(index)+text('root' if at else 'default') for at,(_,index) in enumerate(roots)) 59 manifest+=u32(len(roots))+b''.join(text(name)+u32(index)+text('term')+text('arboricx.abi.tree.v1') for name,index in roots) 60 manifest+=u32(1)+b'\0\5'+text('arboricx')+u32(0) 61 payloads=[bytes([key[0]])+b''.join(u32(child) for child in key[1:]) for key in nodes] 62 return assembled(manifest,struct.pack('>Q',len(nodes))+b''.join(u32(len(p))+p for p in payloads)) 63 64 def groups_cases(): 65 rows=[] 66 for size in range(8): 67 groups=[('dep'+str(i),str(i)*64,[(name,str((i+1)%10)*64) for name in ['x','y%','a.b'][:i%4]]) for i in range(size)] 68 rows.append(groups) 69 return rows 70 71 def groups_tree(groups):return tree_list(pair(chars(ref),pair(chars(pin),tree_list(pair(chars(name),chars(tree)) for name,tree in entries))) for ref,pin,entries in groups) 72 def escape(text):return text.replace('%','%25').replace('\t','%09').replace('\n','%0A').replace('\r','%0D') 73 def lock_bytes(groups):return 'arboricx.source-lock.v1\n'+''.join('pin\t'+escape(ref)+'\t'+pin+'\t'+escape(name)+'\t'+tree+'\n' for ref,pin,entries in groups for name,tree in entries) 74 def lock_rows(groups):return tree_list(pair(chars(ref),pair(chars(pin),pair(chars(name),chars(tree)))) for ref,pin,entries in groups for name,tree in entries) 75 76 def map_cases(): 77 rows=[];header='arboricx.source-map.v1\n' 78 good=[('dep','library.dep','lib/dep.tri',[],'native'),('app','library.app','app.tri',['dep'],'unit')] 79 def text(entries):return header+''.join('\t'.join([kind,name,legacy,path,','.join(deps) or '-'])+'\n' for name,legacy,path,deps,kind in entries) 80 def tree(entries):return tree_list(pair(chars(name),pair(chars(legacy),pair(chars(path),pair(tree_list(chars(dep) for dep in deps),chars(kind))))) for name,legacy,path,deps,kind in entries) 81 for entries in [good[:1],good,[(name,legacy,path,deps,'reexport' if deps else kind) for name,legacy,path,deps,kind in good]]:rows.append((text(entries),result(True,tree(entries)))) 82 valid=text(good) 83 for raw,error in [('', 'unsupported source map version'),(header,'source map requires a unit'),(valid[:-1],'noncanonical source map'),(valid+'\n','blank source map row'),(text(good+[good[0]]),'duplicate source map name'),(valid.replace('lib/dep.tri','../dep.tri'),'invalid source map name, path or fields'),(valid.replace('native\t','unknown\t'),'invalid source map name, path or fields'),(valid.replace('\tdep\n','\tmissing\n'),'undeclared source map dependency'),(valid.replace('library.dep','library..dep'),'invalid source map name, path or fields')]:rows.append((raw,result(False,chars(error)))) 84 return rows 85 86 def main(): 87 args=options(__doc__).parse_args();c=Candidate(args,'source_map');r=c.runner 88 (c.root/'harness.py').write_bytes(Path(__file__).read_bytes()) 89 try: 90 # One compilation loads the producer once. Complete native dependency 91 # interfaces are audited; only consumed probe/worker roots are packed. 92 (c.workspace/'effects.tri').write_bytes((ROOT/'test/httpEffects.tri').read_bytes());(c.workspace/'building_probe.tri').write_text(DRIVER) 93 pin=r.imported(c.build('building_probe'),'probe');c.audit() 94 c.export('source_map',['sourceMapBuild','sourceMapBuildStable']) 95 named=[[]]+[[(f'x{j}',tree) for j,tree in enumerate(tree_cases()[i:i+3])] for i in range(24)] 96 expected=[result(False,chars('bundle requires an export')) if not rows else result(True,byte_tree(bundle_bytes(rows))) for rows in named] 97 expect(c.invoke(pin,'bundle',[tree_list(tree_list(pair(chars(name),tree) for name,tree in rows) for rows in named)]),tree_list(expected),'exact bundles') 98 for i,rows in enumerate(named[1:4]): 99 path=c.root/f'golden-{i}.arboricx';path.write_bytes(bundle_bytes(rows));gold=r.imported(path,'golden-'+str(i)) 100 for name,tree in rows:expect(c.invoke(gold,name,[],label=f'golden-{i}-{name}'),tree,'host golden execution') 101 groups=groups_cases();expect(c.invoke(pin,'locks',[tree_list(groups_tree(g) for g in groups)]),tree_list(pair(chars(lock_bytes(g)),result(True,lock_rows(g))) for g in groups),'locks') 102 maps=map_cases();expect(c.invoke(pin,'maps',[tree_list(chars(text) for text,_ in maps)]),tree_list(expected for _,expected in maps),'maps') 103 numbers=[0,1,255,256,257,65535,65536,2**32-1,2**32,2**64-1,2**64] 104 expect(c.invoke(pin,'integers',[tree_list(n(v) for v in numbers)]),tree_list(pair(byte_tree((v%(2**32)).to_bytes(4,'big')),byte_tree((v%(2**64)).to_bytes(8,'big'))) for v in numbers),'bounded integer fields') 105 args_= [];values=[] 106 for group in groups: 107 locals_=[(ref,pin,'f'*64) for ref,pin,_ in group] 108 args_.append(pair(tree_list(pair(chars(name),pair(chars(pin),chars(receipt))) for name,pin,receipt in locals_),groups_tree(group))) 109 values.append(chars('arboricx.build-receipt.v1\nsource\tsource\ncompiler\tcompiler\noptions\toptions\noutput\tmanifest\n'+''.join('local\t'+name+'\t'+pin+'\t'+receipt+'\n' for name,pin,receipt in locals_)+''.join('pin\t'+ref+'\t'+pin+'\t'+escape(name)+'\t'+tree+'\n' for ref,pin,entries in group for name,tree in entries))) 110 expect(c.invoke(pin,'receipts',[tree_list(args_)]),tree_list(values),'canonical receipts') 111 memo=pair(tree_list([pair(chars('app'),n(23))]),LEAF);manifest_memo=pair(LEAF,tree_list([pair(chars('hash'),n(23))])) 112 expected=[finished(result(True,LEAF))]*3+[finished(result(True,pair(LEAF,n(23)))),finished(result(False,chars('invalid local source name'))),finished(result(False,chars('local-source dependency cycle'))),finished(result(True,pair(n(23),memo))),finished(result(True,pair(n(23),manifest_memo)))] 113 expect(c.invoke(pin,'demand',[LEAF]),tree_list(expected),'memo, cycle and empty demand') 114 assert c.invoke(pin,'wrong',[LEAF])!=result(True,LEAF) 115 c.report.update(complete=True,bundle_cases=len(named),lock_cases=len(groups),map_cases=len(maps),integer_cases=len(numbers),receipt_cases=len(groups),demand_cases=8,wrong_subject_rejected=True) 116 except Exception as error:c.report['failure']=str(error);raise 117 finally:c.save() 118 print('PASS building',c.report['seconds']) 119 120 if __name__=='__main__':main()