bundle_batch.py (20729B)
1 #!/usr/bin/env python3 2 """Opt-in current portable bundle boundary probes; no publication/generation.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 import re 8 import struct 9 import time 10 from collections_batch import ProbeRunner, audit_workspace 11 from native_library_migration import digest 12 from fixture_sources import verify 13 from bundle_dag import merge_bundles, read 14 from base_workloads import encode as n 15 from http_headers import LEAF, TRUE, pair, result, tree_list, byte_tree, expect 16 from arboricx_readers import Reader, Failure, product, string, u32, cases as old_cases 17 from stable_bundle import assembled 18 19 ROOT=Path(__file__).resolve().parent.parent 20 REFERENCE_FIXTURES = ROOT/'test/fixtures' 21 SOURCES={name:'lib/'+name+'.tri' for name in ['base','bytes','conversions','prelude','binary']} 22 SOURCES.update(native_math='lib/math/native.tri',arboricx='lib/arboricx/arboricx.tri') 23 SOURCES.update({'arboricx_'+name:'lib/arboricx/'+name+'.tri' for name in ['common','manifest','nodes','dispatch']}) 24 DRIVER='''!import binary as B 25 !import arboricx as A 26 !import arboricx_nodes as N 27 !import arboricx_manifest as M 28 !import arboricx_dispatch as D 29 !import prelude as P 30 !export * 31 spin x = spin x 32 badTag x = D.runArboricxTyped 6 (spin x) (spin x) 33 badArgs x = A.runArboricxArgsByName [] (spin x) (t t) 34 zeroRead x = P.fst (B.readBytes 0 (spin x)) 35 codec (tag, value) = A.wrapHostValueByTag tag value 99 36 codec _ = t 37 execute bytes = A.readArboricxExecutable bytes 38 validate bytes = A.readArboricxBundle bytes 39 prefix bytes = M.readManifest bytes 40 complete bytes = M.readManifestComplete bytes 41 nodeCheck records = N.validateNodeRecords records 99 42 reconstruct (records, index) = N.nodeIndexToTree records index 43 reconstruct _ = t 44 runLeaf bytes = D.runArboricxTyped 0 bytes [] 45 ''' 46 FIELDS=[b'arboricx.bundle.manifest.v1',b'tree-calculus-executable-object',b'tree-calculus.v1',b'indexed',b'arboricx.indexed.node.v1',b'arboricx.indexed.payload.v1',b'tree-calculus.v1',b'normal-order',b'arboricx.abi.tree.v1'] 47 def manifest(*,minor=1,fields=None,caps=(),roots=((0,b'default'),),exports=((b'main',0,b'term',b'arboricx.abi.tree.v1'),),metadata=(),extensions=()): 48 def tlvs(values):return u32(len(values))+b''.join(struct.pack('>H',tag)+string(value) for tag,value in values) 49 return (b'ARBMNFST'+struct.pack('>HH',1,minor)+b''.join(string(x) for x in (fields or FIELDS)) 50 +u32(len(caps))+b''.join(string(x) for x in caps)+b'\0'+u32(len(roots)) 51 +b''.join(u32(i)+string(role) for i,role in roots)+u32(len(exports)) 52 +b''.join(string(name)+u32(index)+string(kind)+string(abi) for name,index,kind,abi in exports) 53 +tlvs(metadata)+tlvs(extensions)) 54 def nodes(records):return len(records).to_bytes(8,'big')+b''.join(string(x) for x in records) 55 def bad(code,rest=LEAF):return result(False,n(code),rest) 56 def cases(): 57 rows=[] 58 def add(alias,export,args,expected):rows.append((str(len(rows)),alias,export,args,expected)) 59 # Preserve field/truncation oracles, not historical full-reader accidents. 60 for module,label,export,args,expected in old_cases(): 61 if expected is None or label.startswith('full-suffix') or export=='validateNodeRecords':continue 62 if export=='readManifestCore': 63 # Original corpus uses version 1.0. Keep that frozen historical oracle; 64 # new version tests below explicitly reject it. 65 continue 66 add('arboricx_'+module,export,args,expected) 67 for data in [b'',b'a',bytes(range(32))]: 68 for count in [0,1,len(data),len(data)+1,2**32]: 69 add('binary','readBytes',[n(count),byte_tree(data)],result(True,byte_tree(data[:count]),byte_tree(data[count:])) if count<=len(data) else bad(1,byte_tree(data))) 70 for raw in [TRUE,pair(n(256),LEAF),pair(pair(LEAF,LEAF),LEAF)]: 71 add('binary','readU8',[raw],bad(15,raw)) 72 add('binary','readBytes',[TRUE,byte_tree(b'x')],bad(16,byte_tree(b'x'))) 73 add('driver','zeroRead',[LEAF],TRUE) 74 meta=((3,b'description'),);ext=((900,b'opaque'),) 75 raw=manifest(metadata=meta,extensions=ext) 76 core=Reader(raw).core() 77 parsed=pair(core,pair(tree_list([pair(byte_tree(b'\0\3'),byte_tree(b'description'))]),tree_list([pair(byte_tree(struct.pack('>H',900)),byte_tree(b'opaque'))]))) 78 for suffix in [b'',b'TAIL']: 79 add('driver','prefix',[byte_tree(raw+suffix)],result(True,parsed,byte_tree(suffix))) 80 add('driver','complete',[byte_tree(raw+suffix)],bad(12,byte_tree(suffix)) if suffix else result(True,parsed)) 81 # Every truncation of a valid versioned core/full manifest, independent reader. 82 for size in range(len(raw)): 83 data=raw[:size];r=Reader(data) 84 try: 85 c=r.core();m=r.repeated(r.tlv);e=r.repeated(r.tlv);expected=result(True,pair(c,pair(m,e)),byte_tree(r.data)) 86 except Failure as failure:expected=failure.tree 87 add('driver','prefix',[byte_tree(data)],expected) 88 add('driver','prefix',[byte_tree(manifest(minor=0))],bad(11,byte_tree(manifest(minor=0)[12:]))) 89 recordsets=[[],[b'\0'],[b'\0',b'\0'],[b'\1'+u32(0)],[b'\0',b'\1'+u32(0)], 90 [b'\0',b'\1'+u32(1)],[b'\0',b'\2'+u32(0)+u32(0)],[b'\0x'],[b'\1'+u32(99),b'\3']] 91 for records in recordsets: 92 invalid=any(not (x==b'\0' or len(x)==5 and x[0]==1 or len(x)==9 and x[0]==2) for x in records) 93 code=8 if invalid else None;seen=set() 94 if code is None: 95 for i,x in enumerate(records): 96 if any(int.from_bytes(x[j:j+4],'big')>=i for j in range(1,len(x),4)):code=9;break 97 if x in seen:code=7;break 98 seen.add(x) 99 encoded=tree_list([byte_tree(x) for x in records]) 100 add('driver','nodeCheck',[encoded],bad(code,n(99)) if code else result(True,encoded,n(99))) 101 for count in [1,2,8,32]: 102 records=[b'\0']+[b'\1'+u32(i-1) for i in range(1,count)] 103 add('driver','reconstruct',[pair(tree_list([byte_tree(x) for x in records]),n(count-1))],result(True,b'\1'*(count-1)+LEAF)) 104 m=manifest();ns=nodes([b'\0']);bundle=assembled(m,ns) 105 # Container remainder is the bytes after its directory, not EOF. 106 add('driver','execute',[byte_tree(bundle)],result(True,LEAF,byte_tree(bundle[96:]))) 107 add('driver','runLeaf',[byte_tree(bundle)],result(True,pair(n(0),LEAF),byte_tree(bundle[96:]))) 108 for tag in range(7): 109 for value in [LEAF,TRUE,n(256),n(2**32),n([65,255]),n([256]),n([0x10ffff]),n([0xd800]),n([0x110000]),pair(LEAF,LEAF)]: 110 # Structural domains are authored independently, not candidate-derived. 111 from source_order import field, number 112 def nat(v): 113 if v==LEAF:return True 114 if not v or v[0]!=2:return False 115 bit,rest=field(v,0),field(v,1) 116 return bit in [LEAF,TRUE] and not (bit==LEAF and rest==LEAF) and nat(rest) 117 def values(v): 118 result=[] 119 while v and v[0]==2:result.append(field(v,0));v=field(v,1) 120 return result if v==LEAF else None 121 vs=values(value) 122 valid=[True,vs is not None and all(nat(x) and number(x)<=0x10ffff and not 0xd800<=number(x)<=0xdfff for x in vs),nat(value),value in [LEAF,TRUE],vs is not None,vs is not None and all(nat(x) and number(x)<=255 for x in vs),False][tag] 123 add('driver','codec',[pair(n(tag),value)],result(True,pair(n(tag),value),n(99)) if valid else bad(14,n(99))) 124 # Checked container mutations: independent byte offsets and error remainders. 125 for offset, replacement, code, remainder_at in [ 126 (8,b'\0\2',5,32),(10,b'\0\1',5,32),(23,b'\1',5,32),(31,b'!',5,32), 127 (36,b'\0\2',17,96),(38,b'\0\2',17,96),(40,b'\0\1',17,96), 128 (42,b'\0\1',17,96),(63,b'\1',17,96), 129 (64,u32(1),6,96),(64,u32(3),17,96), 130 (44,(0).to_bytes(8,'big'),18,96),(44,(2**63).to_bytes(8,'big'),18,96), 131 (52,(2**63).to_bytes(8,'big'),18,96), 132 (76,(96).to_bytes(8,'big'),18,96), 133 ]: 134 mutated=bundle[:offset]+replacement+bundle[offset+len(replacement):] 135 add('driver','execute',[byte_tree(mutated)],bad(code,byte_tree(mutated[remainder_at:]))) 136 for kw in [dict(caps=(b'fs',)),dict(exports=((b'main',0,b'other',FIELDS[-1]),)), 137 dict(exports=((b'main',0,b'term',b'other'),)),dict(roots=()),dict(exports=()), 138 dict(exports=((b'',0,b'term',FIELDS[-1]),)), 139 dict(exports=((b'main',0,b'term',FIELDS[-1]),)*2), 140 dict(fields=FIELDS[:7]+[b'eager']+FIELDS[8:])]: 141 add('driver','execute',[byte_tree(assembled(manifest(**kw),ns))],bad(13)) 142 for kw in [dict(roots=((1,b'default'),)),dict(exports=((b'main',1,b'term',FIELDS[-1]),)), 143 dict(exports=((b'main',0,b'term',FIELDS[-1]),(b'unselected',99,b'term',FIELDS[-1])))]: 144 data=assembled(manifest(**kw),ns) 145 add('driver','execute',[byte_tree(data)],bad(9,byte_tree(data[96:]))) 146 for payloads,code in [([b'\0',b'\0'],7),([b'\0',b'\1'+u32(1)],9),([b'\0',b'\3'],8)]: 147 add('driver','execute',[byte_tree(assembled(m,nodes(payloads)))],bad(code)) 148 add('driver','execute',[byte_tree(assembled(m+b'TAIL',ns))],bad(12,byte_tree(b'TAIL'))) 149 add('driver','execute',[byte_tree(assembled(m,ns+b'TAIL'))],bad(2,byte_tree(b'TAIL'))) 150 for length in [0,1,4,8]: 151 raw=bytes([255])*length 152 add('arboricx_common','beBytesToNat',[byte_tree(raw)],n(int.from_bytes(raw,'big'))) 153 utf8=[b'', b'ASCII', 'éλ🙂'.encode(), b'\x7f', b'\xc2\x80', b'\xdf\xbf', 154 b'\xe0\xa0\x80', b'\xed\x9f\xbf', b'\xee\x80\x80', b'\xf0\x90\x80\x80', 155 b'\xf4\x8f\xbf\xbf', b'\x80', b'\xc0\x80', b'\xc1\xbf', b'\xe0\x9f\xbf', 156 b'\xed\xa0\x80', b'\xf0\x8f\xbf\xbf', b'\xf4\x90\x80\x80', b'\xf5\x80\x80\x80', b'\xff', 157 b'\xc2', b'\xe1\x80', b'\xf1\x80\x80', b'\xc2x', b'\xe1x\x80', b'\xf1\x80x\x80'] 158 for text in utf8: 159 try:text.decode('utf8');valid=True 160 except UnicodeDecodeError:valid=False 161 add('binary','binaryUtf8?',[byte_tree(text)],TRUE if valid else LEAF) 162 invalid=manifest(exports=((b'\xff',0,b'term',FIELDS[-1]),)) 163 add('driver','execute',[byte_tree(assembled(invalid,ns))],bad(13,byte_tree(invalid[invalid.index(b'\xff')+1:]))) 164 add('driver','execute',[byte_tree(assembled(manifest(metadata=((1,b'\xff'),)),ns))],bad(13)) 165 add('driver','prefix',[byte_tree(b'!'+m[1:])],bad(10,byte_tree(b'!'+m[1:]))) 166 for name in ['id','true','false','append']: 167 path=REFERENCE_FIXTURES/(name+'.arboricx') 168 _,_,dag,exports,_=read(path);trees=[] 169 for tag,*children in dag: 170 trees.append(bytes([tag])+b''.join(trees[child] for child in children)) 171 assert len(trees[-1])<65536, 'bound fixture expansion' 172 add('driver','execute',[byte_tree(path.read_bytes())],result(True,trees[next(iter(exports.values()))],byte_tree(path.read_bytes()[96:]))) 173 add('binary','eof?',[byte_tree(b'x')],bad(2,byte_tree(b'x'))) 174 add('driver','badTag',[LEAF],bad(14)) 175 add('driver','badArgs',[LEAF],bad(14)) 176 # Unknown noncritical sections are allowed, but cannot evade range/profile checks. 177 parts=[m,ns,b'x'];offset=128;directory=[] 178 for identifier,payload in enumerate(parts,1): 179 directory.append(struct.pack('>IHHHHQQI',identifier,1,1 if identifier<3 else 0,0,0,offset,len(payload),0)) 180 offset+=len(payload) 181 optional=b'ARBORICX'+struct.pack('>HHIQQ',1,0,3,0,32)+b''.join(directory)+b''.join(parts) 182 add('driver','execute',[byte_tree(optional)],result(True,LEAF,byte_tree(optional[128:]))) 183 for kw in [dict(metadata=((900,b'\xff'),)),dict(extensions=((1,b'\xff'),))]: 184 data=assembled(manifest(**kw),ns) 185 add('driver','execute',[byte_tree(data)],result(True,LEAF,byte_tree(data[96:]))) 186 missing=bundle[:32]+u32(3)+bundle[36:38]+b'\0\0'+bundle[40:] 187 add('driver','execute',[byte_tree(missing)],bad(4,byte_tree(missing[96:]))) 188 return rows 189 190 def cost_screen(runner,root,modules): 191 baseline=root/'old-nodes.arboricx' 192 merge_bundles(baseline,[(REFERENCE_FIXTURES/'support-library.arboricx', 193 [('arboricx_nodes.validateNodeRecords','validateNodeRecords'),('arboricx_nodes.nodeIndexToTree','nodeIndexToTree')])],prune_unreachable=True) 194 pins={'reference':runner.imported(baseline,'old-nodes'),'candidate':modules['arboricx_nodes']['manifest']} 195 samples=[] 196 # Fork spines repeatedly refer to the oldest (leaf) and newest nodes: output 197 # is linear, while an association-list lookup must cross the growing prefix. 198 for size in [32,128]: 199 records=[b'\0']+[b'\2'+u32(0)+u32(i-1) for i in range(1,size)] 200 encoded=tree_list([byte_tree(x) for x in records]) 201 for name,args,expected in [('validateNodeRecords',[encoded,n(99)],result(True,encoded,n(99))), 202 ('nodeIndexToTree',[encoded,n(size-1)],result(True,b'\2\0'*(size-1)+LEAF))]: 203 argv=[] 204 for i,arg in enumerate(args): 205 path=root/f'cost-{i}.tree';path.write_bytes(arg);argv+=['--arg-tree-file',path] 206 for repetition in range(3): 207 for role in (['reference','candidate'] if repetition%2==0 else ['candidate','reference']): 208 label=f'cost-{name}-{size}-{role}-{repetition}' 209 expect(runner.invoke(label,pins[role],name,argv),expected,label) 210 record=runner.records[-1] 211 samples.append(dict(operation=name,size=size,role=role,warmup=repetition==0, 212 seconds=record['invoke_seconds'],allocated_bytes=record['allocated_bytes'])) 213 return samples 214 215 216 def main(): 217 global REFERENCE_FIXTURES 218 p=argparse.ArgumentParser(description=__doc__);p.add_argument('tricu',type=Path) 219 p.add_argument('--compiler',type=Path,required=True);p.add_argument('--directory',type=Path,required=True) 220 p.add_argument('--resume',action='store_true') 221 p.add_argument('--reference-fixtures',type=Path,required=True,help='authenticated pre-rewrite fixtures') 222 args=p.parse_args() 223 REFERENCE_FIXTURES = args.reference_fixtures.resolve() 224 root=args.directory.resolve();root.mkdir(parents=True,exist_ok=args.resume) 225 workspace=root/'workspace';workspace.mkdir(exist_ok=args.resume) 226 verify(REFERENCE_FIXTURES) 227 for alias,path in SOURCES.items():(workspace/(alias+'.tri')).write_bytes((ROOT/path).read_bytes()) 228 for stale in workspace.glob('*.tri'): 229 if stale.stem not in SOURCES and stale.stem!='driver':stale.unlink() 230 (workspace/'driver.tri').write_text(DRIVER);(root/'harness.py').write_bytes(Path(__file__).read_bytes()) 231 report=dict(complete=False,cases=[],compiler_sha256=hashlib.sha256(args.compiler.read_bytes()).hexdigest(),harness_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()) 232 runner=ProbeRunner(args.tricu,root,900) 233 try: 234 tool=runner.imported(args.compiler,'tool');bundle=root/'driver.arboricx' 235 runner.run('compile',['compile','--file',workspace/'driver.tri','--toolchain',tool,'--tool-store',runner.store,'--store',runner.store,'--output',bundle,'--compile-timeout','300'],330) 236 tool_payload=(runner.store/'objects'/tool[:3]/tool).read_bytes();assert digest('arboricx.module-manifest.v1',tool_payload)==tool 237 compiler=next(row.split('\t')[3] for row in tool_payload.decode().splitlines()[1:] if row.split('\t')[1]=='sourceBuildGraph') 238 report['modules']=audit_workspace(workspace,runner.store,compiler) 239 pin=runner.imported(bundle,'driver');assert pin 240 native=runner.run('native-driver',['import','--file',bundle,'--store',runner.store]) 241 assert re.search(r'Created module alias .* -> ([0-9a-f]{64})',native)[1]==report['modules']['driver']['manifest'] 242 report['bundle_sha256']=hashlib.sha256(bundle.read_bytes()).hexdigest() 243 report['runtime_sha256']=hashlib.sha256(args.tricu.read_bytes()).hexdigest() 244 for label,alias,export,arguments,expected in cases(): 245 argv=[] 246 for i,arg in enumerate(arguments): 247 file=root/f'argument-{i}.tree';file.write_bytes(arg);argv+=['--arg-tree-file',file] 248 actual=runner.invoke('case-'+label,report['modules'][alias]['manifest'],export,argv) 249 expect(actual,expected,label+' '+export);report['cases'].append(dict(label=label,export=export,input_sha256=[hashlib.sha256(x).hexdigest() for x in arguments],result_sha256=hashlib.sha256(actual).hexdigest())) 250 # Authenticate historical controls; they must fail the new independent oracles. 251 old=root/'reference.arboricx' 252 fixture=REFERENCE_FIXTURES/'support-library.arboricx' 253 merge_bundles(old,[(fixture,[('arboricx_manifest.readManifest','prefix'), 254 ('arboricx_common.natAdd','adder'),('arboricx.hostBytes?','bytesValid'), 255 ('arboricx_nodes.validateNodeRecords','nodes')])],prune_unreachable=True) 256 reference=runner.imported(old,'reference') 257 controls=[('prefix',[byte_tree(manifest()+b'TAIL')],result(True,pair(Reader(manifest()).core(),pair(LEAF,LEAF)),byte_tree(b'TAIL'))), 258 ('adder',[n(1),n(1)],n(2)),('bytesValid',[n([256])],LEAF), 259 ('nodes',[tree_list([byte_tree(b'\0'),byte_tree(b'\0')]),n(99)],bad(7,n(99)))] 260 for name,arguments,expected in controls: 261 argv=[] 262 for i,value in enumerate(arguments): 263 path=root/f'control-{i}.tree';path.write_bytes(value);argv+=['--arg-tree-file',path] 264 assert runner.invoke('old-'+name,reference,name,argv)!=expected 265 path=root/'wrong.tree';path.write_bytes(byte_tree(manifest()+b'TAIL')) 266 expected=result(True,pair(Reader(manifest()).core(),pair(LEAF,LEAF)),byte_tree(b'TAIL')) 267 assert runner.invoke('wrong-subject',pin,'complete',['--arg-tree-file',path])!=expected 268 report['cost_screen']=cost_screen(runner,root,report['modules']) 269 report['historical_controls']=4;report['wrong_subject_rejected']=True 270 report['surface_changes']={} 271 previous={name for path in [REFERENCE_FIXTURES/'compiler-library.arboricx',fixture] for name in read(path)[3]} 272 for alias,module in report['modules'].items(): 273 before={name.split('.',1)[1] for name in previous if name.startswith(alias+'.')} 274 if before: 275 after=set(module['exports']) 276 report['surface_changes'][alias]=dict(removed=sorted(before-after),added=sorted(after-before)) 277 expected = { 278 'binary': ({'binaryExpectU8Reply','binaryLegacyExpectCode','binaryLegacyReply','binaryLegacyRollback','binaryReadU8Reply','expectBytes_','readBytes_','readWhile_'}, 279 {'binaryByte?','binaryBytes?','binaryUtf8?','errInvalidBinaryInput','errInvalidCount','repeatParser'}), 280 'arboricx_common': (set('byteNatShiftAppend byteNatShiftAppend_ lookupSectionRecord_ natAdd natDouble natMake natTimes256 readSectionDirectory_ sectionDirectoryHasId?_'.split()), 281 set('errInvalidSectionRange errUnsupportedSection readArboricxContainerChecked sectionRecordSupported? validateSectionRanges'.split())), 282 'arboricx_manifest': (set('findExportByName_ getExportNames_ lookupMetadata_ pairFirst pairSecond readCapabilities_ readExports_ readRoots_ readTLVs_ selectExportOpt skipTLVs_'.split()), 283 set('manifestExportsValid? manifestRootsValid? readManifestComplete readUtf8String validateManifestMetadata validateManifestVersion'.split())), 284 'arboricx_nodes': (set('nodeBuildState nodeBuildStateBuiltTrees nodeBuildStateI nodeBuildStateTargetIndex nodeBuiltTreeIndex nodeBuiltTreeValue nodeChildIndicesInPrefix? nodeIndexInPrefix? nodeIndexToTree_ nodePayloadForkLeftIndex nodePayloadForkRightIndex nodePayloadIndicesValid? nodePayloadStemChildIndex nodePayloadToTreeFromBuilt nodeRecordsValidIndices? nodeRecordsValidIndicesFrom? nodeTreeByIndex nodeTreeByIndex_ nodeU32FromBytes4 readNodeRecords_ withNodePayloadForkIndices withNodePayloadStemIndex'.split()),set()), 285 'arboricx': (set(),{'hostScalar?','readArboricxDecoded'}), 286 'arboricx_dispatch': (set(),set()), 287 } 288 for alias,(removed,added) in expected.items(): 289 change=report['surface_changes'][alias] 290 assert set(change['removed'])==removed and set(change['added'])==added, 'unexpected surface: '+alias 291 for alias,path in SOURCES.items():assert (workspace/(alias+'.tri')).read_bytes()==(ROOT/path).read_bytes() 292 report.update(complete=True,seconds=time.monotonic()-runner.started) 293 print(f'PASS: {len(report["cases"])} binary/bundle cases; {report["seconds"]:.2f}s') 294 finally:(root/'report.json').write_text(json.dumps(report,indent=2)+'\n') 295 if __name__=='__main__':main()