io_v2_compiler_store.py (21990B)
1 #!/usr/bin/env python3 2 """Run actual source_object callers with an isolated v2 IO source projection. 3 No compiler generations, frozen-input edits or default workspace replacement. 4 """ 5 import argparse 6 import hashlib 7 import json 8 import subprocess 9 from pathlib import Path 10 from source_order import Runner 11 from process_rss import run_guarded 12 from stable_bundle import byte_tree 13 from native_indexed_store import indexed_bytes, digest 14 from executable_object import INDEXED 15 from io_v2_indexed_write import dag 16 from io_v2_objects import text, problem 17 from http_headers import number, result, tree_list 18 19 ROOT = Path(__file__).resolve().parent.parent 20 PROBE = '''!import source_object as O 21 !import compiler_io as IO 22 !import io_v2_storage as S 23 !export putRaw 24 !export getRaw 25 !export putTree 26 !export getTree 27 !export copyTree 28 !export replace 29 !export read 30 !export hash 31 !export publish 32 !export parent 33 !export exists 34 putRaw root domain bytes = IO.io (O.sourceObjectStore root domain bytes) 35 getRaw root domain digest = IO.io (O.sourceObjectGet root domain digest) 36 putTree root tree = IO.io (O.sourceObjectPutTree root tree) 37 getTree root digest = IO.io (O.sourceObjectGetTree root digest) 38 copyTree source target digest = IO.io <| IO.onOk_ (O.sourceObjectGetTree source digest) 39 (tree : O.sourceObjectPutTree target tree) 40 replace path bytes = IO.io (IO.atomicWriteBytes path bytes) 41 read path limit = IO.io (S.readBytes path limit) 42 hash limit bytes = IO.io (S.hashBytes limit bytes) 43 publish directory path mode limit bytes = IO.io (S.publishBytes directory path mode limit bytes) 44 parent path = S.parentPath path 45 exists path = IO.io (IO.fileExists path) 46 ''' 47 48 def main(): 49 parser = argparse.ArgumentParser(description=__doc__) 50 parser.add_argument('tricu',type=Path) 51 parser.add_argument('--directory',type=Path,required=True) 52 parser.add_argument('--compiler-smoke',action='store_true',help='also build a candidate stable sourceBuildGraph and compile a real dependency graph; not a generation pair') 53 parser.add_argument('--zig-probe',type=Path,help='execute source-object IO controls on the private Zig host; pure helpers stay on Haskell') 54 args=parser.parse_args() 55 root=args.directory.resolve(); root.mkdir(parents=True,exist_ok=False) 56 projection=root/'projection'; projection.mkdir() 57 (projection/'inputs').mkdir() 58 rows=['tricu.workspace.v1'] 59 for line in (ROOT/'tricu.workspace').read_text().splitlines()[1:]: 60 name,path=line.split('\t') 61 selected=ROOT/path 62 relative='inputs/'+name+'.tri' 63 (projection/relative).write_bytes(selected.read_bytes()) 64 rows.append(name+'\t'+relative) 65 (projection/'tricu.workspace').write_text('\n'.join(rows)+'\n') 66 source=projection/'probe.tri'; source.write_text(PROBE) 67 runner=Runner(args.tricu,root) 68 bundle=root/'probe.arboricx' 69 runner.run('compile',['compile','--file',source,'--store',runner.store,'--output',bundle, 70 '--compile-timeout','300'],seconds=330) 71 pin=runner.imported(bundle,'import') 72 checks=[] 73 candidate=None 74 zig_records=[] 75 def invoke(label,name,values,expected,io=True): 76 argv=['--io'] if io else [] 77 zig_args=[] 78 for i,value in enumerate(values): 79 path=root/f'{label}-{i}.tree'; path.write_bytes(value) 80 argv+=['--arg-tree-file',path] 81 zig_args.append('tree-file:'+str(path)) 82 if args.zig_probe and io: 83 command=[str(args.zig_probe.resolve()),str(bundle),name,*zig_args] 84 out,peak=run_guarded(command,memory_mib=512,seconds=180) 85 (root/(label+'-zig.log')).write_text(out) 86 assert bytes(map(int,out.strip()))==expected,label 87 zig_records.append(dict(label=label,peak_kib=peak,command=command)) 88 else: 89 assert runner.invoke(label,pin,name,argv)==expected,label 90 checks.append(label) 91 def clean(path): 92 assert not list(path.rglob('.tricu-*')),path 93 for i,payload in enumerate([b'',bytes(range(256)),b'a'*4096+b'last\0\xff']): 94 store=root/f'raw-{i}' 95 expected=hashlib.sha256(b'test.raw\0'+payload).hexdigest() 96 path=store/'objects'/expected[:3]/expected 97 for status in ['new','existing']: 98 invoke(f'raw-{i}-{status}','putRaw',[text(str(store)),text('test.raw'),byte_tree(payload)],result(True,text(expected))) 99 assert path.read_bytes()==payload 100 clean(store) 101 invoke(f'raw-get-{i}','getRaw',[text(str(store)),text('test.raw'),text(expected)],result(True,byte_tree(payload))) 102 path.write_bytes(payload+b'excess') 103 invoke(f'raw-corrupt-{i}','putRaw',[text(str(store)),text('test.raw'),byte_tree(payload)], 104 problem('invalid-data','storage/verify','different bytes','unknown')) 105 assert path.read_bytes()==payload+b'excess'; clean(store) 106 invoke(f'raw-corrupt-get-{i}','getRaw',[text(str(store)),text('test.raw'),text(expected)], 107 result(False,text('CAS digest mismatch'))) 108 for i,tree in enumerate([b'\0',number(42)]): 109 store=root/f'tree-{i}' 110 payload=indexed_bytes(tree); expected=digest(INDEXED,payload) 111 path=store/'objects'/expected[:3]/expected 112 for status in ['new','existing']: 113 invoke(f'tree-{i}-{status}','putTree',[text(str(store)),tree],result(True,text(expected))) 114 assert path.read_bytes()==payload; clean(store) 115 invoke(f'tree-get-{i}','getTree',[text(str(store)),text(expected)],result(True,tree)) 116 path.write_bytes(payload+b'excess') 117 invoke(f'tree-excess-{i}','putTree',[text(str(store)),tree], 118 problem('invalid-data','storage/verify','trailing bytes','unknown')) 119 assert path.read_bytes()==payload+b'excess'; clean(store) 120 invoke(f'tree-corrupt-get-{i}','getTree',[text(str(store)),text(expected)], 121 problem('integrity-failed','indexed-tree/read','CAS digest mismatch','unknown')) 122 shared=dag(200,True); expected=digest(INDEXED,shared) 123 store=root/'shared'; shard=store/'objects'/expected[:3]; shard.mkdir(parents=True) 124 (shard/expected).write_bytes(shared) 125 target=root/'shared-copy' 126 invoke('shared-copy','copyTree',[text(str(store)),text(str(target)),text(expected)],result(True,text(expected))) 127 assert (target/'objects'/expected[:3]/expected).read_bytes()==shared; clean(target) 128 alias=root/'alias' 129 for i,payload in enumerate([b'old',b'new\0\xff',b'']): 130 invoke(f'replace-{i}','replace',[text(str(alias)),byte_tree(payload)],result(True,b'\1\0')) 131 assert alias.read_bytes()==payload 132 for i,path in enumerate(['name','/name','/','a/b','a//b','../name','λ/name','name/','/name///','//name','///']): 133 expected_parent=['.','/','/','a','a/','..','λ','.','/','//','///'][i] 134 invoke(f'parent-{i}','parent',[text(path)],text(expected_parent),False) 135 data=root/'bytes'; data.write_bytes(b'ab\0\xff') 136 invoke('read-exact','read',[text(str(data)),number(4)],result(True,byte_tree(data.read_bytes()))) 137 invoke('read-limit','read',[text(str(data)),number(3)],problem('invalid-bound','storage/read','file exceeds byte limit','unknown')) 138 for i,payload in enumerate([b'',b'a'*4096+b'last']): 139 invoke(f'hash-{i}','hash',[number(len(payload)),byte_tree(payload)],result(True,text(hashlib.sha256(payload).hexdigest()))) 140 # Exact chunk boundaries, short final chunks, and the final budget/EOF probe. 141 for size in [0,1,4095,4096,4097,8191,8192,8193]: 142 payload=bytes(i % 256 for i in range(size)) 143 encoded=byte_tree(payload) 144 data.write_bytes(payload) 145 expected_hash=result(True,text(hashlib.sha256(payload).hexdigest())) 146 for limit in sorted({size, max(size,4096), 67108864}): 147 invoke(f'hash-boundary-{size}-{limit}','hash',[number(limit),encoded],expected_hash) 148 invoke(f'read-boundary-{size}','read',[text(str(data)),number(size)],result(True,encoded)) 149 if size: 150 invoke(f'hash-short-budget-{size}','hash',[number(size-1),encoded], 151 problem('invalid-bound','storage/hash','bytes exceed limit','none')) 152 invoke(f'read-short-budget-{size}','read',[text(str(data)),number(size-1)], 153 problem('invalid-bound','storage/read','file exceeds byte limit','unknown')) 154 # Host octet checks and source proper-list checks remain pre-publication, 155 # including faults after one or two complete chunks were already hashed. 156 for size in [0,4095,4096,4097,8192]: 157 prefix=byte_tree(b'a'*size)[:-1] 158 for kind,tail,expected in [ 159 ('octet',tree_list([number(256)]),problem('invalid-argument','digest/update','','none')), 160 ('stem',b'\1\0',problem('invalid-argument','storage/hash','improper byte list','none'))]: 161 malformed=prefix+tail 162 invoke(f'hash-invalid-{kind}-{size}','hash',[number(67108864),malformed],expected) 163 for mode in [0,1]: 164 destination=root/f'invalid-{kind}-{size}-{mode}' 165 # A failed replacement must preserve an already existing alias. 166 if mode: destination.write_bytes(b'prior alias') 167 invoke(f'publish-invalid-{kind}-{size}-{mode}','publish', 168 [text(str(root)),text(str(destination)),number(mode),number(67108864),malformed],expected) 169 assert destination.read_bytes()==b'prior alias' if mode else not destination.exists() 170 clean(root) 171 for label,limit in [('stem',b'\1\0'),('too-large',number(9223372036854775808)), 172 ('noncanonical',b'\2\0\0')]: 173 expected=problem('invalid-bound','storage/limit','invalid byte limit','none') 174 invoke('hash-invalid-limit-'+label,'hash',[limit,byte_tree(b'')],expected) 175 invoke('read-invalid-limit-'+label,'read',[text(str(data)),limit],expected) 176 # Existing immutable winners must match every byte and EOF, independently of 177 # reader chunking. Retain corrupt objects for diagnosis rather than repair. 178 payload=bytes(i % 256 for i in range(8193)) 179 destination=root/'winner-boundaries' 180 for label,winner in [('equal',payload),('short',payload[:-1]),('long',payload+b'x'), 181 *[(f'changed-{at}',payload[:at]+bytes([payload[at]^255])+payload[at+1:]) 182 for at in [0,4095,4096,8192]]]: 183 destination.write_bytes(winner) 184 expected=result(True,b'\0') if label=='equal' else problem('invalid-data','storage/verify','different bytes','unknown') 185 invoke('winner-'+label,'publish',[text(str(root)),text(str(destination)),number(0),number(len(payload)),byte_tree(payload)],expected) 186 assert destination.read_bytes()==winner 187 clean(root) 188 for label,path,wanted in [('file',data,True),('directory',root,False),('missing',root/'absent',False)]: 189 invoke('exists-'+label,'exists',[text(str(path))],result(True,b'\1\0' if wanted else b'\0')) 190 invoke('exists-invalid','exists',[text('bad\0path')],problem('invalid-argument','path/stat','','none')) 191 for mode in [0,1]: 192 destination=root/f'invalid-octet-{mode}' 193 invoke(f'invalid-octet-{mode}','publish',[text(str(root)),text(str(destination)),number(mode),number(4096),tree_list([number(256)])], 194 problem('invalid-argument','digest/update','','none')) 195 assert not destination.exists() 196 # Legitimate concurrent mutable aliases are last-writer-wins. Neither caller 197 # should spuriously verify another writer's replacement against its own bytes. 198 alias_arg=root/'race-path.tree'; alias_arg.write_bytes(text(str(alias))) 199 jobs=[]; payloads=[b'A'*4096+b'first',b'B'*4096+b'second'] 200 for i,payload in enumerate(payloads): 201 argument=root/f'race-{i}.tree'; argument.write_bytes(byte_tree(payload)) 202 command=[str(runner.exe),'arboricx','run','--module',pin,'--name','replace','--store',str(runner.store), 203 '--io','--format','ternary','--arg-tree-file',str(alias_arg),'--arg-tree-file',str(argument), 204 '+RTS','-N1','-M512m','-RTS'] 205 if args.zig_probe: 206 command=[str(args.zig_probe.resolve()),str(bundle),'replace','tree-file:'+str(alias_arg),'tree-file:'+str(argument)] 207 jobs.append(subprocess.Popen(command,stdout=subprocess.PIPE,stderr=subprocess.PIPE)) 208 try: 209 for process in jobs: 210 out,err=process.communicate(timeout=60) 211 assert process.returncode==0,(out,err) 212 assert out.strip()==''.join(map(str,result(True,b'\1\0'))).encode(),out 213 finally: 214 for process in jobs: 215 if process.poll() is None: process.kill() 216 process.wait() 217 assert alias.read_bytes() in payloads 218 checks.append('concurrent-mutable-alias-replacement') 219 missing=root/'must-not-exist' 220 invoke('preflight-limit','publish',[text(str(root)),text(str(missing)),number(0),number(1),byte_tree(b'ab')], 221 problem('invalid-bound','storage/hash','bytes exceed limit','none')) 222 assert not missing.exists(); clean(root) 223 if args.compiler_smoke: 224 compiler_source=projection/'candidate.tri' 225 compiler_source.write_text('!import source_build as Build\n!import compiler_io as IO\n!export sourceBuildGraph\n!export main\nsourceBuildGraph = Build.sourceBuildGraphStable\nmain workspace root identity name = IO.io (Build.sourceBuildGraphStable workspace root identity name)\n') 226 compiler_bundle=root/'candidate.arboricx' 227 runner.run('compile-candidate',['compile','--file',compiler_source,'--store',runner.store, 228 '--output',compiler_bundle,'--compile-timeout','600'],seconds=660) 229 compiler=runner.imported(compiler_bundle,'import-candidate') 230 candidate=dict(manifest=compiler,io_version=2,bundle_sha256=hashlib.sha256(compiler_bundle.read_bytes()).hexdigest()) 231 app=root/'app'; app.mkdir() 232 (app/'dependency.tri').write_text('!export pick\n-- λ 😀 combining: é\npick [x | _] = x\npick _ = 0\n') 233 (app/'main.tri').write_text('!import dependency as D\n!export main\n!export accent\nmain = D.pick [42 7]\naccent = "é"\n') 234 application=root/'application' 235 compiled=root/'application.arboricx' 236 for label in ['cold','warm']: 237 runner.run('candidate-'+label,['compile','--file',app/'main.tri','--store',application, 238 '--output',compiled,'--toolchain',compiler,'--tool-store',runner.store, 239 '--toolchain-io-version','2','--compile-timeout','300'],seconds=330) 240 program=runner.imported(compiled,'import-'+label) 241 assert runner.invoke('run-'+label,program,'main',[])==number(42) 242 assert runner.invoke('accent-'+label,program,'accent',[])==tree_list([number(101),number(769)]) 243 checks.append('actual-v2-compiler-'+label) 244 retained_bundle=root/'retained-application.arboricx' 245 runner.run('retained-compile',['compile','--file',app/'main.tri','--store',root/'retained-application', 246 '--output',retained_bundle,'--compile-timeout','300'],seconds=330) 247 assert compiled.read_bytes()==retained_bundle.read_bytes() 248 checks.append('complete-application-bundle-identity') 249 # Exactly received UTF-8 provenance, not re-encoded/normalized text. 250 # Receipts bind these digests even when source bytes are not retained as 251 # separate objects by the current source build policy. 252 all_objects=[p.read_bytes() for p in (application/'objects').glob('*/*') if p.is_file()] 253 for file in [app/'main.tri',app/'dependency.tri']: 254 expected=hashlib.sha256(b'arboricx.source.v1\0'+file.read_bytes()).hexdigest().encode() 255 assert any(expected in payload for payload in all_objects),file 256 clean(application) 257 if args.zig_probe: 258 from source_order import field, names 259 zig_application=root/'zig-application' 260 zig_application.mkdir() 261 for label in ['cold','warm']: 262 command=[str(args.zig_probe.resolve()),str(compiler_bundle),'main', 263 'text:'+str(app),'text:'+str(zig_application),'text:'+compiler,'text:main'] 264 output,peak=run_guarded(command,memory_mib=1024,seconds=600) 265 (root/('zig-compiler-'+label+'.log')).write_text(output) 266 answer=bytes(map(int,output.strip())) 267 assert field(answer,0)==b'\1\0',('zig-compiler',label,answer) 268 pair_value=field(answer,1,0) 269 manifest,receipt=names(tree_list([field(pair_value,0),field(pair_value,1)])) 270 receipt_path=zig_application/'objects'/receipt[:3]/receipt 271 assert hashlib.sha256(b'arboricx.build-receipt.v1\0'+receipt_path.read_bytes()).hexdigest()==receipt 272 # Load and authenticate the native-produced store through the 273 # independent Haskell host; execute both actual module exports. 274 old_store=runner.store 275 runner.store=zig_application 276 try: 277 assert runner.invoke('zig-run-'+label,manifest,'main',[])==number(42) 278 assert runner.invoke('zig-accent-'+label,manifest,'accent',[])==tree_list([number(101),number(769)]) 279 finally: 280 runner.store=old_store 281 zig_records.append(dict(label='compiler-'+label,peak_kib=peak,manifest=manifest,receipt=receipt,command=command)) 282 checks.append('private-zig-compiler-'+label) 283 zig_bundle=root/'zig-application.arboricx' 284 runner.run('zig-export',['export','--module',manifest,'--store',zig_application,'--output',zig_bundle]) 285 assert zig_bundle.read_bytes()==compiled.read_bytes() 286 checks.append('private-zig-complete-bundle-identity') 287 clean(zig_application) 288 original=(app/'main.tri').read_bytes() 289 accepted_output=compiled.read_bytes() 290 for label,bad,diagnostic in [('bom',b'\xef\xbb\xbf'+original,'initial source BOM is not permitted'), 291 ('utf8',b'\xff','utf8/decode'), 292 ('crlf',original.replace(b'\n',b'\r\n'),'Portable compilation failed: [40, "invalid import reference or qualifier",')]: 293 (app/'main.tri').write_bytes(bad) 294 command=[str(runner.exe),'arboricx','compile','--file',str(app/'main.tri'),'--store',str(application), 295 '--output',str(compiled),'--toolchain',compiler,'--tool-store',str(runner.store), 296 '--toolchain-io-version','2','--compile-timeout','60','+RTS','-N1','-M512m','-RTS'] 297 rejected=subprocess.run(command,capture_output=True,text=True,timeout=90) 298 assert rejected.returncode!=0,(label,rejected.stdout) 299 assert diagnostic in rejected.stdout+rejected.stderr,(label,rejected.stderr) 300 assert compiled.read_bytes()==accepted_output 301 (root/f'rejected-{label}.log').write_text(rejected.stdout+rejected.stderr) 302 if label=='crlf': 303 # Preserve the existing language rejection, rather than hide it 304 # by normalizing line endings in the new transport. 305 reference=[str(runner.exe),'arboricx','compile','--file',str(app/'main.tri'), 306 '--store',str(root/'crlf-reference'),'--output',str(root/'crlf.arboricx'), 307 '+RTS','-N1','-M512m','-RTS'] 308 legacy=subprocess.run(reference,capture_output=True,text=True,timeout=90) 309 assert legacy.returncode!=0 310 assert legacy.stdout+legacy.stderr==rejected.stdout+rejected.stderr 311 if args.zig_probe: 312 # Compare ordinary failure payloads against the same compiled 313 # worker on Haskell, not host-formatted stderr or a fallback. 314 raw_args=[] 315 for i,value in enumerate([str(app),str(zig_application),compiler,'main']): 316 arg=root/f'negative-{label}-{i}.tree' 317 arg.write_bytes(text(value)) 318 raw_args+=['--arg-tree-file',arg] 319 expected_failure=runner.invoke('worker-reject-'+label,compiler,'main',['--io',*raw_args]) 320 assert field(expected_failure,0)==b'\0' 321 command=[str(args.zig_probe.resolve()),str(compiler_bundle),'main', 322 'text:'+str(app),'text:'+str(zig_application),'text:'+compiler,'text:main'] 323 output,peak=run_guarded(command,memory_mib=1024,seconds=180) 324 (root/('zig-rejected-'+label+'.log')).write_text(output) 325 assert bytes(map(int,output.strip()))==expected_failure,label 326 assert zig_bundle.read_bytes()==accepted_output 327 clean(zig_application) 328 zig_records.append(dict(label='compiler-rejected-'+label,peak_kib=peak,command=command)) 329 checks.append('private-zig-compiler-rejects-'+label) 330 checks.append('actual-v2-compiler-rejects-'+label) 331 (app/'main.tri').write_bytes(original) 332 clean(application) 333 files=['lib/source/object.tri','lib/source/io-v2.tri','lib/io-v2/storage.tri','lib/io-v2.tri','lib/io-v2/text.tri','test/io_v2_compiler_store.py','bootstrap/build-stable.arboricx'] 334 if args.zig_probe: files += ['ext/zig/src/io_v2_file_host.zig','ext/zig/src/indexed.zig','ext/zig/src/io_v2_snapshot.zig'] 335 (root/'report.json').write_text(json.dumps(dict(checks=checks,records=runner.records,candidate=candidate,zig_records=zig_records, 336 zig_probe_sha256=hashlib.sha256(args.zig_probe.read_bytes()).hexdigest() if args.zig_probe else None, 337 projected_inputs={p.relative_to(projection).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in projection.rglob('*.tri')}, 338 source_hashes={p:hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in files}),indent=2)+'\n') 339 print(f'{len(checks)} actual compiler-store and storage controls passed') 340 341 if __name__=='__main__': main()