io_compiler_store.py (17535B)
1 #!/usr/bin/env python3 2 """Run actual source_object callers with an isolated v2 IO source projection. 3 No compiler generations, frozen-input edits or default workspace replacement. 4 """ 5 import argparse 6 import hashlib 7 import json 8 import subprocess 9 from pathlib import Path 10 from source_order import Runner 11 from process_rss import run_guarded 12 from stable_bundle import byte_tree 13 from native_indexed_store import indexed_bytes, digest 14 from executable_object import INDEXED 15 from io_indexed_write import dag 16 from io_objects import text, problem 17 from http_headers import number, result, tree_list 18 19 ROOT = Path(__file__).resolve().parent.parent 20 PROBE = '''!import source_object as O 21 !import compiler_io as IO 22 !import io_storage as S 23 !export putRaw 24 !export getRaw 25 !export putTree 26 !export getTree 27 !export copyTree 28 !export replace 29 !export read 30 !export hash 31 !export publish 32 !export parent 33 !export exists 34 putRaw root domain bytes = IO.io (O.sourceObjectStore root domain bytes) 35 getRaw root domain digest = IO.io (O.sourceObjectGet root domain digest) 36 putTree root tree = IO.io (O.sourceObjectPutTree root tree) 37 getTree root digest = IO.io (O.sourceObjectGetTree root digest) 38 copyTree source target digest = IO.io <| IO.onOk_ (O.sourceObjectGetTree source digest) 39 (tree : O.sourceObjectPutTree target tree) 40 replace path bytes = IO.io (IO.atomicWriteBytes path bytes) 41 read path limit = IO.io (S.readBytes path limit) 42 hash limit bytes = IO.io (S.hashBytes limit bytes) 43 publish directory path mode limit bytes = IO.io (S.publishBytes directory path mode limit bytes) 44 parent path = S.parentPath path 45 exists path = IO.io (IO.fileExists path) 46 ''' 47 48 def main(): 49 parser = argparse.ArgumentParser(description=__doc__) 50 parser.add_argument('tricu',type=Path) 51 parser.add_argument('--directory',type=Path,required=True) 52 parser.add_argument('--compiler-smoke',action='store_true',help='also build a candidate stable sourceBuildGraph and compile a real dependency graph; not a generation pair') 53 args=parser.parse_args() 54 root=args.directory.resolve(); root.mkdir(parents=True,exist_ok=False) 55 projection=root/'projection'; projection.mkdir() 56 (projection/'inputs').mkdir() 57 rows=['tricu.workspace.v1'] 58 for line in (ROOT/'tricu.workspace').read_text().splitlines()[1:]: 59 name,path=line.split('\t') 60 selected=ROOT/path 61 relative='inputs/'+name+'.tri' 62 (projection/relative).write_bytes(selected.read_bytes()) 63 rows.append(name+'\t'+relative) 64 (projection/'tricu.workspace').write_text('\n'.join(rows)+'\n') 65 source=projection/'probe.tri'; source.write_text(PROBE) 66 runner=Runner(args.tricu,root) 67 bundle=root/'probe.arboricx' 68 runner.run('compile',['compile','--file',source,'--store',runner.store,'--output',bundle, 69 '--compile-timeout','300'],seconds=330) 70 pin=runner.imported(bundle,'import') 71 checks=[] 72 candidate=None 73 def invoke(label,name,values,expected,io=True): 74 argv=['--io'] if io else [] 75 for i,value in enumerate(values): 76 path=root/f'{label}-{i}.tree'; path.write_bytes(value) 77 argv+=['--arg-tree-file',path] 78 assert runner.invoke(label,pin,name,argv)==expected,label 79 checks.append(label) 80 def clean(path): 81 assert not list(path.rglob('.tricu-*')),path 82 for i,payload in enumerate([b'',bytes(range(256)),b'a'*4096+b'last\0\xff']): 83 store=root/f'raw-{i}' 84 expected=hashlib.sha256(b'test.raw\0'+payload).hexdigest() 85 path=store/'objects'/expected[:3]/expected 86 for status in ['new','existing']: 87 invoke(f'raw-{i}-{status}','putRaw',[text(str(store)),text('test.raw'),byte_tree(payload)],result(True,text(expected))) 88 assert path.read_bytes()==payload 89 clean(store) 90 invoke(f'raw-get-{i}','getRaw',[text(str(store)),text('test.raw'),text(expected)],result(True,byte_tree(payload))) 91 path.write_bytes(payload+b'excess') 92 invoke(f'raw-corrupt-{i}','putRaw',[text(str(store)),text('test.raw'),byte_tree(payload)], 93 problem('invalid-data','storage/verify','different bytes','unknown')) 94 assert path.read_bytes()==payload+b'excess'; clean(store) 95 invoke(f'raw-corrupt-get-{i}','getRaw',[text(str(store)),text('test.raw'),text(expected)], 96 result(False,text('CAS digest mismatch'))) 97 for i,tree in enumerate([b'\0',number(42)]): 98 store=root/f'tree-{i}' 99 payload=indexed_bytes(tree); expected=digest(INDEXED,payload) 100 path=store/'objects'/expected[:3]/expected 101 for status in ['new','existing']: 102 invoke(f'tree-{i}-{status}','putTree',[text(str(store)),tree],result(True,text(expected))) 103 assert path.read_bytes()==payload; clean(store) 104 invoke(f'tree-get-{i}','getTree',[text(str(store)),text(expected)],result(True,tree)) 105 path.write_bytes(payload+b'excess') 106 invoke(f'tree-excess-{i}','putTree',[text(str(store)),tree], 107 problem('invalid-data','storage/verify','trailing bytes','unknown')) 108 assert path.read_bytes()==payload+b'excess'; clean(store) 109 invoke(f'tree-corrupt-get-{i}','getTree',[text(str(store)),text(expected)], 110 problem('integrity-failed','indexed-tree/read','CAS digest mismatch','unknown')) 111 shared=dag(200,True); expected=digest(INDEXED,shared) 112 store=root/'shared'; shard=store/'objects'/expected[:3]; shard.mkdir(parents=True) 113 (shard/expected).write_bytes(shared) 114 target=root/'shared-copy' 115 invoke('shared-copy','copyTree',[text(str(store)),text(str(target)),text(expected)],result(True,text(expected))) 116 assert (target/'objects'/expected[:3]/expected).read_bytes()==shared; clean(target) 117 alias=root/'alias' 118 for i,payload in enumerate([b'old',b'new\0\xff',b'']): 119 invoke(f'replace-{i}','replace',[text(str(alias)),byte_tree(payload)],result(True,b'\1\0')) 120 assert alias.read_bytes()==payload 121 for i,path in enumerate(['name','/name','/','a/b','a//b','../name','λ/name','name/','/name///','//name','///']): 122 expected_parent=['.','/','/','a','a/','..','λ','.','/','//','///'][i] 123 invoke(f'parent-{i}','parent',[text(path)],text(expected_parent),False) 124 data=root/'bytes'; data.write_bytes(b'ab\0\xff') 125 invoke('read-exact','read',[text(str(data)),number(4)],result(True,byte_tree(data.read_bytes()))) 126 invoke('read-limit','read',[text(str(data)),number(3)],problem('invalid-bound','storage/read','file exceeds byte limit','unknown')) 127 for i,payload in enumerate([b'',b'a'*4096+b'last']): 128 invoke(f'hash-{i}','hash',[number(len(payload)),byte_tree(payload)],result(True,text(hashlib.sha256(payload).hexdigest()))) 129 # Exact chunk boundaries, short final chunks, and the final budget/EOF probe. 130 for size in [0,1,4095,4096,4097,8191,8192,8193]: 131 payload=bytes(i % 256 for i in range(size)) 132 encoded=byte_tree(payload) 133 data.write_bytes(payload) 134 expected_hash=result(True,text(hashlib.sha256(payload).hexdigest())) 135 for limit in sorted({size, max(size,4096), 67108864}): 136 invoke(f'hash-boundary-{size}-{limit}','hash',[number(limit),encoded],expected_hash) 137 invoke(f'read-boundary-{size}','read',[text(str(data)),number(size)],result(True,encoded)) 138 if size: 139 invoke(f'hash-short-budget-{size}','hash',[number(size-1),encoded], 140 problem('invalid-bound','storage/hash','bytes exceed limit','none')) 141 invoke(f'read-short-budget-{size}','read',[text(str(data)),number(size-1)], 142 problem('invalid-bound','storage/read','file exceeds byte limit','unknown')) 143 # Host octet checks and source proper-list checks remain pre-publication, 144 # including faults after one or two complete chunks were already hashed. 145 for size in [0,4095,4096,4097,8192]: 146 prefix=byte_tree(b'a'*size)[:-1] 147 for kind,tail,expected in [ 148 ('octet',tree_list([number(256)]),problem('invalid-argument','digest/update','','none')), 149 ('stem',b'\1\0',problem('invalid-argument','storage/hash','improper byte list','none'))]: 150 malformed=prefix+tail 151 invoke(f'hash-invalid-{kind}-{size}','hash',[number(67108864),malformed],expected) 152 for mode in [0,1]: 153 destination=root/f'invalid-{kind}-{size}-{mode}' 154 # A failed replacement must preserve an already existing alias. 155 if mode: destination.write_bytes(b'prior alias') 156 invoke(f'publish-invalid-{kind}-{size}-{mode}','publish', 157 [text(str(root)),text(str(destination)),number(mode),number(67108864),malformed],expected) 158 assert destination.read_bytes()==b'prior alias' if mode else not destination.exists() 159 clean(root) 160 for label,limit in [('stem',b'\1\0'),('too-large',number(9223372036854775808)), 161 ('noncanonical',b'\2\0\0')]: 162 expected=problem('invalid-bound','storage/limit','invalid byte limit','none') 163 invoke('hash-invalid-limit-'+label,'hash',[limit,byte_tree(b'')],expected) 164 invoke('read-invalid-limit-'+label,'read',[text(str(data)),limit],expected) 165 # Existing immutable winners must match every byte and EOF, independently of 166 # reader chunking. Retain corrupt objects for diagnosis rather than repair. 167 payload=bytes(i % 256 for i in range(8193)) 168 destination=root/'winner-boundaries' 169 for label,winner in [('equal',payload),('short',payload[:-1]),('long',payload+b'x'), 170 *[(f'changed-{at}',payload[:at]+bytes([payload[at]^255])+payload[at+1:]) 171 for at in [0,4095,4096,8192]]]: 172 destination.write_bytes(winner) 173 expected=result(True,b'\0') if label=='equal' else problem('invalid-data','storage/verify','different bytes','unknown') 174 invoke('winner-'+label,'publish',[text(str(root)),text(str(destination)),number(0),number(len(payload)),byte_tree(payload)],expected) 175 assert destination.read_bytes()==winner 176 clean(root) 177 for label,path,wanted in [('file',data,True),('directory',root,False),('missing',root/'absent',False)]: 178 invoke('exists-'+label,'exists',[text(str(path))],result(True,b'\1\0' if wanted else b'\0')) 179 invoke('exists-invalid','exists',[text('bad\0path')],problem('invalid-argument','path/stat','','none')) 180 for mode in [0,1]: 181 destination=root/f'invalid-octet-{mode}' 182 invoke(f'invalid-octet-{mode}','publish',[text(str(root)),text(str(destination)),number(mode),number(4096),tree_list([number(256)])], 183 problem('invalid-argument','digest/update','','none')) 184 assert not destination.exists() 185 # Legitimate concurrent mutable aliases are last-writer-wins. Neither caller 186 # should spuriously verify another writer's replacement against its own bytes. 187 alias_arg=root/'race-path.tree'; alias_arg.write_bytes(text(str(alias))) 188 jobs=[]; payloads=[b'A'*4096+b'first',b'B'*4096+b'second'] 189 for i,payload in enumerate(payloads): 190 argument=root/f'race-{i}.tree'; argument.write_bytes(byte_tree(payload)) 191 command=[str(runner.exe),'arboricx','run','--module',pin,'--name','replace','--store',str(runner.store), 192 '--io','--format','ternary','--arg-tree-file',str(alias_arg),'--arg-tree-file',str(argument), 193 '+RTS','-N1','-M512m','-RTS'] 194 jobs.append(subprocess.Popen(command,stdout=subprocess.PIPE,stderr=subprocess.PIPE)) 195 try: 196 for process in jobs: 197 out,err=process.communicate(timeout=60) 198 assert process.returncode==0,(out,err) 199 assert out.strip()==''.join(map(str,result(True,b'\1\0'))).encode(),out 200 finally: 201 for process in jobs: 202 if process.poll() is None: process.kill() 203 process.wait() 204 assert alias.read_bytes() in payloads 205 checks.append('concurrent-mutable-alias-replacement') 206 missing=root/'must-not-exist' 207 invoke('preflight-limit','publish',[text(str(root)),text(str(missing)),number(0),number(1),byte_tree(b'ab')], 208 problem('invalid-bound','storage/hash','bytes exceed limit','none')) 209 assert not missing.exists(); clean(root) 210 if args.compiler_smoke: 211 compiler_source=projection/'candidate.tri' 212 compiler_source.write_text('!import source_build as Build\n!import compiler_io as IO\n!export sourceBuildGraph\n!export main\nsourceBuildGraph = Build.sourceBuildGraphStable\nmain workspace root identity name = IO.io (Build.sourceBuildGraphStable workspace root identity name)\n') 213 compiler_bundle=root/'candidate.arboricx' 214 runner.run('compile-candidate',['compile','--file',compiler_source,'--store',runner.store, 215 '--output',compiler_bundle,'--compile-timeout','600'],seconds=660) 216 compiler=runner.imported(compiler_bundle,'import-candidate') 217 candidate=dict(manifest=compiler,io_version=2,bundle_sha256=hashlib.sha256(compiler_bundle.read_bytes()).hexdigest()) 218 app=root/'app'; app.mkdir() 219 (app/'dependency.tri').write_text('!export pick\n-- λ 😀 combining: é\npick [x | _] = x\npick _ = 0\n') 220 (app/'main.tri').write_text('!import dependency as D\n!export main\n!export accent\nmain = D.pick [42 7]\naccent = "é"\n') 221 application=root/'application' 222 compiled=root/'application.arboricx' 223 for label in ['cold','warm']: 224 runner.run('candidate-'+label,['compile','--file',app/'main.tri','--store',application, 225 '--output',compiled,'--toolchain',compiler,'--tool-store',runner.store, 226 '--toolchain-io-version','2','--compile-timeout','300'],seconds=330) 227 program=runner.imported(compiled,'import-'+label) 228 assert runner.invoke('run-'+label,program,'main',[])==number(42) 229 assert runner.invoke('accent-'+label,program,'accent',[])==tree_list([number(101),number(769)]) 230 checks.append('actual-v2-compiler-'+label) 231 retained_bundle=root/'retained-application.arboricx' 232 runner.run('retained-compile',['compile','--file',app/'main.tri','--store',root/'retained-application', 233 '--output',retained_bundle,'--compile-timeout','300'],seconds=330) 234 assert compiled.read_bytes()==retained_bundle.read_bytes() 235 checks.append('complete-application-bundle-identity') 236 # Exactly received UTF-8 provenance, not re-encoded/normalized text. 237 # Receipts bind these digests even when source bytes are not retained as 238 # separate objects by the current source build policy. 239 all_objects=[p.read_bytes() for p in (application/'objects').glob('*/*') if p.is_file()] 240 for file in [app/'main.tri',app/'dependency.tri']: 241 expected=hashlib.sha256(b'arboricx.source.v1\0'+file.read_bytes()).hexdigest().encode() 242 assert any(expected in payload for payload in all_objects),file 243 clean(application) 244 original=(app/'main.tri').read_bytes() 245 accepted_output=compiled.read_bytes() 246 for label,bad,diagnostic in [('bom',b'\xef\xbb\xbf'+original,'initial source BOM is not permitted'), 247 ('utf8',b'\xff','utf8/decode'), 248 ('crlf',original.replace(b'\n',b'\r\n'),'Portable compilation failed: [40, "invalid import reference or qualifier",')]: 249 (app/'main.tri').write_bytes(bad) 250 command=[str(runner.exe),'arboricx','compile','--file',str(app/'main.tri'),'--store',str(application), 251 '--output',str(compiled),'--toolchain',compiler,'--tool-store',str(runner.store), 252 '--toolchain-io-version','2','--compile-timeout','60','+RTS','-N1','-M512m','-RTS'] 253 rejected=subprocess.run(command,capture_output=True,text=True,timeout=90) 254 assert rejected.returncode!=0,(label,rejected.stdout) 255 assert diagnostic in rejected.stdout+rejected.stderr,(label,rejected.stderr) 256 assert compiled.read_bytes()==accepted_output 257 (root/f'rejected-{label}.log').write_text(rejected.stdout+rejected.stderr) 258 if label=='crlf': 259 # Preserve the existing language rejection, rather than hide it 260 # by normalizing line endings in the new transport. 261 reference=[str(runner.exe),'arboricx','compile','--file',str(app/'main.tri'), 262 '--store',str(root/'crlf-reference'),'--output',str(root/'crlf.arboricx'), 263 '+RTS','-N1','-M512m','-RTS'] 264 legacy=subprocess.run(reference,capture_output=True,text=True,timeout=90) 265 assert legacy.returncode!=0 266 assert legacy.stdout+legacy.stderr==rejected.stdout+rejected.stderr 267 checks.append('actual-v2-compiler-rejects-'+label) 268 (app/'main.tri').write_bytes(original) 269 clean(application) 270 files=['lib/source/object.tri','lib/source/io.tri','lib/io/storage.tri','lib/io.tri','lib/io/text.tri','test/io_compiler_store.py','bootstrap/build-stable.arboricx'] 271 (root/'report.json').write_text(json.dumps(dict(checks=checks,records=runner.records,candidate=candidate, 272 273 projected_inputs={p.relative_to(projection).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in projection.rglob('*.tri')}, 274 source_hashes={p:hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in files}),indent=2)+'\n') 275 print(f'{len(checks)} actual compiler-store and storage controls passed') 276 277 if __name__=='__main__': main()