io_v2_text.py (8444B)
1 #!/usr/bin/env python3 2 """Independent UTF-8 boundaries, exact source provenance and text pipelines.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 from source_order import Runner 8 from http_headers import number, result, pair, tree_list 9 from stable_bundle import byte_tree 10 from process_rss import run_guarded 11 12 ROOT = Path(__file__).resolve().parent.parent 13 14 def text(s): 15 return tree_list([number(ord(c)) for c in s]) 16 17 def problem(category, op, detail, progress='none'): 18 return tree_list([text(category),text(op),text(detail),text(progress)]) 19 20 def main(): 21 parser = argparse.ArgumentParser(description=__doc__) 22 parser.add_argument('tricu', type=Path) 23 parser.add_argument('--directory', type=Path, required=True) 24 parser.add_argument('--zig-probe', type=Path, help='also check the private Zig host against the same independent oracles') 25 args = parser.parse_args() 26 directory = args.directory.resolve() 27 directory.mkdir(parents=True, exist_ok=False) 28 runner = Runner(args.tricu,directory) 29 source = ROOT/'demos/io-v2/text.tri' 30 bundle = directory/'text.arboricx' 31 runner.run('compile',['compile','--file',source,'--store',runner.store,'--output',bundle]) 32 pin = runner.imported(bundle,'import') 33 checks = [] 34 zig_observations = [] 35 def invoke(label,name,values,expected): 36 argv = ['--io'] 37 zig_arguments = [] 38 for i,value in enumerate(values): 39 path = directory/f'{label}-{i}.tree' 40 path.write_bytes(value) 41 argv += ['--arg-tree-file',path] 42 zig_arguments.append('tree-file:'+str(path)) 43 assert runner.invoke(label,pin,name,argv) == expected,label 44 if args.zig_probe: 45 command = [str(args.zig_probe.resolve()),str(bundle),name,*zig_arguments] 46 output,peak = run_guarded(command,memory_mib=256,seconds=180) 47 (directory/(label+'-zig.log')).write_text(output) 48 assert bytes(map(int,output.strip())) == expected,('zig',label) 49 zig_observations.append(dict(label=label,peak_kib=peak,command=command)) 50 checks.append(label) 51 for i,value in enumerate(['','ascii\r\n\0','位馃榾e\u0301茅','\ufeff','\uffff\U0010ffff','馃榾'*1025,'a'*1024+'馃榾'+'b'*1024]): 52 payload = value.encode('utf-8') 53 invoke(f'encode-{i}','encode',[number(len(payload)),text(value)],result(True,byte_tree(payload))) 54 output = directory/f'write-{i}.bin' 55 invoke(f'write-{i}','write',[text(str(output)),text(value)],result(True,b'\0')) 56 assert output.read_bytes() == payload 57 for i,payload in enumerate([b'',b'a\r\n\0', '位馃榾e\u0301茅'.encode(), 58 b'a'*4092+'馃榾'.encode(), b'a'*4091+'馃榾x'.encode(), 59 ('a\ufeffe\u0301\r\n'*100).encode()]): 60 path = directory/f'source-{i}.bin' 61 path.write_bytes(payload) 62 invoke(f'source-{i}','source',[text(str(path)),number(len(payload))], 63 result(True,pair(byte_tree(payload),text(payload.decode('utf-8'))))) 64 assert path.read_bytes() == payload 65 # Confirm accounting at the 4093-byte read boundary and its one-byte EOF 66 # probe, including multiple chunks. These bytes are independently decoded. 67 for size in [4092,4093,4094,8186,8187]: 68 payload=b'a'*size 69 path=directory/f'count-boundary-{size}.bin' 70 path.write_bytes(payload) 71 invoke(f'count-exact-{size}','source',[text(str(path)),number(size)], 72 result(True,pair(byte_tree(payload),text(payload.decode())))) 73 invoke(f'count-short-{size}','source',[text(str(path)),number(size-1)], 74 result(False,tree_list([text('read-source'),number(size), 75 problem('invalid-bound','text/read-source','source exceeds byte limit')]))) 76 payload = 'A馃榾位'.encode() 77 # Every split through the multibyte suffix, with independent expected carry. 78 for i in range(len(payload)+1): 79 prefix = payload[:i] 80 import codecs 81 decoder = codecs.getincrementaldecoder('utf-8')('strict') 82 decoded = decoder.decode(prefix, final=False) 83 carry = decoder.getstate()[0] 84 invoke(f'prefix-{i}','decode',[number(0),byte_tree(prefix)], 85 result(True,tree_list([text(decoded),byte_tree(carry)]))) 86 for i,(payload,cause) in enumerate([ 87 (b'\xef\xbb\xbfabc',problem('invalid-data','text/read-source','initial source BOM is not permitted')), 88 (b'good\xff',problem('invalid-data','utf8/decode','invalid UTF-8 leading byte at byte 4')), 89 (b'\xe2\x82',problem('invalid-data','utf8/decode','incomplete UTF-8 sequence at byte 0')), 90 (b'\xed\xa0\x80',problem('invalid-data','utf8/decode','invalid UTF-8 sequence at byte 0'))]): 91 path = directory/f'invalid-{i}.bin'; path.write_bytes(payload) 92 invoke(f'invalid-source-{i}','source',[text(str(path)),number(len(payload))], 93 result(False,tree_list([text('read-source'),number(len(payload)),cause]))) 94 path = directory/'excess.bin'; path.write_bytes(b'x') 95 invoke('source-limit','source',[text(str(path)),number(0)],result(False,tree_list([ 96 text('read-source'),number(1),problem('invalid-bound','text/read-source','source exceeds byte limit')]))) 97 invoke('encode-limit','encode',[number(1),text('位')],result(False, 98 problem('invalid-bound','text/encode','encoded text exceeds byte limit'))) 99 invoke('surrogate','encode',[number(4096),tree_list([number(0xd800)])],result(False, 100 problem('invalid-argument','utf8/encode',''))) 101 partial = directory/'partial.bin' 102 invalid_text = tree_list([number(97)]*1024+[number(0xd800)]) 103 invoke('write-invalid-tail','write',[text(str(partial)),invalid_text],result(False,tree_list([ 104 text('write-text'),number(1024),problem('invalid-argument','utf8/encode','')]))) 105 assert partial.read_bytes() == b'a'*1024 106 for size in [4096,65536]: 107 payload = (b'ab\r\n\0'+ '位馃榾'.encode())*(size//11) 108 input_path = directory/f'pipeline-{size}.bin'; output = directory/f'pipeline-{size}-out.bin' 109 input_path.write_bytes(payload) 110 invoke(f'pipeline-{size}','roundTrip',[text(str(input_path)),text(str(output)),number(len(payload))],result(True,b'\0')) 111 assert output.read_bytes() == payload 112 if args.zig_probe: 113 # Independent raw-protocol boundaries; excess heads AND tails diverge. 114 # Both backends must reject without demanding them or writing a prefix. 115 boundary = directory/'boundaries.arboricx' 116 runner.run('compile-boundaries',['compile','--file',ROOT/'test/io-v2/zig-utf8.tri','--store',runner.store,'--output',boundary]) 117 original_bundle, original_pin = bundle, pin 118 bundle, pin = boundary, runner.imported(boundary,'import-boundaries') 119 for name,operation in [('excessEncode','utf8/encode'),('excessDecode','utf8/decode'),('excessDigest','digest/update')]: 120 invoke(name,name,[number(0)],result(False,problem('invalid-argument',operation,''))) 121 path = directory/'excess-write' 122 invoke('excessWrite','excessWrite',[text(str(path)),number(0)],result(False,problem('invalid-argument','stream/write',''))) 123 assert path.read_bytes() == b'' 124 bundle, pin = original_bundle, original_pin 125 observations = [r for r in runner.records if r['label'].startswith('pipeline-')] 126 for row in observations: 127 assert row['peak_kib'] <= 256*1024,row 128 assert row['rts_process']['maximum_live_bytes'] <= 128*1024*1024,row 129 files = ['lib/io-v2.tri','lib/io-v2/text.tri','demos/io-v2/text.tri','test/io_v2_text.py', 130 'test/source_order.py','test/rts_metrics.py','src/IOV2/UTF8.hs','src/IOV2/Driver.hs', 131 'src/IOV2/Protocol.hs','bootstrap/build-stable.arboricx'] 132 if args.zig_probe: files += ['test/io-v2/zig-utf8.tri','ext/zig/src/io_v2_utf8.zig','ext/zig/src/io_v2_file_host.zig','ext/zig/src/io_v2_core_probe.zig'] 133 (directory/'report.json').write_text(json.dumps(dict(checks=checks,observations=observations,zig_observations=zig_observations, 134 zig_probe_sha256=hashlib.sha256(args.zig_probe.read_bytes()).hexdigest() if args.zig_probe else None, 135 source_hashes={p:hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in files}),indent=2)+'\n') 136 print(f'{len(checks)} independent text/UTF-8 checks passed'+(' on Haskell and private Zig' if args.zig_probe else '')) 137 138 if __name__ == '__main__': main()