io_indexed_write.py (7682B)
1 #!/usr/bin/env python3 2 """Independent canonical bytes, winner verification and shared-DAG round trips.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 import struct 8 from source_order import Runner 9 from process_rss import run_guarded 10 from native_indexed_store import indexed_bytes, digest 11 from executable_object import INDEXED, indexed_manifest 12 from stable_bundle import assembled 13 from http_headers import number, result, tree_list 14 15 ROOT = Path(__file__).resolve().parent.parent 16 17 def text(s): 18 return tree_list([number(ord(c)) for c in s]) 19 20 def info(payload): 21 return tree_list([number(len(payload)), text(digest(INDEXED, payload))]) 22 23 def problem(category, operation, detail, progress): 24 return result(False, tree_list([text(category), text(operation), text(detail), text(progress)])) 25 26 def dag(depth, fork): 27 records = [struct.pack('>IB', 1, 0)] 28 for i in range(1, depth+1): 29 payload = struct.pack('>BII', 2, i-1, i-1) if fork else struct.pack('>BI', 1, i-1) 30 records.append(struct.pack('>I', len(payload))+payload) 31 return assembled(indexed_manifest(depth), struct.pack('>Q', len(records))+b''.join(records)) 32 33 def main(): 34 parser = argparse.ArgumentParser(description=__doc__) 35 parser.add_argument('tricu', type=Path) 36 parser.add_argument('--directory', type=Path, required=True) 37 args = parser.parse_args() 38 directory = args.directory.resolve() 39 directory.mkdir(parents=True, exist_ok=False) 40 runner = Runner(args.tricu, directory) 41 source = ROOT/'demos/io/indexed-write.tri' 42 bundle = directory/'writer.arboricx' 43 runner.run('compile', ['compile', '--file', source, '--store', runner.store, '--output', bundle]) 44 pin = runner.imported(bundle, 'import') 45 checks = [] 46 def invoke(label, name, values, expected): 47 argv = ['--io'] 48 for i, value in enumerate(values): 49 path = directory/f'{label}-{i}.tree' 50 path.write_bytes(value) 51 argv += ['--arg-tree-file', path] 52 assert runner.invoke(label, pin, name, argv) == expected, label 53 checks.append(label) 54 for i, tree in enumerate([b'\0', b'\1\0', b'\2\1\0\0', number(42)]): 55 payload = indexed_bytes(tree) 56 expected = digest(INDEXED, payload) 57 invoke(f'hash-{i}', 'hash', [number(len(payload)), tree], result(True, info(payload))) 58 path = directory/f'write-{i}.bin' 59 invoke(f'write-{i}', 'write', [text(str(path)), number(len(payload)), text(expected), tree], result(True, info(payload))) 60 assert path.read_bytes() == payload 61 root = directory/f'publish-{i}' 62 root.mkdir() 63 for status in ['published', 'exists']: 64 invoke(f'publish-{i}-{status}', 'publish', [text(str(root)), number(len(payload)), tree], 65 result(True, tree_list([text(status),number(len(payload)),text(expected)]))) 66 assert sorted(p.name for p in root.iterdir()) == [expected] 67 assert (root/expected).read_bytes() == payload 68 (root/expected).write_bytes(b'X'+payload[1:]) 69 invoke(f'corrupt-{i}', 'publish', [text(str(root)), number(len(payload)), tree], 70 problem('integrity-failed','indexed-tree/verify','indexed frame differs from canonical tree','unknown')) 71 assert (root/expected).read_bytes() == b'X'+payload[1:] 72 assert sorted(p.name for p in root.iterdir()) == [expected] 73 (root/expected).write_bytes(payload+b'extra') 74 invoke(f'excess-{i}', 'publish', [text(str(root)), number(len(payload)), tree], 75 problem('invalid-data','indexed-file/verify','trailing bytes','unknown')) 76 assert (root/expected).read_bytes() == payload+b'extra' 77 path = directory/'preserve.bin' 78 path.write_bytes(b'untouched') 79 tree = b'\0' 80 payload = indexed_bytes(tree) 81 for label, limit, expected, answer in [ 82 ('wrong-hash',65536,'0'*64,problem('integrity-failed','indexed-tree/write','CAS digest mismatch','none')), 83 ('limit',0,digest(INDEXED,payload),problem('invalid-bound','indexed-tree/write','encoded tree exceeds limit','none')), 84 ('invalid-hash',65536,'A'*64,problem('invalid-argument','indexed-tree/write','','none'))]: 85 invoke(label, 'write', [text(str(path)),number(limit),text(expected),tree], answer) 86 assert path.read_bytes() == b'untouched' 87 # Raw input uses linear-depth stems, not an expanded exponentially shared 88 # tree. Expectations come from the independent wire assembler. 89 for depth in [1000,10000]: 90 payload = dag(depth,False) 91 tree = b'\1'*depth+b'\0' 92 invoke(f'deep-hash-{depth}','hash',[number(len(payload)),tree],result(True,info(payload))) 93 output = directory/f'deep-{depth}.bin' 94 invoke(f'deep-write-{depth}','write',[text(str(output)),number(len(payload)),text(digest(INDEXED,payload)),tree],result(True,info(payload))) 95 assert output.read_bytes() == payload 96 original_bundle,original_pin = bundle,pin 97 bundle = directory/'transport.arboricx' 98 runner.run('compile-transport',['compile','--file',ROOT/'test/io/indexed-transport.tri','--store',runner.store,'--output',bundle]) 99 pin = runner.imported(bundle,'import-transport') 100 payload = indexed_bytes(b'\0') 101 framed = directory/'framed.bin' 102 framed.write_bytes(payload+b'X') 103 from http_headers import pair 104 invoke('frame-tail','verifyTail',[text(str(framed)),number(len(payload)),b'\0'],result(True,pair(text('data'),tree_list([number(88)])))) 105 framed.write_bytes(payload[:-1]) 106 invoke('frame-truncated','verifyTail',[text(str(framed)),number(len(payload)),b'\0'],problem('integrity-failed','indexed-tree/verify','indexed frame differs from canonical tree','unknown')) 107 invoke('wrong-kind-inert-tree','wrongKind',[number(0)],problem('wrong-kind','indexed-tree/verify','','none')) 108 bundle,pin = original_bundle,original_pin 109 scratch = directory/'spool' 110 scratch.mkdir() 111 for depth, fork in ([(200,True),(1000,False),(10000,False),(100000,False)]): 112 payload = dag(depth,fork) 113 input_path = directory/f'dag-{depth}-{fork}.bin' 114 output = directory/f'dag-{depth}-{fork}-output.bin' 115 input_path.write_bytes(payload) 116 invoke(f'roundtrip-{depth}-{fork}', 'roundTrip', [text(str(input_path)),text(str(output)),text(str(scratch)), 117 number(len(payload)),number(len(payload)),text(digest(INDEXED,payload))], result(True,info(payload))) 118 assert output.read_bytes() == input_path.read_bytes() == payload 119 assert list(scratch.iterdir()) == [] 120 observations = [r for r in runner.records if r['label'].startswith('roundtrip-')] 121 for row in observations: 122 # DAG/table memory is expected to grow; these are regression guards, not 123 # constant-memory claims or a universal safe node-count budget. 124 assert row['peak_kib'] <= 256*1024, row 125 assert row['rts_process']['maximum_live_bytes'] <= 128*1024*1024, row 126 paths = ['test/io/indexed-transport.tri','test/io_indexed_write.py','test/native_indexed_store.py','test/executable_object.py', 127 'test/stable_bundle.py','test/source_order.py','test/rts_metrics.py','lib/io.tri', 128 'demos/io/indexed-write.tri','src/IO/Codec.hs','src/IO/Resources.hs','src/Wire.hs', 129 'bootstrap/build-stable.arboricx'] 130 (directory/'report.json').write_text(json.dumps(dict(checks=checks, observations=observations, 131 scope="haskell-complete", 132 133 source_hashes={p:hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in paths}),indent=2)+'\n') 134 print(f'{len(checks)} independent indexed-writer checks passed'+('')) 135 136 if __name__ == '__main__': 137 main()