io_v2_indexed.py (6668B)
1 #!/usr/bin/env python3 2 """Independent framed/authenticated indexed-read and bounded-transfer controls.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 import struct 8 from source_order import Runner 9 from process_rss import run_guarded 10 from native_indexed_store import indexed_bytes, digest 11 from executable_object import INDEXED, indexed_manifest 12 from http_headers import number, result, tree_list 13 14 ROOT = Path(__file__).resolve().parent.parent 15 16 def text(s): 17 return tree_list([number(ord(c)) for c in s]) 18 19 def problem(category, detail, progress): 20 return result(False, tree_list([text(category), text('indexed-tree/read'), text(detail), text(progress)])) 21 22 def padded(size): 23 # Valid noncritical opaque section grows TRANSPORT, not the decoded DAG. 24 # This is not a large-DAG memory or compiler-capacity benchmark. 25 manifest = indexed_manifest(0) 26 nodes = struct.pack('>QIB', 1, 1, 0) 27 def section(kind, flags, offset, count): 28 return struct.pack('>IHHIQQI', kind, 1, flags, 0, offset, count, 0) 29 offset = 128 + len(manifest) + len(nodes) 30 return (b'ARBORICX' + struct.pack('>HHIQQ', 1, 0, 3, 0, 32) 31 + section(1, 1, 128, len(manifest)) 32 + section(2, 1, 128 + len(manifest), len(nodes)) 33 + section(99, 0, offset, size) + manifest + nodes + b'x' * size) 34 35 def main(): 36 parser = argparse.ArgumentParser(description=__doc__) 37 parser.add_argument('tricu', type=Path) 38 parser.add_argument('--directory', required=True, type=Path) 39 parser.add_argument('--zig-probe', type=Path) 40 args = parser.parse_args() 41 directory = args.directory.resolve() 42 directory.mkdir(parents=True, exist_ok=False) 43 scratch = directory/'spool-λ' 44 scratch.mkdir() 45 runner = Runner(args.tricu, directory) 46 source = ROOT/'demos/io-v2/indexed-read.tri' 47 bundle = directory/'indexed-read.arboricx' 48 runner.run('compile', ['compile', '--file', source, '--store', runner.store, '--output', bundle]) 49 pin = runner.imported(bundle, 'import') 50 checks = [] 51 zig_observations = [] 52 def check(label, payload, size, limit, expected_digest, expected_reply, position): 53 path = directory/(label+'.bin') 54 path.write_bytes(payload) 55 values = [text(str(path)), text(str(scratch)), number(limit), number(size), text(expected_digest)] 56 argv = ['--io'] 57 zig_arguments = [] 58 for i, value in enumerate(values): 59 argument = directory/f'{label}-{i}.tree' 60 argument.write_bytes(value) 61 argv += ['--arg-tree-file', argument] 62 zig_arguments.append('tree-file:'+str(argument)) 63 expected = tree_list([expected_reply, number(position)]) 64 if args.zig_probe: 65 command = [str(args.zig_probe.resolve()),str(bundle),'probe',*zig_arguments] 66 output,peak = run_guarded(command,memory_mib=48,seconds=180) 67 (directory/(label+'-zig.log')).write_text(output) 68 assert bytes(map(int,output.strip())) == expected,label 69 zig_observations.append(dict(label=label,peak_kib=peak,command=command)) 70 else: 71 assert runner.invoke(label, pin, 'probe', argv) == expected, label 72 assert path.read_bytes() == payload, label 73 assert list(scratch.iterdir()) == [], label 74 checks.append(dict(label=label, input_bytes=len(payload), frame_bytes=size)) 75 for i, tree in enumerate([b'\0', b'\1\0', b'\2\1\0\0', number(42)]): 76 payload = indexed_bytes(tree) 77 check(f'tree-{i}', payload+b'next frame', len(payload), len(payload), digest(INDEXED, payload), result(True, tree), len(payload)) 78 payload = indexed_bytes(number(7)) 79 check('wrong-digest', payload, len(payload), len(payload), '0'*64, 80 problem('integrity-failed', 'CAS digest mismatch', 'unknown'), len(payload)) 81 check('truncated', payload[:-1], len(payload), len(payload), digest(INDEXED, payload), 82 problem('invalid-data', 'incomplete indexed frame', 'unknown'), len(payload)-1) 83 check('limit', payload, len(payload), len(payload)-1, digest(INDEXED, payload), 84 problem('invalid-bound', 'invalid snapshot size limit', 'none'), 0) 85 check('bad-digest', payload, len(payload), len(payload), 'A'*64, 86 problem('invalid-argument', '', 'none'), 0) 87 check('bad-bound-wire', payload, 2**63, 2**63, digest(INDEXED, payload), 88 problem('invalid-argument', '', 'none'), 0) 89 malformed = b'not a bundle' 90 check('malformed-authenticated', malformed, len(malformed), len(malformed), digest(INDEXED, malformed), 91 problem('invalid-data', 'invalid indexed frame' if args.zig_probe else 'section extends beyond bundle end', 'unknown'), len(malformed)) 92 check('malformed-unauthenticated', malformed, len(malformed), len(malformed), '0'*64, 93 problem('integrity-failed', 'CAS digest mismatch', 'unknown'), len(malformed)) 94 for size in [4096, 1048576, 16777216, 67108864]: 95 payload = padded(size) 96 check(f'transport-{size}', payload, len(payload), len(payload), digest(INDEXED, payload), result(True, b'\0'), len(payload)) 97 observations = [s for s in runner.records if s['label'].startswith('transport-')] 98 for sample in observations: 99 assert sample['peak_kib'] <= 48*1024, sample 100 assert sample['rts_process']['maximum_live_bytes'] <= 16*1024*1024, sample 101 files = ['lib/io-v2.tri', 'demos/io-v2/indexed-read.tri', 'test/io_v2_indexed.py', 102 'test/source_order.py', 'test/rts_metrics.py', 'test/native_indexed_store.py', 103 'test/executable_object.py', 'test/stable_bundle.py', 'src/ContentStore/Snapshot.hs', 104 'src/ContentStore/Indexed.hs', 'src/Wire.hs', 'src/IOV2/Codec.hs', 105 'src/IOV2/Resources.hs', 'src/IOV2/Driver.hs', 'src/IOV2/Protocol.hs', 106 'bootstrap/build-stable.arboricx'] 107 if args.zig_probe: files += ['ext/zig/src/indexed.zig','ext/zig/src/io_v2_snapshot.zig','ext/zig/src/io_v2_file_host.zig'] 108 (directory/'report.json').write_text(json.dumps(dict(checks=checks, observations=observations, zig_observations=zig_observations, 109 zig_probe_sha256=hashlib.sha256(args.zig_probe.read_bytes()).hexdigest() if args.zig_probe else None, 110 measurement_notes='Opaque extension grows transport, not DAG. RSS is sampled: tiny runs can exit before representative samples; use the 64-MiB control for the transfer RSS guard. No concurrency or large-DAG capacity claim.', 111 source_hashes={p:hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in files}), indent=2)+'\n') 112 print(f'{len(checks)} indexed-read controls passed') 113 114 if __name__ == '__main__': 115 main()