native_indexed_store.py (9525B)
1 #!/usr/bin/env python3 2 """Opt-in current native object/manifest policy checks with retained compiler. 3 4 No generations or fixture publication. Narrow support transport is completely 5 hash-audited; expected object and manifest bytes are authored independently. 6 """ 7 import argparse 8 import hashlib 9 import json 10 from pathlib import Path 11 import re 12 import struct 13 14 from collections_batch import ProbeRunner 15 from bundle_dag import read 16 from executable_object import INDEXED, indexed_manifest, verify_payload 17 from http_headers import LEAF, byte_tree, pair, result, number, tree_list 18 from source_order import end 19 from stable_bundle import assembled 20 21 ROOT = Path(__file__).resolve().parent.parent 22 23 24 def digest(domain, payload): 25 return hashlib.sha256(domain.encode() + b'\0' + payload).hexdigest() 26 27 28 def indexed_bytes(tree): 29 nodes, seen = [], {} 30 def visit(at): 31 tag, cursor = tree[at], at + 1 32 children = [] 33 for _ in range(tag): 34 child, cursor = visit(cursor) 35 children.append(child) 36 key = (tag, *children) 37 if key not in seen: 38 seen[key] = len(nodes) 39 nodes.append(key) 40 return seen[key], cursor 41 root, consumed = visit(0) 42 assert consumed == len(tree) 43 records = [] 44 for tag, *children in nodes: 45 data = bytes([tag]) + b''.join(struct.pack('>I', child) for child in children) 46 records.append(struct.pack('>I', len(data)) + data) 47 return assembled(indexed_manifest(root), struct.pack('>Q', len(nodes)) + b''.join(records)) 48 49 50 def main(): 51 parser = argparse.ArgumentParser(description=__doc__) 52 parser.add_argument('tricu', type=Path) 53 parser.add_argument('--generation', type=Path, required=True) 54 parser.add_argument('--compiler', type=Path, required=True) 55 parser.add_argument('--directory', type=Path, required=True) 56 args = parser.parse_args() 57 root = args.directory.resolve(); root.mkdir(parents=True) 58 runner = ProbeRunner(args.tricu, root, 1500, memory_mib=8192) 59 old = args.generation.resolve() / 'cas' 60 object_source = (ROOT / 'lib/source/object.tri').read_text() 61 module_source = (ROOT / 'lib/source/module.tri').read_text() 62 body = object_source.replace('N.lowerHex', 'nativeLowerHex') + '\n' + module_source[:module_source.index('sourceModulePublish root')] 63 body += module_source[module_source.index('\nsourceModulePublishNames root'):module_source.index('-- One native header interpretation')] 64 body = '\n'.join(line for line in body.splitlines() if not line.startswith(('!import', '!export'))) 65 constructor_names = ('readIndexedTree', 'hashIndexedTree', 'writeIndexedTreeIfAbsent') 66 constructors = [line for line in (ROOT / 'lib/io.tri').read_text().splitlines() 67 if any(line.startswith(name + ' =') for name in constructor_names)] 68 assert sorted(line.split()[0] for line in constructors) == sorted(constructor_names) 69 body = '\n'.join(constructors) + '\n' + body 70 wrapper = ''' 71 publishProbe root tree = io (sourceModulePublishNames root [(pair "value" tree)] ["value"]) 72 readProbe root manifest = io (sourceModuleLoadSelected root manifest "value") 73 decodeProbe bytes = sourceModuleDecode bytes 74 ''' 75 local = set(re.findall(r'^([A-Za-z_][A-Za-z0-9_?\-]*)[^\n=]*=', body, re.M)) 76 words = set(re.findall(r'[A-Za-z_][A-Za-z0-9_?\-]*', body + wrapper)) 77 selected = {} 78 for module in ['prelude', 'io', 'source_resolve', 'source_lexer', 'parser', 'native_math']: 79 pin = (old / 'cache/source-build' / module).read_text().split('\t')[2] 80 payload = (old / 'objects' / pin[:3] / pin).read_bytes() 81 assert digest('arboricx.module-manifest.v1', payload) == pin 82 for row in payload.decode().splitlines()[1:]: 83 _, name, kind, tree, abi = row.split('\t') 84 assert kind == 'arboricx.tree-term.v1' and abi == 'arboricx.abi.tree.v1' 85 if module == 'native_math' and name != 'lowerHex': continue 86 alias = 'nativeLowerHex' if module == 'native_math' else name 87 if alias in words and alias not in local: 88 selected[alias] = tree 89 command = ['export', '--store', old, '--output', root / 'support.arboricx'] 90 for name, tree in selected.items(): command += ['--target', tree, '--name', name] 91 runner.run('support-export', command, 180) 92 _, _, nodes, exports, _ = read(root / 'support.arboricx') 93 assert set(exports) == set(selected) 94 sizes = [] 95 for node in nodes: sizes.append(1 + sum(sizes[child] for child in node[1:])) 96 expanded = sum(sizes[index] for index in exports.values()) 97 assert expanded <= 64 * 1024 * 1024, 'support exceeds unchanged flat transport budget' 98 reply = runner.run('support-import', ['import', '--file', root / 'support.arboricx', '--store', runner.store, '--flat'], 180) 99 support = re.search(r'Created module alias .* -> ([a-f0-9]{64})', reply)[1] 100 payload = (runner.store / 'objects' / support[:3] / support).read_bytes() 101 assert digest('arboricx.module-manifest.v1', payload) == support 102 entries = [row.split('\t') for row in payload.decode().splitlines()[1:]] 103 assert len(entries) == len(selected) and {row[1]: row[3] for row in entries} == selected 104 for pin in set(selected.values()): 105 assert digest('arboricx.tree-term.v1', (runner.store / 'objects' / pin[:3] / pin).read_bytes()) == pin 106 (root / 'support.tri').write_text(f'!export * from {support}\n') 107 (root / 'driver.tri').write_text('!import support *\n!export publishProbe\n!export readProbe\n!export decodeProbe\n' + body + wrapper) 108 compiler = runner.imported(args.compiler, 'compiler-import') 109 runner.run('compile', ['compile', '--file', root / 'driver.tri', '--toolchain', compiler, 110 '--tool-store', runner.store, '--store', runner.store, 111 '--output', root / 'driver.arboricx', '--compile-timeout', '900'], 900) 112 pin = runner.imported(root / 'driver.arboricx', 'driver-import') 113 checks = [] 114 def invoke(label, name, values, io=True): 115 options = ['--io', '--allow-read', root, '--allow-write', root] if io else [] 116 for index, value in enumerate(values): 117 path = root / f'arg-{index}.tree'; path.write_bytes(value) 118 options += ['--arg-tree-file', path] 119 output = runner.run(label, ['run', '--module', pin, '--name', name, '--store', runner.store, 120 '--format', 'ternary', *options], 180) 121 raw = bytes(int(c) for c in re.search(r'(?m)^([012]+)', output)[1]) 122 return raw[:end(raw)] 123 values = [LEAF, b'\1\0', pair(b'\1\0', b'\2\0\0'), number(42)] 124 for index, tree in enumerate(values): 125 store = root / f'native-{index}' 126 path_arg = byte_tree(str(store).encode()) 127 payload = indexed_bytes(tree); verify_payload(INDEXED, payload) 128 tree_pin = digest(INDEXED, payload) 129 manifest = f'arboricx.module-manifest.v1\nexport\tvalue\t{INDEXED}\t{tree_pin}\tarboricx.abi.tree.v1\n'.encode() 130 manifest_pin = digest('arboricx.module-manifest.v1', manifest) 131 expected = result(True, byte_tree(manifest_pin.encode()), LEAF) 132 assert invoke(f'publish-{index}', 'publishProbe', [path_arg, tree]) == expected 133 assert (store / 'objects' / tree_pin[:3] / tree_pin).read_bytes() == payload 134 assert (store / 'objects' / manifest_pin[:3] / manifest_pin).read_bytes() == manifest 135 assert invoke(f'repeat-{index}', 'publishProbe', [path_arg, tree]) == expected 136 assert invoke(f'read-{index}', 'readProbe', [path_arg, byte_tree(manifest_pin.encode())]) == result(True, tree, LEAF) 137 for mode in ['module', 'target']: 138 output = root / f'export-{index}-{mode}.arboricx' 139 runner.run(f'export-{index}-{mode}', ['export', '--store', store, 140 '--' + mode, manifest_pin if mode == 'module' else tree_pin, '--name', 'value', '--output', output], 180) 141 _, _, actual_nodes, exports, _ = read(output) 142 _, _, expected_nodes, expected_exports, _ = read(store / 'objects' / tree_pin[:3] / tree_pin) 143 assert actual_nodes == expected_nodes and exports == {'value': expected_exports['root']} 144 loaded = runner.imported(output, f'reimport-{index}-{mode}') 145 answer = runner.run(f'execute-{index}-{mode}', ['run', '--module', loaded, '--name', 'value', 146 '--store', runner.store, '--format', 'ternary'], 180) 147 assert re.search(r'(?m)^([012]+)', answer)[1] == ''.join(map(str, tree)) 148 old = manifest.replace(INDEXED.encode(), b'arboricx.tree-term.v1') 149 assert invoke(f'old-kind-{index}', 'decodeProbe', [byte_tree(old)], False) == result(False, byte_tree(b'unsupported export kind'), LEAF) 150 (store / 'objects' / tree_pin[:3] / tree_pin).write_bytes(b'corrupt') 151 assert invoke(f'corrupt-read-{index}', 'readProbe', [path_arg, byte_tree(manifest_pin.encode())]) == result(False, byte_tree(b'CAS digest mismatch'), LEAF) 152 assert invoke(f'corrupt-write-{index}', 'publishProbe', [path_arg, tree]) == result(False, byte_tree(b'CAS object exists with different bytes'), LEAF) 153 checks.append(index) 154 (root / 'result.json').write_text(json.dumps(dict(complete=True, checked_values=checks, 155 support_exports=len(selected), expanded_support_bytes=expanded), indent=2) + '\n') 156 print('Native indexed publication/read/manifest/host-export and corruption controls passed.') 157 158 159 if __name__ == '__main__': main()