bootstrap_store_check.py (6331B)
1 #!/usr/bin/env python3 2 """Independent audit of a completed mapped compiler closure and its CAS edges.""" 3 import argparse 4 import hashlib 5 import json 6 from pathlib import Path 7 from generation_backend import BACKENDS, recorded_backend 8 from executable_object import KINDS, FLAT, INDEXED, verify_payload 9 10 parser = argparse.ArgumentParser(description=__doc__) 11 parser.add_argument('run', type=Path) 12 parser.add_argument('--map', type=Path) 13 parser.add_argument('--source-root', type=Path) 14 parser.add_argument('--root') 15 args = parser.parse_args() 16 run = args.run.resolve() 17 provenance = json.loads((run / 'inputs.json').read_text()) if (run / 'inputs.json').exists() else {} 18 source_root = args.source_root or Path(provenance.get('source_root', '.')) 19 root_name = args.root or provenance.get('root', 'source_map') 20 cas = run / 'cas' 21 status = json.loads((run / 'status.json').read_text()) 22 compiler = status.get('compiler_bundle_sha256', run.name) 23 backend = recorded_backend(status) 24 assert recorded_backend(provenance) == backend, 'backend provenance mismatch' 25 worker_export, backend_options = BACKENDS[backend] 26 for record in (status, provenance): 27 if 'backend' in record: 28 assert record.get('backend_options') == backend_options, 'backend options provenance mismatch' 29 assert record.get('worker_export') == worker_export, 'worker entrypoint provenance mismatch' 30 seen = set() 31 export_cache = {} 32 33 34 def digest(domain, payload): 35 return hashlib.sha256(domain.encode() + b'\0' + payload).hexdigest() 36 37 38 def get(domain, name): 39 assert len(name) == 64 and all(c in '0123456789abcdef' for c in name) 40 data = (cas / 'objects' / name[:3] / name).read_bytes() 41 assert digest(domain, data) == name, (domain, name) 42 if domain in KINDS and name not in seen: 43 verify_payload(domain, data) 44 seen.add(name) 45 return data 46 47 48 def exports(manifest): 49 if manifest in export_cache: 50 return export_cache[manifest] 51 rows = get('arboricx.module-manifest.v1', manifest).decode().splitlines() 52 assert rows[0] == 'arboricx.module-manifest.v1' 53 result = {} 54 for row in rows[1:]: 55 tag, name, kind, tree, abi = row.split('\t') 56 assert (tag, abi) == ('export', 'arboricx.abi.tree.v1') and kind in KINDS 57 assert name not in result 58 result[name] = tree 59 get(kind, tree) 60 assert result 61 export_cache[manifest] = result 62 return result 63 64 65 map_bytes = (args.map or (run / 'source-map.v1' if (run / 'source-map.v1').exists() else Path('bootstrap/compiler-source-map.v1'))).read_bytes() 66 map_rows = [row.split('\t') for row in map_bytes.decode().splitlines()[1:]] 67 entries = {row[1]: row for row in map_rows} 68 required = set() 69 def visit(name): 70 if name in required: 71 return 72 required.add(name) 73 for dependency in ([] if entries[name][4] == '-' else entries[name][4].split(',')): 74 visit(dependency) 75 visit(root_name) 76 indices = {} 77 for name in required: 78 checkpoint = cas / 'cache/source-build' / name 79 assert checkpoint.is_file(), ('missing checkpoint', name) 80 version, key, manifest, receipt = checkpoint.read_text().split('\t') 81 assert version == 'arboricx.source-build-cache.v1' 82 indices[name] = (key, manifest, receipt) 83 options = digest('arboricx.build-options.v1', backend_options.encode()) 84 for name, (key, manifest, receipt) in indices.items(): 85 overlay = (run / 'overlay' / (name + '.tri')).read_bytes() 86 rows = get('arboricx.build-receipt.v1', receipt).decode().splitlines() 87 assert rows[:5] == ['arboricx.build-receipt.v1', 88 'source\t' + digest('arboricx.source.v1', overlay), 'compiler\t' + compiler, 89 'options\t' + options, 'output\t' + manifest] 90 key_body = 'arboricx.source-build-key.v1\n' + '\n'.join(rows[1:4]) + '\n' + ( 91 '\n'.join(rows[5:]) + '\n' if rows[5:] else '') 92 assert key == digest('arboricx.source-build-key.v1', key_body.encode()) 93 locals_seen = [] 94 for row in rows[5:]: 95 fields = row.split('\t') 96 if fields[0] == 'local': 97 _, alias, local_manifest, local_receipt = fields 98 assert indices[alias][1:] == (local_manifest, local_receipt) 99 locals_seen.append(alias) 100 else: 101 assert fields[0] == 'pin' 102 _, _, pinned_manifest, export, tree = fields 103 assert exports(pinned_manifest)[export] == tree 104 from native_header import native_header 105 declared, _ = native_header(overlay.decode()) 106 map_row = next(row for row in map_rows if row[1] == name) 107 if map_row[0] == 'native': 108 assert overlay == (source_root / map_row[3]).read_bytes(), ('rewritten native module', name) 109 assert declared == ([] if map_row[4] == '-' else map_row[4].split(',')) 110 assert locals_seen == declared 111 exports(manifest) 112 113 map_receipts = [] 114 for obj in (cas / 'objects').glob('*/*'): 115 with obj.open('rb') as handle: 116 if not handle.read(64).startswith(b'arboricx.source-map-build.v1\n'): 117 continue 118 rows = get('arboricx.source-map-build.v1', obj.name).decode().splitlines() 119 map_hash = rows[1].split('\t')[1] 120 if get('arboricx.source-map.v1', map_hash) != map_bytes: 121 continue 122 if rows[2] != 'compiler\t' + compiler or rows[3].split('\t')[1] != root_name: 123 continue 124 if rows[3] != 'root\t' + root_name + '\t' + '\t'.join(indices[root_name][1:]): 125 continue 126 records = {fields[1]: fields[2:] for fields in (row.split('\t') for row in rows[4:])} 127 expected_records = {alias: [digest('arboricx.source.v1', (source_root / path).read_bytes()), 128 digest('arboricx.source.v1', (run / 'overlay' / (alias + '.tri')).read_bytes())] 129 for _, alias, _, path, _ in map_rows} 130 if records != expected_records: 131 continue 132 map_receipts.append(obj.name) 133 assert map_receipts 134 export_indices = list((cas / 'cache/source-export').glob('*/*')) 135 for index in export_indices: 136 pin = index.read_text() 137 assert len(pin) == 64 and all(c in '0123456789abcdef' for c in pin) 138 with (cas / 'objects' / pin[:3] / pin).open('rb') as stream: 139 domain = INDEXED if stream.read(8) == b'ARBORICX' else FLAT 140 get(domain, pin) 141 print(f'Verified {backend} backend, {len(indices)} modules, {len(seen)} distinct complete trees, ' 142 f'{len(export_indices)} export indices, map receipts {map_receipts}')