library_inspector.py (19620B)
1 #!/usr/bin/env python3 2 """Opt-in Haskell-only publication -> clean fetch -> offline inspector acceptance. 3 4 Build through Nix first. Publishes only into the supplied NEW local directory; 5 never contacts a public registry. Reports and artifacts are retained for reuse. 6 """ 7 import argparse 8 import hashlib 9 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer 10 import json 11 from pathlib import Path 12 import re 13 import shutil 14 import struct 15 import subprocess 16 import threading 17 import os 18 import tempfile 19 import time 20 from shipped_compiler import verify, ROOT 21 from stable_bundle import assembled, sections 22 23 RELEASES = [ 24 ('tricu-base', 'foundation', 'base_release', 'lib/packages/base.tri'), 25 ('tricu-file-io', 'fileio', 'io_release', 'lib/packages/io.tri'), 26 ('arboricx-inspection', 'inspection', 'inspection_release', 'lib/arboricx/inspect.tri'), 27 ] 28 29 30 def cases(directory): 31 """Independent byte fixtures; the Haskell oracle determines acceptance/report.""" 32 directory.mkdir() 33 result = [] 34 for name in ['id', 'append', 'false']: 35 target = directory / (name + '.arboricx') 36 shutil.copyfile(ROOT / 'test/fixtures' / target.name, target) 37 result.append((name, target, True)) 38 manifest, nodes = sections(ROOT / 'test/fixtures/id.arboricx') 39 # Preserve the wire-profile prefix, replace root/export rows independently. 40 at = 12 41 for _ in range(9): 42 size = struct.unpack_from('>I', manifest, at)[0]; at += 4 + size 43 capabilities = struct.unpack_from('>I', manifest, at)[0]; at += 4 44 assert capabilities == 0 45 at += 1 46 prefix = manifest[:at] 47 def text(value): 48 value = value.encode('utf-8') 49 return struct.pack('>I', len(value)) + value 50 def custom(exports, records): 51 m = prefix + struct.pack('>II', 1, 0) + text('root') + struct.pack('>I', len(exports)) 52 for name, root, kind, abi in exports: 53 m += text(name) + struct.pack('>I', root) + text(kind) + text(abi) 54 m += struct.pack('>II', 0, 0) # metadata and extension counts 55 n = struct.pack('>Q', len(records)) + b''.join(struct.pack('>I', len(r)) + r for r in records) 56 return assembled(m, n) 57 abi = 'arboricx.abi.tree.v1' 58 leaf = [b'\0'] 59 samples = [ 60 ('names-and-shared-exports', custom([('tab\tline\n%\x1b é', 0, 'term', abi), ('second', 0, 'term', abi)], leaf), True), 61 ('bad-magic', b'X' + assembled(manifest, nodes)[1:], False), 62 ('truncated', assembled(manifest, nodes)[:-1], False), 63 ('bad-manifest', assembled(b'X' + manifest[1:], nodes), False), 64 ('invalid-unselected-export', custom([('good', 0, 'term', abi), ('bad', 5, 'term', abi)], leaf), False), 65 ('invalid-unselected-abi', custom([('good', 0, 'term', abi), ('bad', 0, 'term', 'wrong')], leaf), False), 66 ('duplicate-export', custom([('same', 0, 'term', abi), ('same', 0, 'term', abi)], leaf), False), 67 ('bad-forward-edge', custom([('root', 0, 'term', abi)], [b'\1' + struct.pack('>I', 0)]), False), 68 ('duplicate-node', custom([('root', 0, 'term', abi)], [b'\0', b'\0']), False), 69 ] 70 # 41 physical nodes denote >2^40 logical occurrences. Inspection must not 71 # reconstruct or unfold this tree merely to report the stored record count. 72 dag = leaf + [b'\2' + struct.pack('>II', i - 1, i - 1) for i in range(1, 41)] 73 samples.append(('exponential-sharing', custom([('huge', 40, 'term', abi)], dag), True)) 74 for label, data, good in samples: 75 path = directory / (label + '.arboricx'); path.write_bytes(data) 76 result.append((label, path, good)) 77 return result 78 79 80 def stage_site(root, site, report): 81 """Administrative, opt-in staging; immutable bytes before discoverability. 82 83 The caller owns this local workspace exclusively. This is not network 84 publication, a signed attestation, or a concurrent index-merge protocol. 85 """ 86 if not report.get('complete'): 87 raise ValueError('Only an accepted publication can be staged') 88 index_path = site / 'packages.json' 89 index = json.loads(index_path.read_text()) 90 if index.get('schema') != 1: 91 raise ValueError('Unsupported site index') 92 releases = list(index['releases']) 93 planned = [] 94 titles = {'tricu-base': 'Foundation: lists, Results and decimal formatting', 95 'tricu-file-io': 'File IO and hashing actions', 96 'arboricx-inspection': 'Arboricx inspection: validate without execution'} 97 for name, release in report['releases'].items(): 98 data = (root / 'producer' / (name + '.arboricx')).read_bytes() 99 if hashlib.sha256(data).hexdigest() != release['bundle'] or not 0 < len(data) <= 1048576: 100 raise ValueError('Invalid site dependency: ' + name) 101 candidate = dict(name=name, version=release['version'], hash=release['bundle'], 102 manifest=release['manifest'], title=titles[name]) 103 existing = [r for r in releases if (r['name'], r['version']) == (name, release['version'])] 104 if existing and (len(existing) != 1 or any(existing[0][k] != candidate[k] for k in ['hash', 'manifest'])): 105 raise ValueError('Immutable site release conflict: ' + name) 106 if not existing: 107 releases.append(candidate) 108 path = site / 'bundles' / (release['bundle'] + '.arboricx') 109 if path.exists() and path.read_bytes() != data: 110 raise ValueError('Corrupt existing site bundle') 111 planned.append((path, data)) 112 # Validate the entire change before writing any new asset. 113 for path, data in planned: 114 if not path.exists(): 115 with path.open('xb') as output: 116 output.write(data) 117 index['releases'] = releases 118 with tempfile.NamedTemporaryFile(dir=site, prefix='.packages-', delete=False) as output: 119 temporary = Path(output.name) 120 output.write((json.dumps(index, indent=2) + '\n').encode()) 121 try: 122 temporary.chmod(0o644) 123 os.replace(temporary, index_path) 124 finally: 125 temporary.unlink(missing_ok=True) 126 127 128 def main(): 129 p = argparse.ArgumentParser(description=__doc__) 130 p.add_argument('tricu', type=Path) 131 p.add_argument('--oracle', required=True, type=Path) 132 p.add_argument('--directory', required=True, type=Path) 133 p.add_argument('--version', default='0.1.0', help='Exact release label for this library set') 134 p.add_argument('--stage-site', type=Path, help='After acceptance, stage bundles/releases in this static-site directory') 135 args = p.parse_args() 136 metadata = verify() 137 tricu, oracle, root = args.tricu.resolve(), args.oracle.resolve(), args.directory.resolve() 138 root.mkdir(parents=True, exist_ok=False) 139 producer, tools, application = root / 'producer', root / 'tools', root / 'application' 140 workspace, consumer = producer / 'workspace', root / 'consumer' 141 workspace.mkdir(parents=True); consumer.mkdir() 142 report = dict(complete=False, compiler_sha256=metadata['sha256'], sources={}, commands=[], releases={}, cases=[]) 143 started = time.monotonic() 144 def record(): 145 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 146 def command(executable, words, seconds=60, success=True, rts=True): 147 remaining = 2400 - (time.monotonic() - started) 148 if remaining <= 0: 149 raise TimeoutError('whole-job deadline exceeded') 150 argv = [str(executable), *map(str, words), *(['+RTS', '-N1', '-M8192m', '-RTS'] if rts else [])] 151 before = time.monotonic() 152 reply = subprocess.run(argv, cwd=consumer, capture_output=True, timeout=min(seconds, remaining)) 153 event = dict(argv=argv, seconds=time.monotonic()-before, code=reply.returncode, 154 stdout=reply.stdout.decode('utf-8', errors='replace'), stderr=reply.stderr.decode('utf-8', errors='replace')) 155 report['commands'].append(event); record() 156 assert (reply.returncode == 0) == success, event 157 return reply.stdout 158 def run(*words, **options): 159 return command(tricu, words, **options).decode('utf-8') 160 def imported(bundle, store): 161 output = run('arboricx', 'import', '--file', bundle, '--store', store) 162 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 163 compiler = imported(ROOT / metadata['bundle'], tools) 164 report['compiler_manifest'] = compiler 165 options = ['--toolchain', compiler, '--tool-store', tools, '--compile-timeout', '900'] 166 sources = {row['alias']: row['path'] for row in metadata['sources']} 167 sources.update(binary='lib/binary.tri', arboricx='lib/arboricx/arboricx.tri', 168 arboricx_common='lib/arboricx/common.tri', arboricx_manifest='lib/arboricx/manifest.tri', 169 arboricx_nodes='lib/arboricx/nodes.tri') 170 sources.update({alias: path for _, _, alias, path in RELEASES}) 171 for alias, path in sources.items(): 172 data = (ROOT / path).read_bytes(); (workspace / (alias + '.tri')).write_bytes(data) 173 report['sources'][path] = hashlib.sha256(data).hexdigest() 174 registry = root / 'registry' 175 for name, binding, alias, _ in RELEASES: 176 bundle = producer / (name + '.arboricx') 177 print('Compiling publication:', name, flush=True) 178 run('arboricx', 'compile', '--file', workspace / (alias + '.tri'), '--store', producer / 'store', 179 '--output', bundle, *options, seconds=930) 180 published = run('package', 'publish', '--root', registry, '--name', name, '--version', args.version, '--file', bundle) 181 assert published == run('package', 'publish', '--root', registry, '--name', name, '--version', args.version, '--file', bundle) 182 _, row = published.splitlines(); _, _, manifest, digest = row.split('\t') 183 report['releases'][name] = dict(version=args.version, manifest=manifest, bundle=digest, binding=binding) 184 record() 185 # Controlled GET peer serves exactly the immutable publication files. Native 186 # route behavior is independently exercised by package_registry.py. 187 requests = [] 188 routes = {} 189 for name, release in report['releases'].items(): 190 digest = release['bundle'] 191 routes['/_arboricx/package/' + name + '/' + release['version']] = registry / 'packages' / name / release['version'] 192 routes['/_arboricx/bundle/hash/' + digest] = registry / 'objects' / digest[:3] / (digest + '.arboricx') 193 class Handler(BaseHTTPRequestHandler): 194 def do_GET(self): 195 requests.append(self.path) 196 path = routes.get(self.path) 197 body = path.read_bytes() if path else b'not found' 198 self.send_response(200 if path else 404) 199 self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) 200 def log_message(self, *_): 201 pass 202 server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) 203 thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() 204 config = consumer / 'registries' 205 config.write_text('arboricx.registries.v1\nhome\thttp://127.0.0.1:' + str(server.server_port) + '\n') 206 project_file, project_lock = consumer / 'tricu.project', consumer / 'tricu.lock' 207 project_source = (ROOT / 'demos/package-inspector/tricu.project').read_text() 208 project_file.write_text(project_source.replace('\t0.1.0\n', '\t' + args.version + '\n')) 209 report['sources']['demos/package-inspector/tricu.project'] = hashlib.sha256(project_source.encode()).hexdigest() 210 def project(action, *extra, success=True): 211 return run('project', action, '--project', project_file, 212 *(['--registries', config] if action in ['resolve', 'update', 'fetch'] else []), 213 *(['--store', application] if action in ['fetch', 'bind', 'check', 'explain'] else []), 214 *extra, success=success) 215 assert not application.exists(), 'consumer starts without producer CAS objects' 216 port = server.server_port 217 try: 218 project('resolve') 219 assert not application.exists(), 'resolution must not import executable bytes' 220 project('bind', success=False) 221 project('fetch') 222 project('resolve') # frozen: no index refresh 223 finally: 224 server.shutdown(); server.server_close(); thread.join() 225 assert len(requests) == 6, requests 226 report['registry_requests'] = list(requests) 227 original_lock = project_lock.read_bytes() 228 (root / 'initial.lock').write_bytes(original_lock) 229 # Actual closed socket: subsequent operations cannot depend on the registry. 230 project('resolve'); project('fetch'); project('bind'); project('check') 231 assert not any(p.is_file() for p in (application / 'aliases').rglob('*')) 232 shutil.copyfile(ROOT / 'demos/package-inspector/main.tri', consumer / 'main.tri') 233 report['sources']['demos/package-inspector/main.tri'] = hashlib.sha256((consumer / 'main.tri').read_bytes()).hexdigest() 234 assert sorted(p.name for p in consumer.glob('*.tri')) == ['fileio.tri', 'foundation.tri', 'inspection.tri', 'main.tri'] 235 print('Compiling consumer with the registry stopped', flush=True) 236 bundle = root / 'inspector.arboricx' 237 run('arboricx', 'compile', '--file', consumer / 'main.tri', '--store', application, '--output', bundle, 238 *options, seconds=930) 239 inspector = imported(bundle, application) 240 report['inspector_manifest'] = inspector 241 def inspect(path, good=True, granted=True): 242 return run('arboricx', 'run', '--store', application, '--module', inspector, '--name', 'main', 243 '--arg', path, '--io', '--no-result', *(['--allow-read', path] if granted else []), success=good, seconds=180) 244 inputs = cases(root / 'cases') 245 # A real executable IO export would abort if the inspector ran it. 246 shutil.copyfile(consumer / 'fileio.tri', root / 'cases/fileio.tri') 247 payload = root / 'cases/payload.tri' 248 payload.write_text('!import fileio as IO\n!export main\nmain = IO.io (IO.abortIO "INSPECTED EXPORT WAS EXECUTED")\n') 249 inert = root / 'cases/inert-io.arboricx' 250 run('arboricx', 'compile', '--file', payload, '--store', application, '--output', inert, *options, seconds=930) 251 inputs.append(('inert-io', inert, True)) 252 inputs.extend((name, producer / (name + '.arboricx'), True) for name in report['releases']) 253 inputs.append(('inspector-self', bundle, True)) 254 for label, path, good in inputs: 255 print('Inspecting:', label, flush=True) 256 expected = command(oracle, [path], success=good, rts=False).decode('utf-8') 257 actual = inspect(path, good) 258 if good: 259 assert actual == expected, (label, actual, expected) 260 assert actual.splitlines()[1] == 'bundle-sha256\t' + hashlib.sha256(path.read_bytes()).hexdigest() 261 else: 262 assert 'arboricx.inspect.v1' not in actual, (label, actual) 263 report['cases'].append(dict(label=label, valid=good, report=actual)); record() 264 inspect(root / 'cases/id.arboricx', False, False) 265 inspect(root / 'cases/missing.arboricx', False) 266 # Deliberate one-dependency update: add a public constant to the foundation 267 # interface, preserving the three operations used by the authored consumer. 268 # This is a local acceptance release, never staged in the site's public index. 269 updated_source = workspace / 'base_update_release.tri' 270 updated_source.write_text((ROOT / 'lib/packages/base.tri').read_text().replace( 271 '!export showNumber\n', '!export showNumber\n!export revision\n') + '\nrevision = 1\n') 272 report['update_source_sha256'] = hashlib.sha256(updated_source.read_bytes()).hexdigest() 273 updated_bundle = producer / 'base-update.arboricx' 274 run('arboricx', 'compile', '--file', updated_source, '--store', producer / 'store', 275 '--output', updated_bundle, *options, seconds=930) 276 next_version = 'project-update-test' if args.version != 'project-update-test' else 'project-update-test-2' 277 update_release = run('package', 'publish', '--root', registry, '--name', 'tricu-base', 278 '--version', next_version, '--file', updated_bundle).splitlines()[1].split('\t') 279 _, _, new_manifest, new_digest = update_release 280 assert new_manifest != report['releases']['tricu-base']['manifest'] 281 routes['/_arboricx/package/tricu-base/' + next_version] = registry / 'packages/tricu-base' / next_version 282 routes['/_arboricx/bundle/hash/' + new_digest] = registry / 'objects' / new_digest[:3] / (new_digest + '.arboricx') 283 project_file.write_text(project_file.read_text().replace('tricu-base\t' + args.version + '\n', 'tricu-base\t' + next_version + '\n')) 284 project('check', success=False) 285 server = ThreadingHTTPServer(('127.0.0.1', port), Handler) 286 thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() 287 try: 288 project('resolve', success=False) 289 assert len(requests) == 6, requests 290 diff = project('update', '--dependency', 'foundation') 291 assert 'Updated foundation' in diff and 'Updated fileio' not in diff and 'Updated inspection' not in diff 292 report['update_diff'] = diff 293 project('check', success=False) 294 project('bind', success=False) # no implicit fetch 295 project('fetch') 296 finally: 297 server.shutdown(); server.server_close(); thread.join() 298 assert len(requests) == 8, requests 299 def lock_rows(data): 300 return dict((line.split('\t')[0], line) for line in data.decode().splitlines()[1:]) 301 before, after = lock_rows(original_lock), lock_rows(project_lock.read_bytes()) 302 assert before['fileio'] == after['fileio'] and before['inspection'] == after['inspection'] 303 assert before['foundation'] != after['foundation'] 304 project('fetch'); project('bind'); project('check') 305 updated_inspector_bundle = root / 'inspector-updated.arboricx' 306 run('arboricx', 'compile', '--file', consumer / 'main.tri', '--store', application, 307 '--output', updated_inspector_bundle, *options, seconds=930) 308 updated_inspector = imported(updated_inspector_bundle, application) 309 report['updated_inspector_manifest'] = updated_inspector 310 report['consumer_identity_unchanged'] = updated_inspector == inspector 311 assert report['consumer_identity_unchanged'], 'pinned inspector fixture changed after the additive foundation update' 312 inspector = updated_inspector 313 target = root / 'cases/id.arboricx' 314 assert inspect(target) == command(oracle, [target], rts=False).decode('utf-8') 315 # Explain and compare the retained locks after the registry has closed. 316 report['project_explanation'] = project('explain') 317 report['project_diff'] = run('project', 'diff', '--before', root / 'initial.lock', 318 '--after', project_lock, '--store', application) 319 assert 'export revision: added' in report['project_diff'] 320 foundation_diff = report['project_diff'].split('Binding foundation\n')[1].split('Binding inspection\n')[0] 321 assert foundation_diff.count('tree identical') == 3 and 'tree different' not in foundation_diff 322 assert 'local module: verified' in report['project_explanation'] 323 report['updated_registry_requests'] = list(requests) 324 report['complete'] = True; report['seconds'] = time.monotonic() - started; record() 325 if args.stage_site: 326 stage_site(root, args.stage_site.resolve(), report) 327 report['staged_site'] = str(args.stage_site.resolve()); record() 328 print('PASS project dependencies, clean fetch, offline inspector, isolated dependency update, Haskell parity and negative controls') 329 print('Inspector:', inspector, '\nStore:', application) 330 331 332 if __name__ == '__main__': 333 main()