library_inspector.py (21136B)
1 #!/usr/bin/env python3 2 """Opt-in Haskell-only publication -> clean fetch -> offline inspector acceptance. 3 4 Build through Nix first. Publishes only into the supplied NEW local directory; 5 never contacts a public registry. Reports and artifacts are retained for reuse. 6 """ 7 import argparse 8 import hashlib 9 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer 10 import json 11 from pathlib import Path 12 import re 13 import shutil 14 import struct 15 import subprocess 16 import threading 17 import os 18 import tempfile 19 import time 20 from shipped_compiler import verify, ROOT 21 from stable_bundle import assembled, sections 22 23 RELEASES = [ 24 ('tricu-base', 'foundation', 'base_release', 'lib/packages/base.tri'), 25 ('tricu-file-io', 'fileio', 'io_release', 'lib/packages/io.tri'), 26 ('arboricx-inspection', 'inspection', 'inspection_release', 'lib/arboricx/inspect.tri'), 27 ] 28 29 30 def cases(directory): 31 """Independent byte fixtures; the Haskell oracle determines acceptance/report.""" 32 directory.mkdir() 33 result = [] 34 for name in ['id', 'append', 'false']: 35 target = directory / (name + '.arboricx') 36 shutil.copyfile(ROOT / 'test/fixtures' / target.name, target) 37 result.append((name, target, True)) 38 manifest, nodes = sections(ROOT / 'test/fixtures/id.arboricx') 39 # Preserve the wire-profile prefix, replace root/export rows independently. 40 at = 12 41 for _ in range(9): 42 size = struct.unpack_from('>I', manifest, at)[0]; at += 4 + size 43 capabilities = struct.unpack_from('>I', manifest, at)[0]; at += 4 44 assert capabilities == 0 45 at += 1 46 prefix = manifest[:at] 47 def text(value): 48 value = value.encode('utf-8') 49 return struct.pack('>I', len(value)) + value 50 def custom(exports, records): 51 m = prefix + struct.pack('>II', 1, 0) + text('root') + struct.pack('>I', len(exports)) 52 for name, root, kind, abi in exports: 53 m += text(name) + struct.pack('>I', root) + text(kind) + text(abi) 54 m += struct.pack('>II', 0, 0) # metadata and extension counts 55 n = struct.pack('>Q', len(records)) + b''.join(struct.pack('>I', len(r)) + r for r in records) 56 return assembled(m, n) 57 abi = 'arboricx.abi.tree.v1' 58 leaf = [b'\0'] 59 samples = [ 60 ('names-and-shared-exports', custom([('tab\tline\n%\x1b é', 0, 'term', abi), ('second', 0, 'term', abi)], leaf), True), 61 ('bad-magic', b'X' + assembled(manifest, nodes)[1:], False), 62 ('truncated', assembled(manifest, nodes)[:-1], False), 63 ('bad-manifest', assembled(b'X' + manifest[1:], nodes), False), 64 ('invalid-unselected-export', custom([('good', 0, 'term', abi), ('bad', 5, 'term', abi)], leaf), False), 65 ('invalid-unselected-abi', custom([('good', 0, 'term', abi), ('bad', 0, 'term', 'wrong')], leaf), False), 66 ('duplicate-export', custom([('same', 0, 'term', abi), ('same', 0, 'term', abi)], leaf), False), 67 ('bad-forward-edge', custom([('root', 0, 'term', abi)], [b'\1' + struct.pack('>I', 0)]), False), 68 ('duplicate-node', custom([('root', 0, 'term', abi)], [b'\0', b'\0']), False), 69 ] 70 # 41 physical nodes denote >2^40 logical occurrences. Inspection must not 71 # reconstruct or unfold this tree merely to report the stored record count. 72 dag = leaf + [b'\2' + struct.pack('>II', i - 1, i - 1) for i in range(1, 41)] 73 samples.append(('exponential-sharing', custom([('huge', 40, 'term', abi)], dag), True)) 74 for label, data, good in samples: 75 path = directory / (label + '.arboricx'); path.write_bytes(data) 76 result.append((label, path, good)) 77 return result 78 79 80 def stage_site(root, site, report): 81 """Administrative, opt-in staging; immutable bytes before discoverability. 82 83 The caller owns this local workspace exclusively. This is not network 84 publication, a signed attestation, or a concurrent index-merge protocol. 85 """ 86 version = report.get('io_version', 1) 87 if type(version) is not int or version not in (1, 2): 88 raise ValueError('Unsupported publication IO version') 89 if not report.get('complete'): 90 raise ValueError('Only an accepted publication can be staged') 91 index_path = site / 'packages.json' 92 index = json.loads(index_path.read_text()) 93 if index.get('schema') != 1: 94 raise ValueError('Unsupported site index') 95 releases = list(index['releases']) 96 planned = [] 97 titles = {'tricu-base': 'Foundation: lists, Results and decimal formatting', 98 'tricu-file-io': 'File IO and hashing actions', 99 'arboricx-inspection': 'Arboricx inspection: validate without execution'} 100 for name, release in report['releases'].items(): 101 data = (root / 'producer' / (name + '.arboricx')).read_bytes() 102 if hashlib.sha256(data).hexdigest() != release['bundle'] or not 0 < len(data) <= 1048576: 103 raise ValueError('Invalid site dependency: ' + name) 104 candidate = dict(name=name, version=release['version'], hash=release['bundle'], 105 manifest=release['manifest'], title=titles[name]) 106 existing = [r for r in releases if (r['name'], r['version']) == (name, release['version'])] 107 if existing and (len(existing) != 1 or any(existing[0][k] != candidate[k] for k in ['hash', 'manifest'])): 108 raise ValueError('Immutable site release conflict: ' + name) 109 if not existing: 110 releases.append(candidate) 111 path = site / 'bundles' / (release['bundle'] + '.arboricx') 112 if path.exists() and path.read_bytes() != data: 113 raise ValueError('Corrupt existing site bundle') 114 planned.append((path, data)) 115 # Validate the entire change before writing any new asset. 116 for path, data in planned: 117 if not path.exists(): 118 with path.open('xb') as output: 119 output.write(data) 120 index['releases'] = releases 121 with tempfile.NamedTemporaryFile(dir=site, prefix='.packages-', delete=False) as output: 122 temporary = Path(output.name) 123 output.write((json.dumps(index, indent=2) + '\n').encode()) 124 try: 125 temporary.chmod(0o644) 126 os.replace(temporary, index_path) 127 finally: 128 temporary.unlink(missing_ok=True) 129 130 131 def main(): 132 p = argparse.ArgumentParser(description=__doc__) 133 p.add_argument('tricu', type=Path) 134 p.add_argument('--oracle', required=True, type=Path) 135 p.add_argument('--directory', required=True, type=Path) 136 p.add_argument('--io-version', type=int, choices=[2], default=2, help='Consumer IO protocol (current sources require v2)') 137 p.add_argument('--version', help='Exact release label for this library set') 138 p.add_argument('--stage-site', type=Path, help='After acceptance, stage bundles/releases in this static-site directory') 139 args = p.parse_args() 140 if os.geteuid() == 0: 141 p.error('v2 OS-permission negative control requires a non-root user') 142 base_version = '0.5.0-io-v2' 143 args.version = args.version or base_version 144 demo = 'demos/package-inspector' 145 releases = RELEASES 146 metadata = verify() 147 tricu, oracle, root = args.tricu.resolve(), args.oracle.resolve(), args.directory.resolve() 148 root.mkdir(parents=True, exist_ok=False) 149 producer, tools, application = root / 'producer', root / 'tools', root / 'application' 150 workspace, consumer = producer / 'workspace', root / 'consumer' 151 workspace.mkdir(parents=True); consumer.mkdir() 152 # Isolate generated sibling sources from any enclosing checkout workspace. 153 for selected in (workspace, consumer): 154 (selected / "tricu.workspace").write_text("tricu.workspace.v1\n") 155 report = dict(complete=False, io_version=args.io_version, compiler_io_version=metadata['io_version'], compiler_sha256=metadata['sha256'], sources={}, commands=[], releases={}, cases=[]) 156 started = time.monotonic() 157 def record(): 158 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 159 def command(executable, words, seconds=60, success=True, rts=True): 160 remaining = 2400 - (time.monotonic() - started) 161 if remaining <= 0: 162 raise TimeoutError('whole-job deadline exceeded') 163 argv = [str(executable), *map(str, words), *(['+RTS', '-N1', '-M8192m', '-RTS'] if rts else [])] 164 before = time.monotonic() 165 reply = subprocess.run(argv, cwd=consumer, capture_output=True, timeout=min(seconds, remaining)) 166 event = dict(argv=argv, seconds=time.monotonic()-before, code=reply.returncode, 167 stdout=reply.stdout.decode('utf-8', errors='replace'), stderr=reply.stderr.decode('utf-8', errors='replace')) 168 report['commands'].append(event); record() 169 assert (reply.returncode == 0) == success, event 170 return reply.stdout 171 def run(*words, **options): 172 return command(tricu, words, **options).decode('utf-8') 173 def imported(bundle, store): 174 output = run('arboricx', 'import', '--file', bundle, '--store', store) 175 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 176 compiler = imported(ROOT / metadata['bundle'], tools) 177 report['compiler_manifest'] = compiler 178 options = ['--toolchain', compiler, '--toolchain-io-version', str(metadata['io_version']), '--tool-store', tools, '--compile-timeout', '900'] 179 sources = {row['alias']: row['path'] for row in metadata['sources']} 180 sources.update(binary='lib/binary.tri', arboricx='lib/arboricx/arboricx.tri', 181 arboricx_common='lib/arboricx/common.tri', arboricx_manifest='lib/arboricx/manifest.tri', 182 arboricx_nodes='lib/arboricx/nodes.tri') 183 sources.update({alias: path for _, _, alias, path in releases}) 184 sources.update(io='lib/io.tri', io_storage='lib/io/storage.tri', io_text='lib/io/text.tri') 185 for alias, path in sources.items(): 186 data = (ROOT / path).read_bytes(); (workspace / (alias + '.tri')).write_bytes(data) 187 report['sources'][path] = hashlib.sha256(data).hexdigest() 188 registry = root / 'registry' 189 for name, binding, alias, _ in releases: 190 bundle = producer / (name + '.arboricx') 191 print('Compiling publication:', name, flush=True) 192 run('arboricx', 'compile', '--file', workspace / (alias + '.tri'), '--store', producer / 'store', 193 '--output', bundle, *options, seconds=930) 194 published = run('package', 'publish', '--root', registry, '--name', name, '--version', args.version, '--file', bundle) 195 assert published == run('package', 'publish', '--root', registry, '--name', name, '--version', args.version, '--file', bundle) 196 _, row = published.splitlines(); _, _, manifest, digest = row.split('\t') 197 report['releases'][name] = dict(version=args.version, manifest=manifest, bundle=digest, binding=binding) 198 record() 199 # Controlled GET peer serves exactly the immutable publication files. Native 200 # route behavior is independently exercised by package_registry.py. 201 requests = [] 202 routes = {} 203 for name, release in report['releases'].items(): 204 digest = release['bundle'] 205 routes['/_arboricx/package/' + name + '/' + release['version']] = registry / 'packages' / name / release['version'] 206 routes['/_arboricx/bundle/hash/' + digest] = registry / 'objects' / digest[:3] / (digest + '.arboricx') 207 class Handler(BaseHTTPRequestHandler): 208 def do_GET(self): 209 requests.append(self.path) 210 path = routes.get(self.path) 211 body = path.read_bytes() if path else b'not found' 212 self.send_response(200 if path else 404) 213 self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) 214 def log_message(self, *_): 215 pass 216 server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) 217 thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() 218 config = consumer / 'registries' 219 config.write_text('arboricx.registries.v1\nhome\thttp://127.0.0.1:' + str(server.server_port) + '\n') 220 project_file, project_lock = consumer / 'tricu.project', consumer / 'tricu.lock' 221 project_source = (ROOT / demo / 'tricu.project').read_text() 222 project_file.write_text(project_source.replace('\t' + base_version + '\n', '\t' + args.version + '\n')) 223 report['sources'][demo + '/tricu.project'] = hashlib.sha256(project_source.encode()).hexdigest() 224 def project(action, *extra, success=True): 225 return run('project', action, '--project', project_file, 226 *(['--registries', config] if action in ['resolve', 'update', 'fetch'] else []), 227 *(['--store', application] if action in ['fetch', 'bind', 'check', 'explain'] else []), 228 *extra, success=success) 229 assert not application.exists(), 'consumer starts without producer CAS objects' 230 port = server.server_port 231 try: 232 project('resolve') 233 assert not application.exists(), 'resolution must not import executable bytes' 234 project('bind', success=False) 235 project('fetch') 236 project('resolve') # frozen: no index refresh 237 finally: 238 server.shutdown(); server.server_close(); thread.join() 239 assert len(requests) == 6, requests 240 report['registry_requests'] = list(requests) 241 original_lock = project_lock.read_bytes() 242 (root / 'initial.lock').write_bytes(original_lock) 243 # Actual closed socket: subsequent operations cannot depend on the registry. 244 project('resolve'); project('fetch'); project('bind'); project('check') 245 assert not any(p.is_file() for p in (application / 'aliases').rglob('*')) 246 shutil.copyfile(ROOT / demo / 'main.tri', consumer / 'main.tri') 247 report['sources'][demo + '/main.tri'] = hashlib.sha256((consumer / 'main.tri').read_bytes()).hexdigest() 248 assert sorted(p.name for p in consumer.glob('*.tri')) == ['fileio.tri', 'foundation.tri', 'inspection.tri', 'main.tri'] 249 print('Compiling consumer with the registry stopped', flush=True) 250 bundle = root / 'inspector.arboricx' 251 run('arboricx', 'compile', '--file', consumer / 'main.tri', '--store', application, '--output', bundle, 252 *options, seconds=930) 253 inspector = imported(bundle, application) 254 report['inspector_manifest'] = inspector 255 def inspect(path, good=True, flags=None): 256 if flags is None: 257 flags = [] 258 return run('arboricx', 'run', '--store', application, '--module', inspector, '--name', 'main', 259 '--arg', path, '--io', '--no-result', *flags, success=good, seconds=180) 260 inputs = cases(root / 'cases') 261 # A real executable IO export would abort if the inspector ran it. 262 shutil.copyfile(consumer / 'fileio.tri', root / 'cases/fileio.tri') 263 payload = root / 'cases/payload.tri' 264 payload.write_text('!import fileio as IO\n!export main\nmain = IO.io (IO.abortIO "INSPECTED EXPORT WAS EXECUTED")\n') 265 inert = root / 'cases/inert-io.arboricx' 266 run('arboricx', 'compile', '--file', payload, '--store', application, '--output', inert, *options, seconds=930) 267 inputs.append(('inert-io', inert, True)) 268 inputs.extend((name, producer / (name + '.arboricx'), True) for name in report['releases']) 269 inputs.append(('inspector-self', bundle, True)) 270 for label, path, good in inputs: 271 print('Inspecting:', label, flush=True) 272 expected = command(oracle, [path], success=good, rts=False).decode('utf-8') 273 actual = inspect(path, good) 274 if good: 275 assert actual == expected, (label, actual, expected) 276 assert actual.splitlines()[1] == 'bundle-sha256\t' + hashlib.sha256(path.read_bytes()).hexdigest() 277 else: 278 assert 'arboricx.inspect.v1' not in actual, (label, actual) 279 report['cases'].append(dict(label=label, valid=good, report=actual)); record() 280 target = root / 'cases/id.arboricx' 281 inspect(target, False, ['--allow-read', target]) 282 inspect(target, False, ['--unsafe-io']) 283 denied = root / 'cases/os-denied.arboricx' 284 shutil.copyfile(target, denied) 285 denied.chmod(0) 286 try: 287 inspect(denied, False) 288 finally: 289 denied.chmod(0o600) 290 report['os_permission_denial_checked'] = True 291 report['v1_grants_rejected'] = True 292 oversized = root / 'cases/over-budget.arboricx' 293 data = target.read_bytes() 294 oversized.write_bytes(data + b'\0' * (1048577-len(data))) 295 assert 'arboricx.inspect.v1' not in inspect(oversized, False) 296 report['explicit_byte_budget_checked'] = True 297 298 299 inspect(root / 'cases/missing.arboricx', False) 300 # Deliberate one-dependency update: add a public constant to the foundation 301 # interface, preserving the three operations used by the authored consumer. 302 # This is a local acceptance release, never staged in the site's public index. 303 updated_source = workspace / 'base_update_release.tri' 304 updated_source.write_text((ROOT / 'lib/packages/base.tri').read_text().replace( 305 '!export showNumber\n', '!export showNumber\n!export revision\n') + '\nrevision = 1\n') 306 report['update_source_sha256'] = hashlib.sha256(updated_source.read_bytes()).hexdigest() 307 updated_bundle = producer / 'base-update.arboricx' 308 run('arboricx', 'compile', '--file', updated_source, '--store', producer / 'store', 309 '--output', updated_bundle, *options, seconds=930) 310 next_version = 'project-update-test' if args.version != 'project-update-test' else 'project-update-test-2' 311 update_release = run('package', 'publish', '--root', registry, '--name', 'tricu-base', 312 '--version', next_version, '--file', updated_bundle).splitlines()[1].split('\t') 313 _, _, new_manifest, new_digest = update_release 314 assert new_manifest != report['releases']['tricu-base']['manifest'] 315 routes['/_arboricx/package/tricu-base/' + next_version] = registry / 'packages/tricu-base' / next_version 316 routes['/_arboricx/bundle/hash/' + new_digest] = registry / 'objects' / new_digest[:3] / (new_digest + '.arboricx') 317 project_file.write_text(project_file.read_text().replace('tricu-base\t' + args.version + '\n', 'tricu-base\t' + next_version + '\n')) 318 project('check', success=False) 319 server = ThreadingHTTPServer(('127.0.0.1', port), Handler) 320 thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() 321 try: 322 project('resolve', success=False) 323 assert len(requests) == 6, requests 324 diff = project('update', '--dependency', 'foundation') 325 assert 'Updated foundation' in diff and 'Updated fileio' not in diff and 'Updated inspection' not in diff 326 report['update_diff'] = diff 327 project('check', success=False) 328 project('bind', success=False) # no implicit fetch 329 project('fetch') 330 finally: 331 server.shutdown(); server.server_close(); thread.join() 332 assert len(requests) == 8, requests 333 def lock_rows(data): 334 return dict((line.split('\t')[0], line) for line in data.decode().splitlines()[1:]) 335 before, after = lock_rows(original_lock), lock_rows(project_lock.read_bytes()) 336 assert before['fileio'] == after['fileio'] and before['inspection'] == after['inspection'] 337 assert before['foundation'] != after['foundation'] 338 project('fetch'); project('bind'); project('check') 339 updated_inspector_bundle = root / 'inspector-updated.arboricx' 340 run('arboricx', 'compile', '--file', consumer / 'main.tri', '--store', application, 341 '--output', updated_inspector_bundle, *options, seconds=930) 342 updated_inspector = imported(updated_inspector_bundle, application) 343 report['updated_inspector_manifest'] = updated_inspector 344 report['consumer_identity_unchanged'] = updated_inspector == inspector 345 assert report['consumer_identity_unchanged'], 'pinned inspector fixture changed after the additive foundation update' 346 inspector = updated_inspector 347 target = root / 'cases/id.arboricx' 348 assert inspect(target) == command(oracle, [target], rts=False).decode('utf-8') 349 # Explain and compare the retained locks after the registry has closed. 350 report['project_explanation'] = project('explain') 351 report['project_diff'] = run('project', 'diff', '--before', root / 'initial.lock', 352 '--after', project_lock, '--store', application) 353 assert 'export revision: added' in report['project_diff'] 354 foundation_diff = report['project_diff'].split('Binding foundation\n')[1].split('Binding inspection\n')[0] 355 assert foundation_diff.count('tree identical') == 3 and 'tree different' not in foundation_diff 356 assert 'local module: verified' in report['project_explanation'] 357 report['updated_registry_requests'] = list(requests) 358 report['complete'] = True; report['seconds'] = time.monotonic() - started; record() 359 if args.stage_site: 360 stage_site(root, args.stage_site.resolve(), report) 361 report['staged_site'] = str(args.stage_site.resolve()); record() 362 print('PASS project dependencies, clean fetch, offline inspector, isolated dependency update, Haskell parity and negative controls') 363 print('Inspector:', inspector, '\nStore:', application) 364 365 366 if __name__ == '__main__': 367 main()