package_registry.py (13960B)
1 #!/usr/bin/env python3 2 """Haskell-only package protocol controls; optional native server + offline build. 3 4 Default: small existing bundle, controlled HTTP peer, no source compilation. 5 --native-server: a compiled module exporting main root port, as documented in 6 validation.md. Runs the actual Arboricx routes and compiles a tiny consumer. 7 """ 8 import argparse 9 import hashlib 10 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer 11 import json 12 from pathlib import Path 13 import re 14 import socket 15 import subprocess 16 import tempfile 17 import threading 18 import time 19 import urllib.error 20 import urllib.request 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def main(): 26 p = argparse.ArgumentParser(description=__doc__) 27 p.add_argument('tricu', type=Path) 28 group = p.add_mutually_exclusive_group() 29 group.add_argument('--native-server', type=Path, help='IO server bundle exporting main root port') 30 group.add_argument('--prepare-server', type=Path, help='Compile current native server into a new retained directory') 31 args = p.parse_args() 32 tricu = args.tricu.resolve() 33 if args.prepare_server: 34 prepared = args.prepare_server.resolve() 35 prepared.mkdir(parents=True, exist_ok=False) 36 workspace = prepared / 'workspace' 37 workspace.mkdir() 38 metadata = json.loads((ROOT / 'bootstrap/stable-compiler.json').read_text()) 39 # Freeze workspace inputs; the native compiler resolves the actual closure. 40 sources = dict(line.split('\t') for line in 41 (ROOT / 'tricu.workspace').read_text().splitlines()[1:]) 42 hashes = {} 43 for alias, path in sources.items(): 44 data = (ROOT / path).read_bytes() 45 (workspace / (alias + '.tri')).write_bytes(data) 46 hashes[path] = hashlib.sha256(data).hexdigest() 47 (workspace / 'main.tri').write_text( 48 '!import io as IO\n!import arboricx_streaming as Server\n' 49 '!import arboricx_handlers as Registry\n!export main\n' 50 'main root port = IO.io (Server.serve root 1048576 Registry.stdoutLog ' 51 '(IO.ipv4 [127 0 0 1] port) 16 (_ : IO.pure t))\n') 52 args.native_server = prepared / 'server.arboricx' 53 command = [str(tricu), 'arboricx', 'compile', '--file', str(workspace / 'main.tri'), 54 '--store', str(prepared / 'store'), '--output', str(args.native_server), 55 '--compile-timeout', '900', '+RTS', '-N1', '-M8192m', '-RTS'] 56 reply = subprocess.run(command, capture_output=True, timeout=930) 57 (prepared / 'build.log').write_bytes(reply.stdout + reply.stderr) 58 (prepared / 'preparation.json').write_text(json.dumps(dict(command=command, 59 compiler_sha256=metadata['sha256'], sources=hashes, exit_code=reply.returncode), indent=2)+'\n') 60 assert reply.returncode == 0, (reply.stdout + reply.stderr).decode() 61 with tempfile.TemporaryDirectory(prefix='tricu-package-') as temp: 62 root = Path(temp) 63 registry = root / 'registry' 64 store = root / 'store' 65 config, lock = root / 'registries', root / 'library.lock' 66 requests = [] 67 overrides = {} 68 69 def run(*words, success=True, seconds=60): 70 reply = subprocess.run([str(tricu), *map(str, words), '+RTS', '-N1', '-M2048m', '-RTS'], 71 capture_output=True, text=True, timeout=seconds) 72 assert (reply.returncode == 0) == success, (words, reply.stdout, reply.stderr) 73 return reply.stdout 74 75 def package(*words, **kw): 76 return run('package', *words, **kw) 77 78 library = ROOT / 'test/fixtures/id.arboricx' 79 if args.native_server: 80 source = root / 'library.tri' 81 source.write_text('!export id\n!export marker\nid x = x\nmarker = 7\n') 82 library = root / 'library.arboricx' 83 run('arboricx', 'compile', '--file', source, '--store', root / 'producer-store', 84 '--output', library, seconds=120) 85 record = package('publish', '--root', registry, '--name', 'identity', '--version', '1.0', 86 '--file', library) 87 _, row = record.splitlines() 88 name, version, manifest, bundle = row.split('\t') 89 index_path = '/_arboricx/package/identity/1.0' 90 bundle_path = '/_arboricx/bundle/hash/' + bundle 91 bundle_file = registry / 'objects' / bundle[:3] / (bundle + '.arboricx') 92 assert hashlib.sha256(bundle_file.read_bytes()).hexdigest() == bundle 93 94 class Handler(BaseHTTPRequestHandler): 95 def do_GET(self): 96 requests.append(self.path) 97 if self.path in overrides: 98 status, body = overrides[self.path] 99 elif self.path == index_path: 100 status, body = 200, record.encode() 101 elif self.path == bundle_path: 102 status, body = 200, bundle_file.read_bytes() 103 else: 104 status, body = 404, b'not found' 105 self.send_response(status) 106 self.send_header('Content-Length', str(len(body))) 107 self.send_header('Location', '/must-not-follow') 108 self.end_headers() 109 self.wfile.write(body) 110 111 def log_message(self, *_): 112 pass 113 114 server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) 115 thread = threading.Thread(target=server.serve_forever, daemon=True) 116 thread.start() 117 url = 'http://127.0.0.1:' + str(server.server_port) 118 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 119 resolve = ['resolve', '--registries', config, '--registry', 'home', '--name', name, 120 '--version', version, '--lock', lock] 121 fetch = ['fetch', '--registries', config, '--lock', lock, '--store', store] 122 try: 123 package(*resolve) 124 original_lock = lock.read_bytes() 125 package('resolve', '--registries', config, '--registry', 'home', '--name', name, 126 '--version', version, '--lock', config, success=False) 127 assert requests == [index_path] 128 # Exact same version cannot be changed, even with --update. 129 overrides[index_path] = (200, record.replace(manifest, 'a' * 64).encode()) 130 package(*resolve, '--update', success=False) 131 assert lock.read_bytes() == original_lock 132 overrides[index_path] = (302, b'redirect') 133 package(*resolve, success=False) 134 assert '/must-not-follow' not in requests 135 overrides[index_path] = (200, b'x' * 4097) 136 package(*resolve, success=False) 137 overrides[index_path] = (200, record.replace('identity\t', 'other\t').encode()) 138 package(*resolve, success=False) 139 overrides.clear() 140 # A different version is an explicit selection change, not mutation. 141 selection = root / 'selection.lock' 142 selection.write_bytes(original_lock) 143 overrides['/_arboricx/package/identity/2.0'] = (200, record.replace('identity\t1.0\t', 'identity\t2.0\t').encode()) 144 change = ['resolve', '--registries', config, '--registry', 'home', '--name', name, 145 '--version', '2.0', '--lock', selection] 146 package(*change, success=False) 147 assert selection.read_bytes() == original_lock 148 package(*change, '--update') 149 assert b'identity\t2.0\t' in selection.read_bytes() 150 overrides.clear() 151 # Fetch uses only transport pins, not a fresh index result. 152 requests.clear() 153 overrides[bundle_path] = (200, b'corrupt') 154 package(*fetch, success=False) 155 assert not (store / 'objects' / manifest[:3] / manifest).exists() 156 overrides.clear() 157 # A valid bundle at the selected pin must still match the module pin. 158 lock.write_bytes(original_lock.replace(manifest.encode(), b'a' * 64)) 159 package(*fetch, success=False) 160 assert not (store / 'objects').exists() 161 lock.write_bytes(original_lock) 162 package(*fetch) 163 assert requests == [bundle_path] * 3 164 requests.clear() 165 package(*fetch) 166 assert requests == [] 167 # Verification includes the exported trees, not merely the manifest. 168 manifest_file = store / 'objects' / manifest[:3] / manifest 169 tree_pin = manifest_file.read_text().splitlines()[1].split('\t')[3] 170 tree_file = store / 'objects' / tree_pin[:3] / tree_pin 171 tree_bytes = tree_file.read_bytes() 172 tree_file.write_bytes(b'corrupt tree') 173 package(*fetch, success=False) 174 tree_file.unlink() 175 package(*fetch, success=False) 176 tree_file.write_bytes(tree_bytes) 177 assert requests == [] 178 # Explicit trust and URL binding, including when content is cached. 179 config.write_text('arboricx.registries.v1\nhome\t' + url + '\nhome\t' + url + '\n') 180 package(*fetch, success=False) 181 config.write_text('arboricx.registries.v1\n') 182 package(*fetch, success=False) 183 config.write_text('arboricx.registries.v1\nhome\thttps://localhost\n') 184 package(*fetch, success=False) 185 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 186 package('bind', '--lock', lock, '--store', store, '--output', root / 'identity.tri') 187 assert (root / 'identity.tri').read_text() == '!export * from ' + manifest + '\n' 188 assert requests == [] 189 package('bind', '--lock', lock, '--store', store, '--output', lock, success=False) 190 assert lock.read_bytes() == original_lock 191 finally: 192 server.shutdown() 193 server.server_close() 194 thread.join() 195 196 # Cached verification/fetch and binding work with the peer stopped. 197 package(*fetch) 198 package('bind', '--lock', lock, '--store', store, '--output', root / 'identity.tri') 199 # Corrupt local content is not accepted or silently repaired. 200 object_file = store / 'objects' / manifest[:3] / manifest 201 original = object_file.read_bytes() 202 object_file.write_bytes(b'corrupt') 203 package(*fetch, success=False) 204 object_file.write_bytes(original) 205 206 if args.native_server: 207 from http_headers import number 208 with socket.socket() as sock: 209 sock.bind(('127.0.0.1', 0)) 210 port = sock.getsockname()[1] 211 port_file = root / 'port.tree' 212 port_file.write_bytes(number(port)) 213 runtime = root / 'server-store' 214 imported = run('arboricx', 'import', '--file', args.native_server.resolve(), '--store', runtime) 215 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported)[1] 216 log = open(root / 'server.log', 'w+') 217 proc = subprocess.Popen([str(tricu), 'arboricx', 'run', '--module', pin, '--name', 'main', 218 '--store', str(runtime), '--arg', str(registry), '--arg-tree-file', str(port_file), 219 '--io', '+RTS', '-N1', '-M2048m', '-RTS'], stdout=log, stderr=log) 220 url = 'http://127.0.0.1:' + str(port) 221 # Ignore environmental proxies for our local protocol probes. 222 opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) 223 try: 224 for _ in range(100): 225 if proc.poll() is not None: 226 log.seek(0) 227 raise AssertionError(log.read()) 228 try: 229 with opener.open(url + '/_arboricx/health', timeout=2) as response: 230 assert response.read() == b'OK\n' 231 break 232 except (OSError, urllib.error.URLError): 233 time.sleep(.1) 234 else: 235 raise AssertionError('native server startup timeout') 236 for suffix in ['../x', 'identity/../x', 'identity/1.0/extra', 'identity/%31', 'a'*129 + '/1', 'identity/' + '1'*129]: 237 try: 238 opener.open(url + '/_arboricx/package/' + suffix, timeout=10) 239 raise AssertionError('unsafe index path accepted: ' + suffix) 240 except urllib.error.HTTPError as e: 241 assert e.code == 400, (suffix, e.code) 242 native_store = root / 'native-store' 243 native_lock = root / 'native.lock' 244 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 245 package('resolve', '--registries', config, '--registry', 'home', '--name', name, 246 '--version', version, '--lock', native_lock) 247 package('fetch', '--registries', config, '--lock', native_lock, '--store', native_store) 248 finally: 249 proc.terminate() 250 try: 251 proc.wait(timeout=5) 252 except subprocess.TimeoutExpired: 253 proc.kill() 254 proc.wait() 255 log.close() 256 # The actual compiler consumes a hash-pinned workspace binding offline. 257 package('bind', '--lock', native_lock, '--store', native_store, '--output', root / 'identity.tri') 258 (root / 'app.tri').write_text('!import identity as I\n!export main\nmain = I.id 42\n') 259 value = run('eval', root / 'app.tri', '--store', native_store, '--format', 'number', seconds=120) 260 assert value.strip() == '42', value 261 print('PASS: exact registry resolution, verified fetching, immutable locks, offline binding' + 262 ('; native server and offline compilation' if args.native_server else '')) 263 264 265 if __name__ == '__main__': 266 main()