package_registry.py (13765B)
1 #!/usr/bin/env python3 2 """Haskell-only package protocol controls; optional native server + offline build. 3 4 Default: small existing bundle, controlled HTTP peer, no source compilation. 5 --native-server: a compiled module exporting main root port, as documented in 6 validation.md. Runs the actual Arboricx routes and compiles a tiny consumer. 7 """ 8 import argparse 9 import hashlib 10 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer 11 import json 12 from pathlib import Path 13 import re 14 import socket 15 import subprocess 16 import tempfile 17 import threading 18 import time 19 import urllib.error 20 import urllib.request 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def main(): 26 p = argparse.ArgumentParser(description=__doc__) 27 p.add_argument('tricu', type=Path) 28 group = p.add_mutually_exclusive_group() 29 group.add_argument('--native-server', type=Path) 30 group.add_argument('--prepare-server', type=Path, help='Compile current native server into a new retained directory') 31 args = p.parse_args() 32 tricu = args.tricu.resolve() 33 if args.prepare_server: 34 prepared = args.prepare_server.resolve() 35 prepared.mkdir(parents=True, exist_ok=False) 36 workspace = prepared / 'workspace' 37 workspace.mkdir() 38 metadata = json.loads((ROOT / 'bootstrap/stable-compiler.json').read_text()) 39 sources = {row['alias']: row['path'] for row in metadata['sources']} 40 sources.update(http='lib/http.tri', socket='lib/socket.tri', arboricx_server='lib/arboricx/server.tri') 41 hashes = {} 42 for alias, path in sources.items(): 43 data = (ROOT / path).read_bytes() 44 (workspace / (alias + '.tri')).write_bytes(data) 45 hashes[path] = hashlib.sha256(data).hexdigest() 46 (workspace / 'main.tri').write_text('!import io *\n!import arboricx_server as S\n!export main\n' 47 'main root port = io (S.arboricxServer root "127.0.0.1" port)\n') 48 args.native_server = prepared / 'server.arboricx' 49 command = [str(tricu), 'arboricx', 'compile', '--file', str(workspace / 'main.tri'), 50 '--store', str(prepared / 'store'), '--output', str(args.native_server), 51 '--compile-timeout', '900', '+RTS', '-N1', '-M8192m', '-RTS'] 52 reply = subprocess.run(command, capture_output=True, timeout=930) 53 (prepared / 'build.log').write_bytes(reply.stdout + reply.stderr) 54 (prepared / 'preparation.json').write_text(json.dumps(dict(command=command, 55 compiler_sha256=metadata['sha256'], sources=hashes, exit_code=reply.returncode), indent=2)+'\n') 56 assert reply.returncode == 0, (reply.stdout + reply.stderr).decode() 57 with tempfile.TemporaryDirectory(prefix='tricu-package-') as temp: 58 root = Path(temp) 59 registry = root / 'registry' 60 store = root / 'store' 61 config, lock = root / 'registries', root / 'library.lock' 62 requests = [] 63 overrides = {} 64 65 def run(*words, success=True, seconds=60): 66 reply = subprocess.run([str(tricu), *map(str, words), '+RTS', '-N1', '-M2048m', '-RTS'], 67 capture_output=True, text=True, timeout=seconds) 68 assert (reply.returncode == 0) == success, (words, reply.stdout, reply.stderr) 69 return reply.stdout 70 71 def package(*words, **kw): 72 return run('package', *words, **kw) 73 74 library = ROOT / 'test/fixtures/id.arboricx' 75 if args.native_server: 76 source = root / 'library.tri' 77 source.write_text('!export id\n!export marker\nid x = x\nmarker = 7\n') 78 library = root / 'library.arboricx' 79 run('arboricx', 'compile', '--file', source, '--store', root / 'producer-store', 80 '--output', library, seconds=120) 81 record = package('publish', '--root', registry, '--name', 'identity', '--version', '1.0', 82 '--file', library) 83 _, row = record.splitlines() 84 name, version, manifest, bundle = row.split('\t') 85 index_path = '/_arboricx/package/identity/1.0' 86 bundle_path = '/_arboricx/bundle/hash/' + bundle 87 bundle_file = registry / 'objects' / bundle[:3] / (bundle + '.arboricx') 88 assert hashlib.sha256(bundle_file.read_bytes()).hexdigest() == bundle 89 90 class Handler(BaseHTTPRequestHandler): 91 def do_GET(self): 92 requests.append(self.path) 93 if self.path in overrides: 94 status, body = overrides[self.path] 95 elif self.path == index_path: 96 status, body = 200, record.encode() 97 elif self.path == bundle_path: 98 status, body = 200, bundle_file.read_bytes() 99 else: 100 status, body = 404, b'not found' 101 self.send_response(status) 102 self.send_header('Content-Length', str(len(body))) 103 self.send_header('Location', '/must-not-follow') 104 self.end_headers() 105 self.wfile.write(body) 106 107 def log_message(self, *_): 108 pass 109 110 server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) 111 thread = threading.Thread(target=server.serve_forever, daemon=True) 112 thread.start() 113 url = 'http://127.0.0.1:' + str(server.server_port) 114 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 115 resolve = ['resolve', '--registries', config, '--registry', 'home', '--name', name, 116 '--version', version, '--lock', lock] 117 fetch = ['fetch', '--registries', config, '--lock', lock, '--store', store] 118 try: 119 package(*resolve) 120 original_lock = lock.read_bytes() 121 package('resolve', '--registries', config, '--registry', 'home', '--name', name, 122 '--version', version, '--lock', config, success=False) 123 assert requests == [index_path] 124 # Exact same version cannot be changed, even with --update. 125 overrides[index_path] = (200, record.replace(manifest, 'a' * 64).encode()) 126 package(*resolve, '--update', success=False) 127 assert lock.read_bytes() == original_lock 128 overrides[index_path] = (302, b'redirect') 129 package(*resolve, success=False) 130 assert '/must-not-follow' not in requests 131 overrides[index_path] = (200, b'x' * 4097) 132 package(*resolve, success=False) 133 overrides[index_path] = (200, record.replace('identity\t', 'other\t').encode()) 134 package(*resolve, success=False) 135 overrides.clear() 136 # A different version is an explicit selection change, not mutation. 137 selection = root / 'selection.lock' 138 selection.write_bytes(original_lock) 139 overrides['/_arboricx/package/identity/2.0'] = (200, record.replace('identity\t1.0\t', 'identity\t2.0\t').encode()) 140 change = ['resolve', '--registries', config, '--registry', 'home', '--name', name, 141 '--version', '2.0', '--lock', selection] 142 package(*change, success=False) 143 assert selection.read_bytes() == original_lock 144 package(*change, '--update') 145 assert b'identity\t2.0\t' in selection.read_bytes() 146 overrides.clear() 147 # Fetch uses only transport pins, not a fresh index result. 148 requests.clear() 149 overrides[bundle_path] = (200, b'corrupt') 150 package(*fetch, success=False) 151 assert not (store / 'objects' / manifest[:3] / manifest).exists() 152 overrides.clear() 153 # A valid bundle at the selected pin must still match the module pin. 154 lock.write_bytes(original_lock.replace(manifest.encode(), b'a' * 64)) 155 package(*fetch, success=False) 156 assert not (store / 'objects').exists() 157 lock.write_bytes(original_lock) 158 package(*fetch) 159 assert requests == [bundle_path] * 3 160 requests.clear() 161 package(*fetch) 162 assert requests == [] 163 # Verification includes the exported trees, not merely the manifest. 164 manifest_file = store / 'objects' / manifest[:3] / manifest 165 tree_pin = manifest_file.read_text().splitlines()[1].split('\t')[3] 166 tree_file = store / 'objects' / tree_pin[:3] / tree_pin 167 tree_bytes = tree_file.read_bytes() 168 tree_file.write_bytes(b'corrupt tree') 169 package(*fetch, success=False) 170 tree_file.unlink() 171 package(*fetch, success=False) 172 tree_file.write_bytes(tree_bytes) 173 assert requests == [] 174 # Explicit trust and URL binding, including when content is cached. 175 config.write_text('arboricx.registries.v1\nhome\t' + url + '\nhome\t' + url + '\n') 176 package(*fetch, success=False) 177 config.write_text('arboricx.registries.v1\n') 178 package(*fetch, success=False) 179 config.write_text('arboricx.registries.v1\nhome\thttps://localhost\n') 180 package(*fetch, success=False) 181 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 182 package('bind', '--lock', lock, '--store', store, '--output', root / 'identity.tri') 183 assert (root / 'identity.tri').read_text() == '!export * from ' + manifest + '\n' 184 assert requests == [] 185 package('bind', '--lock', lock, '--store', store, '--output', lock, success=False) 186 assert lock.read_bytes() == original_lock 187 finally: 188 server.shutdown() 189 server.server_close() 190 thread.join() 191 192 # Cached verification/fetch and binding work with the peer stopped. 193 package(*fetch) 194 package('bind', '--lock', lock, '--store', store, '--output', root / 'identity.tri') 195 # Corrupt local content is not accepted or silently repaired. 196 object_file = store / 'objects' / manifest[:3] / manifest 197 original = object_file.read_bytes() 198 object_file.write_bytes(b'corrupt') 199 package(*fetch, success=False) 200 object_file.write_bytes(original) 201 202 if args.native_server: 203 from http_headers import number 204 with socket.socket() as sock: 205 sock.bind(('127.0.0.1', 0)) 206 port = sock.getsockname()[1] 207 port_file = root / 'port.tree' 208 port_file.write_bytes(number(port)) 209 runtime = root / 'server-store' 210 imported = run('arboricx', 'import', '--file', args.native_server.resolve(), '--store', runtime) 211 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported)[1] 212 log = open(root / 'server.log', 'w+') 213 proc = subprocess.Popen([str(tricu), 'arboricx', 'run', '--module', pin, '--name', 'main', 214 '--store', str(runtime), '--arg', str(registry), '--arg-tree-file', str(port_file), 215 '--io', '--allow-read', str(registry), '+RTS', '-N1', '-M2048m', '-RTS'], stdout=log, stderr=log) 216 url = 'http://127.0.0.1:' + str(port) 217 # Ignore environmental proxies for our local protocol probes. 218 opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) 219 try: 220 for _ in range(100): 221 if proc.poll() is not None: 222 log.seek(0) 223 raise AssertionError(log.read()) 224 try: 225 with opener.open(url + '/_arboricx/health', timeout=2) as response: 226 assert response.read() == b'OK\n' 227 break 228 except (OSError, urllib.error.URLError): 229 time.sleep(.1) 230 else: 231 raise AssertionError('native server startup timeout') 232 for suffix in ['../x', 'identity/../x', 'identity/1.0/extra', 'identity/%31', 'a'*129 + '/1', 'identity/' + '1'*129]: 233 try: 234 opener.open(url + '/_arboricx/package/' + suffix, timeout=10) 235 raise AssertionError('unsafe index path accepted: ' + suffix) 236 except urllib.error.HTTPError as e: 237 assert e.code == 400, (suffix, e.code) 238 native_store = root / 'native-store' 239 native_lock = root / 'native.lock' 240 config.write_text('arboricx.registries.v1\nhome\t' + url + '\n') 241 package('resolve', '--registries', config, '--registry', 'home', '--name', name, 242 '--version', version, '--lock', native_lock) 243 package('fetch', '--registries', config, '--lock', native_lock, '--store', native_store) 244 finally: 245 proc.terminate() 246 try: 247 proc.wait(timeout=5) 248 except subprocess.TimeoutExpired: 249 proc.kill() 250 proc.wait() 251 log.close() 252 # The actual compiler consumes a hash-pinned workspace binding offline. 253 package('bind', '--lock', native_lock, '--store', native_store, '--output', root / 'identity.tri') 254 (root / 'app.tri').write_text('!import identity as I\n!export main\nmain = I.id 42\n') 255 value = run('eval', root / 'app.tri', '--store', native_store, '--format', 'number', seconds=120) 256 assert value.strip() == '42', value 257 print('PASS: exact registry resolution, verified fetching, immutable locks, offline binding' + 258 ('; native server and offline compilation' if args.native_server else '')) 259 260 261 if __name__ == '__main__': 262 main()