release.py (14972B)
1 #!/usr/bin/env python3 2 """Fast filesystem release/seed checks; no application compilation or evaluation.""" 3 import argparse 4 import hashlib 5 import importlib.util 6 import json 7 import re 8 import struct 9 from pathlib import Path 10 import subprocess 11 import tempfile 12 13 14 def main(): 15 parser = argparse.ArgumentParser(description=__doc__) 16 parser.add_argument('--site', type=Path, required=True) 17 parser.add_argument('--deps', type=Path, required=True) 18 parser.add_argument('--seeder', type=Path, required=True) 19 args = parser.parse_args() 20 # Every relative runtime/Worker/metadata URL stays in the same release tree. 21 assets = json.loads((args.site / 'assets.json').read_text()) 22 encoded = json.dumps(assets['files'], sort_keys=True, separators=(',', ':')).encode() 23 assert assets['version'] == hashlib.sha256(b'arboricx.site-assets.v1\0' + encoded).hexdigest() 24 assert assets['base'] == 'assets/' + assets['version'] + '/' 25 html = (args.site / 'index.html').read_text() 26 for entry in ['dist/app.js', 'style.css', 'images/favicon.png', 'images/icon.svg', 'images/apple-touch-icon.png']: 27 assert '"./' + assets['base'] + entry + '"' in html 28 assert 'https://arbori.cx/' + assets['base'] + 'images/preview.png' in html 29 for name, size in [('preview.png', (1200, 630)), ('favicon.png', (32, 32)), ('apple-touch-icon.png', (180, 180))]: 30 png = (args.site / assets['base'] / 'images' / name).read_bytes() 31 assert png[:8] == b'\x89PNG\r\n\x1a\n' 32 assert (int.from_bytes(png[16:20], 'big'), int.from_bytes(png[20:24], 'big')) == size 33 assert not (args.site / 'dist').exists() 34 llms = (args.site / 'llms.txt').read_text() 35 for path in re.findall(r'\]\((\./[^)]+)\)', llms): 36 assert (args.site / path).is_file(), path 37 assert assets['base'] in path 38 assert 'href="./llms.txt"' in html 39 support = args.site / 'nix-support' 40 args.site = args.site / assets['base'] 41 assert sorted(p.relative_to(args.site).as_posix() for p in args.site.rglob('*') if p.is_file()) == [r['path'] for r in assets['files']] 42 for row in assets['files']: 43 data = (args.site / row['path']).read_bytes() 44 assert len(data) == row['size'] 45 assert hashlib.sha256(data).hexdigest() == row['sha256'] 46 pack = args.deps / 'share/arboricx-site-deps' 47 manifest = json.loads((pack / 'dependencies.json').read_text()) 48 rows = manifest['bundles'] 49 catalogue = json.loads((args.site / 'catalogue.json').read_text()) 50 compiler = json.loads((args.site / 'compiler.json').read_text()) 51 index = json.loads((args.site / 'packages.json').read_text()) 52 environments = json.loads((args.site / 'environments.json').read_text()) 53 examples = json.loads((args.site / 'reference/examples.json').read_text()) 54 quickstart = (args.site / 'reference/getting-started.md').read_text() 55 for code in re.findall(r'```tricu\n(.*?)```', quickstart, re.S): 56 assert any(e['source'].strip() == code.strip() for e in examples) 57 libraries = json.loads((args.site / 'reference/libraries.json').read_text())['releases'] 58 published = [r for r in index['releases'] if r['name'].startswith('tricu-lib-')] 59 assert {(r['name'], r['version']) for r in libraries} == {(r['name'], r['version']) for r in published} 60 # Independent reader: public names are the bundle's export table, not source 61 # declaration guesses (which would miss inherited/reexported names). 62 def public_names(data): 63 sections = struct.unpack_from('>I', data, 12)[0] 64 manifest = None 65 for i in range(sections): 66 at = 32 + i * 32 67 tag = struct.unpack_from('>I', data, at)[0] 68 offset, size = struct.unpack_from('>QQ', data, at + 12) 69 if tag == 1: manifest = data[offset:offset + size] 70 assert manifest is not None 71 offset = 12 72 def word(): 73 nonlocal offset 74 value = struct.unpack_from('>I', manifest, offset)[0]; offset += 4 75 return value 76 def text(): 77 nonlocal offset 78 size = word(); value = manifest[offset:offset + size]; offset += size 79 return value.decode() 80 for _ in range(9): text() 81 for _ in range(word()): text() 82 offset += 1 83 for _ in range(word()): word(); text() 84 result = [] 85 for _ in range(word()): result.append(text()); word(); text(); text() 86 return result 87 for row in libraries: 88 release = next(r for r in published if (r['name'],r['version']) == (row['name'],row['version'])) 89 assert row['manifest'] == release['manifest'] and row['hash'] == release['hash'] 90 data = (pack / 'objects' / row['hash'][:3] / (row['hash'] + '.arboricx')).read_bytes() 91 assert row['names'] == public_names(data) 92 reference = (args.site / 'reference' / row['reference']).read_text() 93 assert '!import ' + row['manifest'] + ' as ' + row['alias'] in reference 94 assert all(name in reference for name in row['names']) 95 # All relative static Markdown links resolve inside the released reference tree. 96 for path in (args.site / 'reference').rglob('*.md'): 97 for href in re.findall(r'\]\(([^ )]+)(?: +[^)]*)?\)', path.read_text()): 98 if ':' in href or href.startswith('#'): continue 99 destination = (path.parent / href.split('#')[0]).resolve() 100 assert destination.is_file(), (path, href) 101 102 count = len(rows) + len(manifest['releases']) 103 assert {row['hash'] for row in rows} == {entry['hash'] for entry in [*catalogue, compiler, *index['releases'], *[p for p in environments['profiles'].values() if p]]} 104 for row in rows: 105 expected = sorted([entry['id'] for entry in catalogue if entry['hash'] == row['hash']] 106 + (['compiler'] if compiler['hash'] == row['hash'] else []) 107 + ['release:' + r['name'] + '@' + r['version'] for r in index['releases'] if r['hash'] == row['hash']] 108 + ['environment:' + n for n, p in environments['profiles'].items() if p and p['hash'] == row['hash']]) 109 assert row['uses'] == expected 110 assert not list(args.site.rglob('*.arboricx')) 111 assert not (args.site / 'bundles').exists() 112 assert all('path' not in entry for entry in [*catalogue, compiler]) 113 # Production uses only native-runtime Worker drivers; JS is test-only. 114 assert not list(args.site.rglob('conformance')) 115 for retired in ['worker.js', 'compiler-worker.js']: 116 assert not (args.site / 'dist' / retired).exists() 117 entries = {'dist/app.js', 'dist/wasm-worker.js', 'dist/local-module-worker.js', 118 'dist/zig-helpers.js', 'runtime/haskell/client.mjs', 119 'runtime/haskell/worker.mjs', 'runtime/zig/site-session.mjs'} 120 assert {p.relative_to(args.site).as_posix() for p in args.site.rglob('*') 121 if p.suffix in ['.js', '.mjs']} == entries 122 graphs = json.loads((support / 'bundle-graphs.json').read_text()) 123 assert set(graphs) == entries | {'dist/probe-api.js'} 124 for entry, graph in graphs.items(): 125 assert set(graph['outputs']) == {'bundled/' + entry} 126 assert not graph['outputs']['bundled/' + entry]['imports'] 127 assert not any('/conformance/' in p for p in graph['inputs']) 128 file = support / 'probe-api.js' if entry == 'dist/probe-api.js' else args.site / entry 129 assert file.stat().st_size == graph['outputs']['bundled/' + entry]['bytes'] 130 assert 'runtime/haskell/shim/index.js' in graphs['runtime/haskell/worker.mjs']['inputs'] 131 assert not any('runtime/' in p for p in graphs['dist/app.js']['inputs']) 132 assert not (args.site / 'dist/probe-api.js').exists() 133 # Preserve the no-static-bundle-fallback control in each bundled download owner. 134 for script in ['dist/wasm-worker.js', 'dist/zig-helpers.js', 135 'runtime/haskell/worker.mjs', 'runtime/zig/site-session.mjs']: 136 source = (args.site / script).read_text() 137 assert '/_arboricx/bundle/hash/' in source and 'bundles/' not in source 138 for backend, binary in [('haskell', 'reactor.wasm'), ('zig', 'arboricx.wasm')]: 139 assert (args.site / 'runtime' / backend / binary).read_bytes()[:8] == b'\x00asm\x01\x00\x00\x00' 140 runtime = args.site / 'runtime/haskell' 141 pins = json.loads((support / 'haskell-pins.mjs').read_text().split('export default ', 1)[1].split(';', 1)[0]) 142 for name in ['compiler.dag', 'environment.tsv', 'compiler-exports.tsv']: 143 assert hashlib.sha256((runtime / 'data' / name).read_bytes()).hexdigest() == pins[name] 144 dag = (runtime / 'data/compiler.dag').read_bytes() 145 count_nodes = int.from_bytes(dag[:4], 'little') 146 table = [row.split('\t') for row in (runtime / 'data/compiler-exports.tsv').read_text().splitlines()] 147 assert len({name for name, _ in table}) == len(table) 148 assert all(index.isdecimal() and 0 <= int(index) < count_nodes for _, index in table) 149 roots = dict(table) 150 assert int(roots[compiler['export']]) == int.from_bytes(dag[4:8], 'little') 151 assert all('source_module.' + name in roots for name in 152 ['sourceModuleHeaderStage', 'sourceModuleUseShape', 'sourceModuleExportShape', 'sourceModuleQualified']) 153 spec = importlib.util.spec_from_file_location('seed', args.seeder) 154 module = importlib.util.module_from_spec(spec) 155 spec.loader.exec_module(module) 156 relative = Path('objects') / rows[0]['hash'][:3] / (rows[0]['hash'] + '.arboricx') 157 def rejected(call): 158 try: 159 call() 160 except (ValueError, OSError): 161 return 162 raise AssertionError('Expected integrity rejection') 163 rejected(lambda: module.seed(pack, Path('relative-store'))) 164 with tempfile.TemporaryDirectory(prefix='arboricx-seed-test-') as temp: 165 compiler_row = next(row for row in rows if 'compiler' in row['uses']) 166 assert 0 < compiler_row['size'] <= 2 * 1_048_576 167 # The compiler ceiling applies even when that artifact is also a named 168 # release. Ordinary package roles independently admit exactly 16 MiB. 169 limits_pack = Path(temp) / 'limits-pack' 170 limits_pack.mkdir() 171 for changes in [dict(uses=[]), dict(uses=['compiler', 'compiler']), 172 dict(uses='compiler'), dict(size=2_097_152 + 1), 173 dict(uses=['compiler', 'demo'], size=2_097_152 + 1), 174 dict(uses=['demo'], size=16_777_216 + 1)]: 175 (limits_pack / 'dependencies.json').write_text(json.dumps( 176 dict(schema=1, bundles=[dict(compiler_row, **changes)]))) 177 target = Path(temp) / 'rejected-size-role' 178 rejected(lambda: module.seed(limits_pack, target)) 179 assert not target.exists() 180 # Boundary controls must not depend on the current compiler's size. 181 # Constructor-K compilation reduced it below the former 1 MiB floor. 182 for size, uses in [(2_097_152, ['compiler', 'release:compiler@boundary']), 183 (16_777_216, ['release:boundary@0.1.0'])]: 184 data = b'x' * size 185 digest = hashlib.sha256(data).hexdigest() 186 obj = limits_pack / 'objects' / digest[:3] / (digest + '.arboricx') 187 obj.parent.mkdir(parents=True, exist_ok=True); obj.write_bytes(data) 188 (limits_pack / 'dependencies.json').write_text(json.dumps(dict(schema=1, 189 bundles=[dict(hash=digest, size=len(data), uses=uses)]))) 190 assert module.seed(limits_pack, Path(temp) / ('boundary-' + str(size))) == (1, 1) 191 concurrent = Path(temp) / 'concurrent' 192 children = [subprocess.Popen([str(args.deps / 'bin/arboricx-seed-site'), str(concurrent)], 193 stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] 194 for child in children: 195 stdout, stderr = child.communicate(timeout=30) 196 assert child.returncode == 0, stderr 197 assert module.seed(pack, concurrent) == (0, count) 198 root = Path(temp) / 'registry' 199 assert module.seed(pack, root) == (count, count) 200 for release in manifest['releases']: 201 assert (root / release['path']).read_bytes() == (pack / release['path']).read_bytes() 202 # An existing different release label blocks all new installations. 203 conflict = Path(temp) / 'release-conflict' 204 record = conflict / manifest['releases'][0]['path'] 205 record.parent.mkdir(parents=True) 206 record.write_bytes(b'wrong release') 207 rejected(lambda: module.seed(pack, conflict)) 208 assert not list(conflict.rglob('*.arboricx')) 209 assert record.read_bytes() == b'wrong release' 210 object_path = root / relative 211 before = (object_path.stat().st_ino, object_path.stat().st_mtime_ns) 212 old = root / 'objects/old/retained.arboricx' 213 old.parent.mkdir();old.write_bytes(b'prior release retained') 214 assert module.seed(pack, root) == (0, count) 215 assert before == (object_path.stat().st_ino, object_path.stat().st_mtime_ns) 216 assert old.read_bytes() == b'prior release retained' 217 # Exercise the installed wrapper, not just the Python function. 218 subprocess.run([str(args.deps / 'bin/arboricx-seed-site'), str(root)], check=True) 219 object_path.chmod(0o644) 220 original = object_path.read_bytes() 221 object_path.write_bytes(bytes([original[0] ^ 1]) + original[1:]) 222 corrupt = object_path.read_bytes() 223 rejected(lambda: module.seed(pack, root)) 224 assert object_path.read_bytes() == corrupt 225 # Reject symlink objects rather than following/overwriting them. 226 object_path.unlink();object_path.symlink_to(pack / relative) 227 rejected(lambda: module.seed(pack, root)) 228 assert object_path.is_symlink() 229 # Corrupt packaged input must fail before creating the target store. 230 bad_pack = Path(temp) / 'bad-pack' 231 bad_pack.mkdir() 232 (bad_pack / 'dependencies.json').write_text(json.dumps({'schema':1,'bundles':[rows[0]]})) 233 bad_object = bad_pack / relative 234 bad_object.parent.mkdir(parents=True);bad_object.write_bytes(corrupt) 235 target = Path(temp) / 'not-created' 236 rejected(lambda: module.seed(bad_pack, target)) 237 assert not target.exists() 238 # A corrupt existing object blocks installing ALL other missing objects. 239 partial = Path(temp) / 'partial' 240 partial_object = partial / relative 241 partial_object.parent.mkdir(parents=True);partial_object.write_bytes(corrupt) 242 rejected(lambda: module.seed(pack, partial)) 243 assert len(list(partial.rglob('*.arboricx'))) == 1 244 assert not list(root.rglob('.site-seed-*')) 245 print(f'PASS static separation; {len(rows)} pinned dependencies; first/repeat/wrapper seeding; ' 246 'concurrent seeding; old-release retention; corrupt source/existing and symlink rejection; atomic cleanup') 247 248 249 if __name__ == '__main__': 250 main()