release.py (16117B)
1 #!/usr/bin/env python3 2 """Fast filesystem release/seed checks; no application compilation or evaluation.""" 3 import argparse 4 import hashlib 5 import importlib.util 6 import json 7 import re 8 import struct 9 from pathlib import Path 10 import subprocess 11 import tempfile 12 13 14 def main(): 15 parser = argparse.ArgumentParser(description=__doc__) 16 parser.add_argument('--site', type=Path, required=True) 17 parser.add_argument('--deps', type=Path, required=True) 18 parser.add_argument('--seeder', type=Path, required=True) 19 args = parser.parse_args() 20 # Every relative runtime/Worker/metadata URL stays in the same release tree. 21 assets = json.loads((args.site / 'assets.json').read_text()) 22 encoded = json.dumps(assets['files'], sort_keys=True, separators=(',', ':')).encode() 23 assert assets['version'] == hashlib.sha256(b'arboricx.site-assets.v1\0' + encoded).hexdigest() 24 assert assets['base'] == 'assets/' + assets['version'] + '/' 25 html = (args.site / 'index.html').read_text() 26 for entry in ['dist/app.js', 'style.css', 'theme.js', 'images/favicon.png', 'images/icon.svg', 'images/apple-touch-icon.png']: 27 assert '"./' + assets['base'] + entry + '"' in html 28 assert 'https://arbori.cx/' + assets['base'] + 'images/preview.png' in html 29 for name, size in [('preview.png', (1200, 630)), ('favicon.png', (32, 32)), ('apple-touch-icon.png', (180, 180))]: 30 png = (args.site / assets['base'] / 'images' / name).read_bytes() 31 assert png[:8] == b'\x89PNG\r\n\x1a\n' 32 assert (int.from_bytes(png[16:20], 'big'), int.from_bytes(png[20:24], 'big')) == size 33 assert not (args.site / 'dist').exists() 34 llms = (args.site / 'llms.txt').read_text() 35 links = re.findall(r'\]\(([^)]+)\)', llms) 36 assert links 37 for url in links: 38 prefix = 'https://arbori.cx/' 39 assert url.startswith(prefix + assets['base'] + 'reference/'), url 40 assert (args.site / url.removeprefix(prefix)).is_file(), url 41 assert 'href="./llms.txt"' in html 42 support = args.site / 'nix-support' 43 args.site = args.site / assets['base'] 44 assert sorted(p.relative_to(args.site).as_posix() for p in args.site.rglob('*') if p.is_file()) == [r['path'] for r in assets['files']] 45 for row in assets['files']: 46 data = (args.site / row['path']).read_bytes() 47 assert len(data) == row['size'] 48 assert hashlib.sha256(data).hexdigest() == row['sha256'] 49 pack = args.deps / 'share/arboricx-site-deps' 50 manifest = json.loads((pack / 'dependencies.json').read_text()) 51 rows = manifest['bundles'] 52 catalogue = json.loads((args.site / 'catalogue.json').read_text()) 53 compiler = json.loads((args.site / 'compiler.json').read_text()) 54 index = json.loads((args.site / 'packages.json').read_text()) 55 environments = json.loads((args.site / 'environments.json').read_text()) 56 examples = json.loads((args.site / 'reference/examples.json').read_text()) 57 quickstart = (args.site / 'reference/getting-started.md').read_text() 58 for code in re.findall(r'```tricu\n(.*?)```', quickstart, re.S): 59 assert any(e['source'].strip() == code.strip() for e in examples) 60 libraries = json.loads((args.site / 'reference/libraries.json').read_text())['releases'] 61 worked = (args.site / 'reference/examples.md').read_text() 62 for example in examples: 63 name = example['expression'] 64 assert re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name) 65 assert re.search(r'^' + re.escape(name) + r'\s*=', example['source'], re.M) 66 assert f'Click the underlined `{name}` definition' in worked 67 assert 'Returned value (next run)' in worked and 'Run IO' in worked 68 # Site instructions describe the source/name/card workflow in both text 69 # references and the examples embedded in the library explorer. 70 instructions = [quickstart, worked, llms, *[r.get('example', '') for r in libraries], 71 *[p.read_text() for p in (args.site / 'reference/libraries').glob('*.md')]] 72 for text in instructions: 73 for retired in ['Evaluation panel', 'Evaluated result', 'Text / line IO', 74 'enter its expression', 'settings and Evaluate', 'Expression:']: 75 assert retired not in text, retired 76 published = [r for r in index['releases'] if r['name'].startswith('tricu-lib-')] 77 assert {(r['name'], r['version']) for r in libraries} == {(r['name'], r['version']) for r in published} 78 # Independent reader: public names are the bundle's export table, not source 79 # declaration guesses (which would miss inherited/reexported names). 80 def public_names(data): 81 sections = struct.unpack_from('>I', data, 12)[0] 82 manifest = None 83 for i in range(sections): 84 at = 32 + i * 32 85 tag = struct.unpack_from('>I', data, at)[0] 86 offset, size = struct.unpack_from('>QQ', data, at + 12) 87 if tag == 1: manifest = data[offset:offset + size] 88 assert manifest is not None 89 offset = 12 90 def word(): 91 nonlocal offset 92 value = struct.unpack_from('>I', manifest, offset)[0]; offset += 4 93 return value 94 def text(): 95 nonlocal offset 96 size = word(); value = manifest[offset:offset + size]; offset += size 97 return value.decode() 98 for _ in range(9): text() 99 for _ in range(word()): text() 100 offset += 1 101 for _ in range(word()): word(); text() 102 result = [] 103 for _ in range(word()): result.append(text()); word(); text(); text() 104 return result 105 for row in libraries: 106 release = next(r for r in published if (r['name'],r['version']) == (row['name'],row['version'])) 107 assert row['manifest'] == release['manifest'] and row['hash'] == release['hash'] 108 data = (pack / 'objects' / row['hash'][:3] / (row['hash'] + '.arboricx')).read_bytes() 109 assert row['names'] == public_names(data) 110 reference = (args.site / 'reference' / row['reference']).read_text() 111 assert '!import ' + row['manifest'] + ' as ' + row['alias'] in reference 112 assert all(name in reference for name in row['names']) 113 # All relative static Markdown links resolve inside the released reference tree. 114 for path in (args.site / 'reference').rglob('*.md'): 115 for href in re.findall(r'\]\(([^ )]+)(?: +[^)]*)?\)', path.read_text()): 116 if ':' in href or href.startswith('#'): continue 117 destination = (path.parent / href.split('#')[0]).resolve() 118 assert destination.is_file(), (path, href) 119 120 count = len(rows) + len(manifest['releases']) 121 assert {row['hash'] for row in rows} == {entry['hash'] for entry in [*catalogue, compiler, *index['releases'], *[p for p in environments['profiles'].values() if p]]} 122 for row in rows: 123 expected = sorted([entry['id'] for entry in catalogue if entry['hash'] == row['hash']] 124 + (['compiler'] if compiler['hash'] == row['hash'] else []) 125 + ['release:' + r['name'] + '@' + r['version'] for r in index['releases'] if r['hash'] == row['hash']] 126 + ['environment:' + n for n, p in environments['profiles'].items() if p and p['hash'] == row['hash']]) 127 assert row['uses'] == expected 128 assert not list(args.site.rglob('*.arboricx')) 129 assert not (args.site / 'bundles').exists() 130 assert all('path' not in entry for entry in [*catalogue, compiler]) 131 # Production uses only native-runtime Worker drivers; JS is test-only. 132 assert not list(args.site.rglob('conformance')) 133 for retired in ['worker.js', 'compiler-worker.js']: 134 assert not (args.site / 'dist' / retired).exists() 135 entries = {'dist/app.js', 'dist/wasm-worker.js', 'dist/local-module-worker.js', 136 'runtime/haskell/client.mjs', 137 'runtime/haskell/worker.mjs'} 138 # The early theme script is copied verbatim, not an esbuild entrypoint. 139 assert {p.relative_to(args.site).as_posix() for p in args.site.rglob('*') 140 if p.suffix in ['.js', '.mjs']} == entries | {'theme.js'} 141 graphs = json.loads((support / 'bundle-graphs.json').read_text()) 142 assert set(graphs) == entries | {'dist/probe-api.js'} 143 for entry, graph in graphs.items(): 144 assert set(graph['outputs']) == {'bundled/' + entry} 145 assert not graph['outputs']['bundled/' + entry]['imports'] 146 assert not any('/conformance/' in p for p in graph['inputs']) 147 file = support / 'probe-api.js' if entry == 'dist/probe-api.js' else args.site / entry 148 assert file.stat().st_size == graph['outputs']['bundled/' + entry]['bytes'] 149 assert 'runtime/haskell/shim/index.js' in graphs['runtime/haskell/worker.mjs']['inputs'] 150 assert not any('runtime/' in p for p in graphs['dist/app.js']['inputs']) 151 assert not (args.site / 'dist/probe-api.js').exists() 152 # Preserve the no-static-bundle-fallback control in each bundled download owner. 153 for script in ['dist/wasm-worker.js', 'runtime/haskell/worker.mjs']: 154 source = (args.site / script).read_text() 155 assert '/_arboricx/bundle/hash/' in source and 'bundles/' not in source 156 for backend, binary in [('haskell', 'reactor.wasm')]: 157 assert (args.site / 'runtime' / backend / binary).read_bytes()[:8] == b'\x00asm\x01\x00\x00\x00' 158 assert {p.name for p in (args.site / 'runtime').iterdir()} == {'haskell'} 159 runtime = args.site / 'runtime/haskell' 160 pins = json.loads((support / 'haskell-pins.mjs').read_text().split('export default ', 1)[1].split(';', 1)[0]) 161 for name in ['compiler.dag', 'environment.tsv', 'compiler-exports.tsv']: 162 assert hashlib.sha256((runtime / 'data' / name).read_bytes()).hexdigest() == pins[name] 163 dag = (runtime / 'data/compiler.dag').read_bytes() 164 count_nodes = int.from_bytes(dag[:4], 'little') 165 table = [row.split('\t') for row in (runtime / 'data/compiler-exports.tsv').read_text().splitlines()] 166 assert len({name for name, _ in table}) == len(table) 167 assert all(index.isdecimal() and 0 <= int(index) < count_nodes for _, index in table) 168 roots = dict(table) 169 assert int(roots[compiler['export']]) == int.from_bytes(dag[4:8], 'little') 170 assert all('source_module.' + name in roots for name in 171 ['sourceModuleHeaderStage', 'sourceModuleUseShape', 'sourceModuleExportShape', 'sourceModuleQualified']) 172 spec = importlib.util.spec_from_file_location('seed', args.seeder) 173 module = importlib.util.module_from_spec(spec) 174 spec.loader.exec_module(module) 175 relative = Path('objects') / rows[0]['hash'][:3] / (rows[0]['hash'] + '.arboricx') 176 def rejected(call): 177 try: 178 call() 179 except (ValueError, OSError): 180 return 181 raise AssertionError('Expected integrity rejection') 182 rejected(lambda: module.seed(pack, Path('relative-store'))) 183 with tempfile.TemporaryDirectory(prefix='arboricx-seed-test-') as temp: 184 compiler_row = next(row for row in rows if 'compiler' in row['uses']) 185 assert 0 < compiler_row['size'] <= 2 * 1_048_576 186 # The compiler ceiling applies even when that artifact is also a named 187 # release. Ordinary package roles independently admit exactly 16 MiB. 188 limits_pack = Path(temp) / 'limits-pack' 189 limits_pack.mkdir() 190 for changes in [dict(uses=[]), dict(uses=['compiler', 'compiler']), 191 dict(uses='compiler'), dict(size=2_097_152 + 1), 192 dict(uses=['compiler', 'demo'], size=2_097_152 + 1), 193 dict(uses=['demo'], size=16_777_216 + 1)]: 194 (limits_pack / 'dependencies.json').write_text(json.dumps( 195 dict(schema=1, bundles=[dict(compiler_row, **changes)]))) 196 target = Path(temp) / 'rejected-size-role' 197 rejected(lambda: module.seed(limits_pack, target)) 198 assert not target.exists() 199 # Boundary controls must not depend on the current compiler's size. 200 # Constructor-K compilation reduced it below the former 1 MiB floor. 201 for size, uses in [(2_097_152, ['compiler', 'release:compiler@boundary']), 202 (16_777_216, ['release:boundary@0.1.0'])]: 203 data = b'x' * size 204 digest = hashlib.sha256(data).hexdigest() 205 obj = limits_pack / 'objects' / digest[:3] / (digest + '.arboricx') 206 obj.parent.mkdir(parents=True, exist_ok=True); obj.write_bytes(data) 207 (limits_pack / 'dependencies.json').write_text(json.dumps(dict(schema=1, 208 bundles=[dict(hash=digest, size=len(data), uses=uses)]))) 209 assert module.seed(limits_pack, Path(temp) / ('boundary-' + str(size))) == (1, 1) 210 concurrent = Path(temp) / 'concurrent' 211 children = [subprocess.Popen([str(args.deps / 'bin/arboricx-seed-site'), str(concurrent)], 212 stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] 213 for child in children: 214 stdout, stderr = child.communicate(timeout=30) 215 assert child.returncode == 0, stderr 216 assert module.seed(pack, concurrent) == (0, count) 217 root = Path(temp) / 'registry' 218 assert module.seed(pack, root) == (count, count) 219 for release in manifest['releases']: 220 assert (root / release['path']).read_bytes() == (pack / release['path']).read_bytes() 221 # An existing different release label blocks all new installations. 222 conflict = Path(temp) / 'release-conflict' 223 record = conflict / manifest['releases'][0]['path'] 224 record.parent.mkdir(parents=True) 225 record.write_bytes(b'wrong release') 226 rejected(lambda: module.seed(pack, conflict)) 227 assert not list(conflict.rglob('*.arboricx')) 228 assert record.read_bytes() == b'wrong release' 229 object_path = root / relative 230 before = (object_path.stat().st_ino, object_path.stat().st_mtime_ns) 231 old = root / 'objects/old/retained.arboricx' 232 old.parent.mkdir();old.write_bytes(b'prior release retained') 233 assert module.seed(pack, root) == (0, count) 234 assert before == (object_path.stat().st_ino, object_path.stat().st_mtime_ns) 235 assert old.read_bytes() == b'prior release retained' 236 # Exercise the installed wrapper, not just the Python function. 237 subprocess.run([str(args.deps / 'bin/arboricx-seed-site'), str(root)], check=True) 238 object_path.chmod(0o644) 239 original = object_path.read_bytes() 240 object_path.write_bytes(bytes([original[0] ^ 1]) + original[1:]) 241 corrupt = object_path.read_bytes() 242 rejected(lambda: module.seed(pack, root)) 243 assert object_path.read_bytes() == corrupt 244 # Reject symlink objects rather than following/overwriting them. 245 object_path.unlink();object_path.symlink_to(pack / relative) 246 rejected(lambda: module.seed(pack, root)) 247 assert object_path.is_symlink() 248 # Corrupt packaged input must fail before creating the target store. 249 bad_pack = Path(temp) / 'bad-pack' 250 bad_pack.mkdir() 251 (bad_pack / 'dependencies.json').write_text(json.dumps({'schema':1,'bundles':[rows[0]]})) 252 bad_object = bad_pack / relative 253 bad_object.parent.mkdir(parents=True);bad_object.write_bytes(corrupt) 254 target = Path(temp) / 'not-created' 255 rejected(lambda: module.seed(bad_pack, target)) 256 assert not target.exists() 257 # A corrupt existing object blocks installing ALL other missing objects. 258 partial = Path(temp) / 'partial' 259 partial_object = partial / relative 260 partial_object.parent.mkdir(parents=True);partial_object.write_bytes(corrupt) 261 rejected(lambda: module.seed(pack, partial)) 262 assert len(list(partial.rglob('*.arboricx'))) == 1 263 assert not list(root.rglob('.site-seed-*')) 264 print(f'PASS static separation; {len(rows)} pinned dependencies; first/repeat/wrapper seeding; ' 265 'concurrent seeding; old-release retention; corrupt source/existing and symlink rejection; atomic cleanup') 266 267 268 if __name__ == '__main__': 269 main()