prepare-libraries.py (8117B)
1 #!/usr/bin/env python3 2 """Publish the complete declared interface of every lib/**/*.tri module. 3 4 Retained modules are projected from authenticated compiler/support fixtures. 5 Other library sources are compiled against those exact modules with the packaged 6 stable compiler. The native host supplies every manifest identity. Preparation 7 stages immutable release assets locally. Run with a Nix-built host. 8 """ 9 import argparse 10 import hashlib 11 import json 12 from pathlib import Path 13 import re 14 import subprocess 15 import sys 16 import tempfile 17 18 HERE = Path(__file__).resolve().parent 19 ROOT = HERE.parent.parent 20 sys.path.insert(0, str(ROOT / 'test')) 21 from bundle_dag import read, merge_bundles 22 from fixture_sources import source_rows, verify, current_mismatches 23 from shipped_compiler import verify as verify_compiler 24 25 26 def library_sources(): 27 """Coverage is filesystem-derived; newly added libraries cannot be omitted.""" 28 fixtures = ROOT / 'test/fixtures' 29 retained = {row[3]: row for row in source_rows(fixtures)} 30 result = [] 31 for path in sorted((ROOT / 'lib').rglob('*.tri')): 32 relative = str(path.relative_to(ROOT)) 33 row = retained.get(relative) 34 alias = row[2] if row else re.sub(r'[^A-Za-z0-9_]', '_', str(path.relative_to(ROOT / 'lib').with_suffix(''))) 35 result.append((alias, relative, row[0] if row else None)) 36 if len({r[0] for r in result}) != len(result): 37 raise ValueError('Library module alias collision') 38 return result 39 40 41 def main(): 42 parser = argparse.ArgumentParser(description=__doc__) 43 parser.add_argument('tricu', type=Path) 44 parser.add_argument('--version', required=True, help='New immutable release version') 45 parser.add_argument('--stage-site', action='store_true') 46 args = parser.parse_args() 47 if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,127}', args.version): 48 raise ValueError('Invalid version') 49 verify(ROOT / 'test/fixtures') 50 compiler_metadata = verify_compiler() 51 if current_mismatches(ROOT / 'test/fixtures', ROOT): 52 raise ValueError('Working sources differ from authenticated fixtures') 53 compiler = json.loads((HERE / 'compiler.json').read_text()) 54 if hashlib.sha256((ROOT / 'test/fixtures/compiler-library.arboricx').read_bytes()).hexdigest() != compiler['hash']: 55 raise ValueError('Site/compiler fixture mismatch') 56 host = args.tricu.resolve() 57 index = json.loads((HERE / 'packages.json').read_text()) 58 releases = list(index['releases']) 59 with tempfile.TemporaryDirectory(prefix='tricu-all-libraries-') as temporary: 60 work = Path(temporary) 61 workspace = work / 'workspace' 62 workspace.mkdir() 63 (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n') 64 artifacts = {} 65 def run(*words): 66 reply = subprocess.run([str(host), *map(str, words), '+RTS', '-N1', '-M2048m', '-RTS'], 67 cwd=workspace, capture_output=True, timeout=300) 68 if reply.returncode: 69 raise RuntimeError((reply.stdout + reply.stderr).decode()) 70 return reply.stdout.decode() 71 def publish(alias, relative, bundle): 72 data = bundle.read_bytes() 73 _, _, nodes, exports, _ = read(bundle) 74 if not exports or len(exports) > 4096 or len(nodes) > 100000 or len(data) > 16777216: 75 raise ValueError('Library exceeds browser transport bounds: ' + relative) 76 name = 'tricu-lib-' + alias.replace('_', '-') 77 output = run('package', 'publish', '--root', work / 'registry', '--name', name, 78 '--version', args.version, '--file', bundle) 79 _, row = output.strip().splitlines() 80 actual_name, version, manifest, digest = row.split('\t') 81 assert (actual_name, version, digest) == (name, args.version, hashlib.sha256(data).hexdigest()) 82 # Independently import the exact transport bytes used by browser fetch. 83 imported = run('arboricx', 'import', '--indexed', '--store', work / 'store', '--file', bundle) 84 assert re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported)[1] == manifest 85 release = dict(name=name, version=version, manifest=manifest, hash=digest, title=relative) 86 old = next((r for r in releases if (r['name'], r['version']) == (name, version)), None) 87 if old and (old['hash'], old['manifest']) != (digest, manifest): 88 raise ValueError('Immutable release changed: ' + name) 89 if not old: 90 releases.append(release) 91 artifacts[digest] = data 92 (workspace / (alias + '.tri')).write_text('!export * from ' + manifest + '\n') 93 print(f'{relative}: {len(exports)} exports, {len(data)} bytes, {manifest}', flush=True) 94 95 imported_compiler = run('arboricx', 'import', '--indexed', '--store', work / 'tools', 96 '--file', ROOT / compiler_metadata['bundle']) 97 compiler_manifest = re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported_compiler)[1] 98 sources = library_sources() 99 package_names = ['tricu-lib-' + alias.replace('_', '-') for alias, _, _ in sources] 100 if len(set(package_names)) != len(package_names): 101 raise ValueError('Library package name collision') 102 for alias, relative, cohort in sources: 103 if cohort is None: 104 continue 105 original = ROOT / 'test/fixtures' / (cohort + '.arboricx') 106 names = [(name, name[len(alias) + 1:]) for name in read(original)[3] if name.startswith(alias + '.')] 107 if not names: 108 raise ValueError('Missing retained public interface: ' + relative) 109 bundle = work / (alias + '.arboricx') 110 merge_bundles(bundle, [(original, names)], prune_unreachable=True) 111 publish(alias, relative, bundle) 112 113 # Source modules declare these explicit build inputs for library consumers. 114 (workspace / 'recursion_support.tri').write_text('!import prelude as B\n!export pair\npair = B.pair\n') 115 (workspace / 'recursion.tri').write_text('!export * from recursion_list\n') 116 (workspace / 'format_support.tri').write_text(''.join('!export * from ' + name + '\n' for name in 117 ['prelude', 'parser', 'source_lexer', 'source_equations', 'source_ast'])) 118 (workspace / 'bundle_support.tri').write_text('!export * from prelude\n!export * from source_bundle\n') 119 for alias, relative, cohort in sources: 120 if cohort is None: 121 (workspace / (alias + '.tri')).write_bytes((ROOT / relative).read_bytes()) 122 for alias, relative, cohort in sources: 123 if cohort is not None: 124 continue 125 bundle = work / (alias + '.arboricx') 126 run('arboricx', 'compile', '--file', workspace / (alias + '.tri'), '--store', work / 'store', 127 '--toolchain', compiler_manifest, '--tool-store', work / 'tools', 128 '--output', bundle, '--compile-timeout', '270') 129 publish(alias, relative, bundle) 130 if args.stage_site: 131 # Validate the whole batch before modifying the site. Existing object 132 # and release bytes are immutable; historical releases stay indexed. 133 for digest, data in artifacts.items(): 134 target = HERE / 'bundles' / (digest + '.arboricx') 135 if target.exists() and target.read_bytes() != data: 136 raise ValueError('Existing bundle integrity failure') 137 for digest, data in artifacts.items(): 138 (HERE / 'bundles' / (digest + '.arboricx')).write_bytes(data) 139 target = HERE / 'packages.json' 140 pending = target.with_suffix('.json.tmp') 141 pending.write_text(json.dumps(dict(schema=1, releases=releases), indent=2) + '\n') 142 pending.replace(target) 143 print(f'Validated {len(sources)} complete library interfaces' + ('; staged site assets' if args.stage_site else ' (not staged)')) 144 145 146 if __name__ == '__main__': 147 main()