prepare-libraries.py (9340B)
1 #!/usr/bin/env python3 2 """Publish the complete declared interface of every lib/**/*.tri module. 3 4 Retained modules are projected from authenticated compiler/support fixtures. 5 Other library sources are compiled against those exact modules with the packaged 6 stable compiler. The native host supplies every manifest identity. Preparation 7 stages immutable release assets locally. Run with a Nix-built host. 8 """ 9 import argparse 10 import hashlib 11 import json 12 from pathlib import Path 13 import re 14 import subprocess 15 import sys 16 import tempfile 17 18 HERE = Path(__file__).resolve().parent 19 ROOT = HERE.parent.parent 20 sys.path.insert(0, str(ROOT / 'test')) 21 from bundle_dag import read, merge_bundles 22 from fixture_sources import source_rows, verify, current_mismatches 23 from shipped_compiler import verify as verify_compiler 24 25 26 def workspace_aliases(text): 27 """Canonical authored aliases; filesystem coverage still supplies all sources.""" 28 rows = text.splitlines() 29 if not rows or rows[0] != 'tricu.workspace.v1': 30 raise ValueError('Invalid workspace header') 31 aliases, paths = set(), {} 32 for line in rows[1:]: 33 if not line.strip() or line.startswith('#'): 34 continue 35 fields = line.split('\t') 36 if len(fields) != 2: 37 raise ValueError('Invalid workspace row') 38 name, path = fields 39 if (not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]*', name) or name in aliases or 40 path in paths or Path(path).is_absolute() or '..' in Path(path).parts): 41 raise ValueError('Invalid or duplicate workspace alias/path') 42 aliases.add(name); paths[path] = name 43 return paths 44 45 46 def library_sources(workspace_text=None): 47 """Coverage is filesystem-derived; newly added libraries cannot be omitted.""" 48 fixtures = ROOT / 'test/fixtures' 49 retained = {row[3]: row for row in source_rows(fixtures)} 50 declared = workspace_aliases((ROOT / 'tricu.workspace').read_text() if workspace_text is None else workspace_text) 51 result = [] 52 for path in sorted((ROOT / 'lib').rglob('*.tri')): 53 relative = str(path.relative_to(ROOT)) 54 row = retained.get(relative) 55 if row and relative in declared and declared[relative] != row[2]: 56 raise ValueError('Fixture/workspace alias mismatch: ' + relative) 57 alias = row[2] if row else declared.get(relative, re.sub(r'[^A-Za-z0-9_]', '_', str(path.relative_to(ROOT / 'lib').with_suffix('')))) 58 result.append((alias, relative, row[0] if row else None)) 59 if len({r[0] for r in result}) != len(result): 60 raise ValueError('Library module alias collision') 61 return result 62 63 64 def main(): 65 parser = argparse.ArgumentParser(description=__doc__) 66 parser.add_argument('tricu', type=Path) 67 parser.add_argument('--version', required=True, help='New immutable release version') 68 parser.add_argument('--stage-site', action='store_true') 69 args = parser.parse_args() 70 if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,127}', args.version): 71 raise ValueError('Invalid version') 72 verify(ROOT / 'test/fixtures') 73 compiler_metadata = verify_compiler() 74 if current_mismatches(ROOT / 'test/fixtures', ROOT): 75 raise ValueError('Working sources differ from authenticated fixtures') 76 compiler = json.loads((HERE / 'compiler.json').read_text()) 77 if hashlib.sha256((ROOT / 'test/fixtures/compiler-library.arboricx').read_bytes()).hexdigest() != compiler['hash']: 78 raise ValueError('Site/compiler fixture mismatch') 79 host = args.tricu.resolve() 80 index = json.loads((HERE / 'packages.json').read_text()) 81 releases = list(index['releases']) 82 with tempfile.TemporaryDirectory(prefix='tricu-all-libraries-') as temporary: 83 work = Path(temporary) 84 workspace = work / 'workspace' 85 workspace.mkdir() 86 (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n') 87 artifacts = {} 88 def run(*words): 89 reply = subprocess.run([str(host), *map(str, words), '+RTS', '-N1', '-M2048m', '-RTS'], 90 cwd=workspace, capture_output=True, timeout=300) 91 if reply.returncode: 92 raise RuntimeError((reply.stdout + reply.stderr).decode()) 93 return reply.stdout.decode() 94 def publish(alias, relative, bundle): 95 data = bundle.read_bytes() 96 _, _, nodes, exports, _ = read(bundle) 97 if not exports or len(exports) > 4096 or len(nodes) > 100000 or len(data) > 16777216: 98 raise ValueError('Library exceeds browser transport bounds: ' + relative) 99 name = 'tricu-lib-' + alias.replace('_', '-') 100 output = run('package', 'publish', '--root', work / 'registry', '--name', name, 101 '--version', args.version, '--file', bundle) 102 _, row = output.strip().splitlines() 103 actual_name, version, manifest, digest = row.split('\t') 104 assert (actual_name, version, digest) == (name, args.version, hashlib.sha256(data).hexdigest()) 105 # Independently import the exact transport bytes used by browser fetch. 106 imported = run('arboricx', 'import', '--indexed', '--store', work / 'store', '--file', bundle) 107 assert re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported)[1] == manifest 108 release = dict(name=name, version=version, manifest=manifest, hash=digest, title=relative) 109 old = next((r for r in releases if (r['name'], r['version']) == (name, version)), None) 110 if old and (old['hash'], old['manifest']) != (digest, manifest): 111 raise ValueError('Immutable release changed: ' + name) 112 if not old: 113 releases.append(release) 114 artifacts[digest] = data 115 (workspace / (alias + '.tri')).write_text('!export * from ' + manifest + '\n') 116 print(f'{relative}: {len(exports)} exports, {len(data)} bytes, {manifest}', flush=True) 117 118 imported_compiler = run('arboricx', 'import', '--indexed', '--store', work / 'tools', 119 '--file', ROOT / compiler_metadata['bundle']) 120 compiler_manifest = re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported_compiler)[1] 121 sources = library_sources() 122 package_names = ['tricu-lib-' + alias.replace('_', '-') for alias, _, _ in sources] 123 if len(set(package_names)) != len(package_names): 124 raise ValueError('Library package name collision') 125 for alias, relative, cohort in sources: 126 if cohort is None: 127 continue 128 original = ROOT / 'test/fixtures' / (cohort + '.arboricx') 129 names = [(name, name[len(alias) + 1:]) for name in read(original)[3] if name.startswith(alias + '.')] 130 if not names: 131 raise ValueError('Missing retained public interface: ' + relative) 132 bundle = work / (alias + '.arboricx') 133 merge_bundles(bundle, [(original, names)], prune_unreachable=True) 134 publish(alias, relative, bundle) 135 136 # Source modules declare these explicit build inputs for library consumers. 137 (workspace / 'recursion_support.tri').write_text('!import prelude as B\n!export pair\npair = B.pair\n') 138 (workspace / 'recursion.tri').write_text('!export * from recursion_list\n') 139 (workspace / 'format_support.tri').write_text(''.join('!export * from ' + name + '\n' for name in 140 ['prelude', 'parser', 'source_lexer', 'source_equations', 'source_ast'])) 141 (workspace / 'bundle_support.tri').write_text('!export * from prelude\n!export * from source_bundle\n') 142 for alias, relative, cohort in sources: 143 if cohort is None: 144 (workspace / (alias + '.tri')).write_bytes((ROOT / relative).read_bytes()) 145 for alias, relative, cohort in sources: 146 if cohort is not None: 147 continue 148 bundle = work / (alias + '.arboricx') 149 print('Compiling ' + relative + ' as ' + alias, flush=True) 150 run('arboricx', 'compile', '--file', workspace / (alias + '.tri'), '--store', work / 'store', 151 '--toolchain', compiler_manifest, '--tool-store', work / 'tools', 152 '--output', bundle, '--compile-timeout', '270') 153 publish(alias, relative, bundle) 154 if args.stage_site: 155 # Validate the whole batch before modifying the site. Existing object 156 # and release bytes are immutable; historical releases stay indexed. 157 for digest, data in artifacts.items(): 158 target = HERE / 'bundles' / (digest + '.arboricx') 159 if target.exists() and target.read_bytes() != data: 160 raise ValueError('Existing bundle integrity failure') 161 for digest, data in artifacts.items(): 162 (HERE / 'bundles' / (digest + '.arboricx')).write_bytes(data) 163 target = HERE / 'packages.json' 164 pending = target.with_suffix('.json.tmp') 165 pending.write_text(json.dumps(dict(schema=1, releases=releases), indent=2) + '\n') 166 pending.replace(target) 167 print(f'Validated {len(sources)} complete library interfaces' + ('; staged site assets' if args.stage_site else ' (not staged)')) 168 169 170 if __name__ == '__main__': 171 main()