prepare-showcase.py (24548B)
1 #!/usr/bin/env python3 2 """Opt-in Haskell publication of accepted library roots and native parser/lexer demos. 3 4 Compiles locally against authenticated retained fixtures and matching sources. 5 Stages release assets after bounded acceptance. 6 """ 7 import argparse 8 import hashlib 9 import json 10 import os 11 from pathlib import Path 12 import re 13 import subprocess 14 import sys 15 import tempfile 16 import time 17 18 HERE = Path(__file__).resolve().parent 19 ROOT = HERE.parent.parent 20 sys.path.insert(0, str(ROOT / 'test')) 21 from bundle_dag import read, merge_bundles 22 from fixture_sources import current_mismatches 23 from shipped_compiler import verify 24 sys.path.insert(0, str(HERE)) 25 from prepare import of_string, of_number 26 27 LIBRARIES = [ 28 ('arboricx-server', 'arboricx_server', 'support', 'Arboricx registry server (native sockets and filesystem)'), 29 ('tricu-http', 'http', 'support', 'HTTP parsing, responses and native socket handlers'), 30 ('tricu-intensional-contracts', 'intensional', 'support', 'Intensional contracts: tree shapes and numeric structure'), 31 ('tricu-graphs', 'graph', 'compiler', 'Dependency ordering and cycle witnesses (validated small graphs)'), 32 ('tricu-parser', 'parser', 'compiler', 'Parser combinators: diagnostics, consumption and commitment'), 33 ('tricu-math', 'native_math', 'compiler', 'Binary arithmetic and ASCII classifiers (canonical naturals only)'), 34 ('tricu-lexer', 'lexer', 'compiler', 'Located character scans and token positions'), 35 ('tricu-binary', 'binary', 'support', 'Checked byte readers, prefix parsing and UTF-8 validation'), 36 ('tricu-patterns', 'patterns', 'support', 'Ordered predicate/action dispatch and thunked conditionals'), 37 ] 38 39 40 def parser_cases(): 41 def success(names, rest='', committed=True): 42 return dict(status='ok', names=', '.join(names), count=len(names), remaining=rest, 43 consumed=True, committed=committed) 44 def failure(expected, rest, consumed, committed): 45 return dict(status='error', expected=expected, remaining=rest, consumed=consumed, committed=committed) 46 return [ 47 ('parseNames', 'red,green,blue', success(['red', 'green', 'blue'])), 48 ('parseNames', 'red', success(['red'], committed=False)), 49 ('parseNames', 'red,,blue', failure('lowercase name', ',blue', True, True)), 50 ('parseNames', 'red,', failure('lowercase name', '', True, True)), 51 ('parseNames', '', failure('lowercase name', '', False, False)), 52 ('parseNames', 'Red', failure('lowercase name', 'Red', False, False)), 53 ('parseNames', 'red green', failure('end of input', ' green', True, False)), 54 ('parseNames', 'red,green!', failure('end of input', '!', True, True)), 55 ('parsePrefix', 'red green', success(['red'], ' green', False)), 56 ('parsePrefix', 'red,green!', success(['red', 'green'], '!')), 57 ('parsePrefix', 'red,', failure('lowercase name', '', True, True)), 58 ] 59 60 61 def record_tree(record): 62 def scalar(value): 63 if isinstance(value, bool): return '10' if value else '0' 64 if isinstance(value, int): return of_number(value) 65 return of_string(value) 66 def items(values): return ''.join('2' + value for value in values) + '0' 67 return items([items([of_string(key), of_number(2 if isinstance(value, bool) else 1 if isinstance(value, int) else 0), scalar(value)]) 68 for key, value in record.items()]) 69 70 71 def lexer_cases(): 72 # Independent character-counting expectations, including LF, tabs and Unicode. 73 return [ 74 ('scanWord', 'hello,world', dict(matched='hello', remaining=',world', line=1, column=6, offset=5, consumed=True)), 75 ('scanWord', '', dict(matched='', remaining='', line=1, column=1, offset=0, consumed=False)), 76 ('scanWord', 'Hello', dict(matched='', remaining='Hello', line=1, column=1, offset=0, consumed=False)), 77 ('scanWord', 'abcé', dict(matched='abc', remaining='é', line=1, column=4, offset=3, consumed=True)), 78 ('scanWhitespace', ' \n\tword', dict(matched=' \n\t', remaining='word', line=2, column=2, offset=3, consumed=True)), 79 ('scanWhitespace', '\n\nend', dict(matched='\n\n', remaining='end', line=3, column=1, offset=2, consumed=True)), 80 ('scanWhitespace', '\r\nword', dict(matched='', remaining='\r\nword', line=1, column=1, offset=0, consumed=False)), 81 ('scanWhitespace', 'é', dict(matched='', remaining='é', line=1, column=1, offset=0, consumed=False)), 82 ] 83 84 85 def stage(site, artifacts, releases, demo, goldens): 86 demos = demo if isinstance(demo, list) else [demo] 87 vectors = goldens if isinstance(goldens, list) else [goldens] 88 if [e['id'] for e in demos] != [g['id'] for g in vectors]: raise ValueError('Demo/golden identity mismatch') 89 index = json.loads((site / 'packages.json').read_text()) 90 catalogue = json.loads((site / 'catalogue.json').read_text()) 91 if index.get('schema') != 1: raise ValueError('Unsupported package index') 92 keys = [(r['name'], r['version']) for r in index['releases']] 93 if len(keys) != len(set(keys)): raise ValueError('Duplicate existing release') 94 ids = [entry['id'] for entry in catalogue] 95 if len(ids) != len(set(ids)): raise ValueError('Duplicate existing demo') 96 for release in releases: 97 previous = next((r for r in index['releases'] if (r['name'], r['version']) == (release['name'], release['version'])), None) 98 if previous and previous != release: raise ValueError('Immutable release conflict: ' + release['name']) 99 if not previous: index['releases'].append(release) 100 for demo in demos: 101 previous = next((e for e in catalogue if e['id'] == demo['id']), None) 102 if previous and previous != demo: 103 # Catalogue selection is mutable; published name/version pins are not. 104 # Only advance an existing native demo through a new accepted release 105 # of the same package, retaining its previously indexed release. 106 old = [r for r in index['releases'] if r.get('demo') == demo['id'] 107 and r['hash'] == previous['hash'] and (r['name'], r['version']) in keys] 108 fresh = [r for r in releases if r.get('demo') == demo['id'] 109 and r['hash'] == demo['hash'] and (r['name'], r['version']) not in keys 110 and any(r['name'] == prior['name'] for prior in old)] 111 if previous.get('preparation') != 'native' or demo.get('preparation') != 'native' or not fresh: 112 raise ValueError('Existing demo differs; release explicitly instead of overwriting') 113 catalogue[catalogue.index(previous)] = demo 114 if not previous: catalogue.append(demo) 115 for pin, path in artifacts.items(): 116 data = path.read_bytes() 117 if hashlib.sha256(data).hexdigest() != pin: raise ValueError('Artifact changed before staging') 118 destination = site / 'bundles' / (pin + '.arboricx') 119 if destination.exists() and destination.read_bytes() != data: raise ValueError('Corrupt existing bundle') 120 # Immutable bytes first, then discovery. Failed staging can leave orphan assets, 121 # never an index advertising absent bytes. This is an exclusive local operation. 122 for pin, path in artifacts.items(): 123 destination = site / 'bundles' / (pin + '.arboricx') 124 if not destination.exists(): 125 with destination.open('xb') as output: output.write(path.read_bytes()) 126 for name, value in [('packages.json', index), ('catalogue.json', catalogue)] + [(g['id'] + '-demo.json', g) for g in vectors]: 127 with tempfile.NamedTemporaryFile(dir=site, delete=False) as output: 128 temporary = Path(output.name) 129 output.write((json.dumps(value, indent=2) + '\n').encode()) 130 try: 131 temporary.chmod(0o644) 132 os.replace(temporary, site / name) 133 finally: temporary.unlink(missing_ok=True) 134 135 136 def main(): 137 parser = argparse.ArgumentParser(description=__doc__) 138 parser.add_argument('tricu', type=Path) 139 parser.add_argument('--directory', type=Path, required=True, help='New retained output directory') 140 parser.add_argument('--reuse-directory', action='store_true', help='Reuse a local CAS; sources are rewritten and native receipts checked again') 141 parser.add_argument('--demos-only', action='store_true', help='Refresh parser/lexer demos against existing library releases without republishing libraries') 142 parser.add_argument('--stage-site', action='store_true', help='After acceptance, stage into this static site') 143 parser.add_argument('--version', default='0.1.1', help='New exact release label for the package set; existing versions remain immutable') 144 args = parser.parse_args() 145 metadata = verify() 146 descriptor = json.loads((HERE / 'compiler.json').read_text()) 147 compiler_library = ROOT / 'test/fixtures/compiler-library.arboricx' 148 if (descriptor['schema'], descriptor['backend'], descriptor['export']) != (1, 'stable', 'source_unit.compileStableModuleDefinitions'): 149 raise ValueError('Invalid browser compiler descriptor') 150 if hashlib.sha256(compiler_library.read_bytes()).hexdigest() != descriptor['hash']: 151 raise ValueError('Browser compiler differs from authenticated compiler fixture') 152 if compiler_library.stat().st_size > 2_097_152: raise ValueError('Browser compiler exceeds draft-compilation profile') 153 if (HERE / 'bundles' / (descriptor['hash'] + '.arboricx')).read_bytes() != compiler_library.read_bytes(): 154 raise ValueError('Browser compiler asset differs from authenticated compiler fixture') 155 mismatches = current_mismatches(ROOT / 'test/fixtures', ROOT) 156 if mismatches: raise ValueError('Current sources differ from retained fixtures: ' + str(mismatches)) 157 root = args.directory.resolve(); root.mkdir(parents=True, exist_ok=args.reuse_directory) 158 workspace = root / 'workspace'; workspace.mkdir(exist_ok=args.reuse_directory) 159 host = args.tricu.resolve(); store = root / 'store' 160 started = time.monotonic() 161 report = dict(complete=False, compiler=metadata['sha256'], hostSha256=hashlib.sha256(host.read_bytes()).hexdigest(), 162 commands=[], releases=[], sourceSha256={}, fixtureSha256={}) 163 def save(): (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 164 def run(*words, seconds=60, strip=True, success=True): 165 remaining = 600 - (time.monotonic() - started) 166 if remaining <= 0: raise TimeoutError('Showcase preparation whole-job deadline exceeded') 167 command = [str(host), *map(str, words), '+RTS', '-N1', '-M2048m', '-RTS'] 168 before = time.monotonic() 169 reply = subprocess.run(command, cwd=ROOT, capture_output=True, timeout=min(seconds, remaining)) 170 report['commands'].append(dict(argv=command[1:], seconds=time.monotonic()-before, code=reply.returncode, 171 stdout=reply.stdout.decode(), stderr=reply.stderr.decode())) 172 save() 173 if (reply.returncode == 0) != success: raise RuntimeError(report['commands'][-1]) 174 return reply.stdout.decode().strip() if strip else reply.stdout.decode() 175 def imported(bundle, target=store): 176 reply = run('arboricx', 'import', '--indexed', '--store', target, '--file', bundle) 177 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 178 def extracted(alias, cohort): 179 original = ROOT / 'test/fixtures' / (cohort + '-library.arboricx') 180 report['fixtureSha256'][cohort] = hashlib.sha256(original.read_bytes()).hexdigest() 181 names = [(name, name.split('.', 1)[1]) for name in read(original)[3] if name.startswith(alias + '.')] 182 if not names: raise ValueError('No retained public roots: ' + alias) 183 bundle = root / (alias + '.arboricx') 184 # Exact constructor/child-ID equivalence, without evaluating/unfolding. 185 merge_bundles(bundle, [(original, names)], prune_unreachable=True) 186 return bundle 187 artifacts = {} 188 def published(name, title, bundle, demo=None): 189 data = bundle.read_bytes() 190 limit = 16_777_216 191 if not 0 < len(data) <= limit or len(read(bundle)[2]) > 100000: 192 raise ValueError('Package exceeds browser input bounds: ' + name) 193 result = run('package', 'publish', '--root', root / 'registry', '--name', name, 194 '--version', args.version, '--file', bundle) 195 _, row = result.splitlines(); actual_name, version, manifest, pin = row.split('\t') 196 assert actual_name == name and pin == hashlib.sha256(data).hexdigest() 197 # Independent clean indexed import must produce the advertised interface. 198 assert imported(bundle, root / 'consumer') == manifest 199 release = dict(name=name, version=version, hash=pin, manifest=manifest, title=title) 200 if demo: release['demo'] = demo 201 artifacts[pin] = bundle; report['releases'].append(release) 202 return release 203 # The editable demos use the site's exact published manifest references. 204 releases = json.loads((HERE / 'packages.json').read_text())['releases'] 205 refs = set() 206 for name in ['parser', 'lexer']: 207 refs.update(re.findall(r'^!import ([0-9a-f]{64}) ', (HERE / 'demos' / (name + '.tri')).read_text(), re.M)) 208 for ref in sorted(refs): 209 release = next(r for r in releases if r['manifest'] == ref) 210 bundle = HERE / 'bundles' / (release['hash'] + '.arboricx') 211 assert hashlib.sha256(bundle.read_bytes()).hexdigest() == release['hash'] 212 assert imported(bundle) == ref 213 compiler = imported(ROOT / metadata['bundle'], root / 'tools') 214 def compile_unit(alias): 215 bundle = root / (alias + '.arboricx') 216 run('arboricx', 'compile', '--file', workspace / (alias + '.tri'), '--store', store, 217 '--toolchain', compiler, '--tool-store', root / 'tools', '--output', bundle, 218 '--compile-timeout', '180', seconds=195) 219 return bundle, imported(bundle) 220 for name in ['parser', 'lexer']: 221 source = HERE / 'demos' / (name + '.tri') 222 data = source.read_bytes() 223 (workspace / (name + '_demo.tri')).write_bytes(data) 224 report['sourceSha256'][str(source.relative_to(ROOT))] = hashlib.sha256(data).hexdigest() 225 if not args.demos_only: 226 published('tricu-compiler', 'Compiler library (stable browser source-unit API; native helpers)', compiler_library) 227 builder = published('tricu-source-builder', 'Stable native application compiler (modules, imports and publication)', ROOT / metadata['bundle']) 228 report['sourceBuilderSha256'] = metadata['sha256'] 229 report['browserCompilerSha256'] = descriptor['hash'] 230 for name, alias, cohort, title in LIBRARIES: 231 published(name, title, extracted(alias, cohort)) 232 def tree_list(values): return ''.join('2' + tree for tree in values) + '0' 233 def result_tree(success, value, rest): return '2' + ('10' if success else '0') + '2' + value + rest 234 checks = [ 235 ('tricu-math', 'add', [of_number(123456789), of_number(11)], of_number(123456800)), 236 ('tricu-math', 'divMod', [of_number(100), of_number(7)], '2' + of_number(14) + of_number(2)), 237 ('tricu-math', 'divMod', [of_number(100), of_number(0)], '2' + of_number(0) + of_number(100)), 238 ('tricu-binary', 'readU8', [tree_list([of_number(65), of_number(66)])], 239 result_tree(True, of_number(65), tree_list([of_number(66)]))), 240 ('tricu-binary', 'readU8', [tree_list([of_number(256)])], 241 result_tree(False, of_number(15), tree_list([of_number(256)]))), 242 ('tricu-binary', 'binaryUtf8?', [tree_list([of_number(195), of_number(169)])], '10'), 243 ('tricu-binary', 'binaryUtf8?', [tree_list([of_number(192), of_number(175)])], '0'), 244 ] 245 report['libraryCases'] = [] 246 for name, export, trees, expected in checks: 247 manifest = next(r['manifest'] for r in report['releases'] if r['name'] == name) 248 arguments = [] 249 for i, tree in enumerate(trees): 250 path = root / ('library-arg-' + str(i) + '.tree'); path.write_bytes(bytes(map(int, tree))) 251 arguments.extend(['--arg-tree-file', path]) 252 actual = run('arboricx', 'run', '--store', root / 'consumer', '--module', manifest, 253 '--name', export, *arguments, '--format', 'ternary') 254 assert actual == expected, (name, export, actual[:200], expected) 255 report['libraryCases'].append(dict(name=name, export=export, arguments=trees, result=actual)) 256 for alias in ['prelude', 'parser', 'lexer', 'source_lexer', 'source_equations', 'source_ast', 'native_math']: 257 pin = imported(extracted(alias, 'compiler')) 258 (workspace / (alias + '.tri')).write_text('!export * from ' + pin + '\n') 259 for alias, source in [('recursion', ROOT / 'lib/recursion/list.tri'), ('formatter', ROOT / 'lib/tools/format-tri.tri'), 260 ('parser_demo', HERE / 'demos/parser.tri'), ('lexer_demo', HERE / 'demos/lexer.tri')]: 261 data = source.read_bytes(); (workspace / (alias + '.tri')).write_bytes(data) 262 report['sourceSha256'][str(source.relative_to(ROOT))] = hashlib.sha256(data).hexdigest() 263 (workspace / 'recursion_support.tri').write_text('!import prelude as B\n!export pair\npair = B.pair\n') 264 (workspace / 'format_support.tri').write_text(''.join('!export * from ' + name + '\n' for name in 265 ['prelude', 'parser', 'source_lexer', 'source_equations', 'source_ast'])) 266 # Exercise the actual published builder through its native CLI ABI, with a 267 # local dependency and a fresh store. No sockets, public registry or fallback. 268 (workspace / 'probe_dep.tri').write_text('!export answer\nanswer = 42\n') 269 (workspace / 'builder_probe.tri').write_text('!import probe_dep as D\n!export main\nmain = D.answer\n') 270 probe = root / 'builder-probe.arboricx' 271 run('arboricx', 'compile', '--file', workspace / 'builder_probe.tri', '--store', root / 'builder-probe-store', 272 '--toolchain', builder['manifest'], '--tool-store', root / 'consumer', '--output', probe, 273 '--compile-timeout', '45', seconds=60) 274 probe_manifest = imported(probe, root / 'builder-probe-consumer') 275 assert run('arboricx', 'run', '--store', root / 'builder-probe-consumer', '--module', probe_manifest, 276 '--name', 'main', '--format', 'number') == '42' 277 (workspace / 'missing_dep.tri').write_text('!import absent_showcase_dependency as M\n!export main\nmain = M.answer\n') 278 run('arboricx', 'compile', '--file', workspace / 'missing_dep.tri', '--store', root / 'builder-probe-store', 279 '--toolchain', builder['manifest'], '--tool-store', root / 'consumer', '--output', root / 'must-not-exist.arboricx', 280 '--compile-timeout', '45', seconds=60, success=False) 281 assert not (root / 'must-not-exist.arboricx').exists(), 'failed build emitted executable' 282 recursion_bundle, _ = compile_unit('recursion') 283 published('tricu-list-traversal', 'Early-stopping list folds with retained input suffixes', recursion_bundle) 284 patterns = next(r for r in report['releases'] if r['name'] == 'tricu-patterns') 285 traversal = next(r for r in report['releases'] if r['name'] == 'tricu-list-traversal') 286 assert imported(artifacts[patterns['hash']]) == patterns['manifest'] 287 assert imported(artifacts[traversal['hash']]) == traversal['manifest'] 288 (workspace / 'published_patterns.tri').write_text('!export * from ' + patterns['manifest'] + '\n') 289 (workspace / 'published_traversal.tri').write_text('!export * from ' + traversal['manifest'] + '\n') 290 probe_source = (HERE / 'test/showcase-library-probe.tri').read_bytes() 291 report['sourceSha256']['ext/demo-site/test/showcase-library-probe.tri'] = hashlib.sha256(probe_source).hexdigest() 292 (workspace / 'library_probe.tri').write_bytes(probe_source) 293 library_probe, library_manifest = compile_unit('library_probe') 294 expected = tree_list([of_number(13), of_number(99), '0', '2' + of_number(3) + tree_list([of_number(0), of_number(9)])]) 295 actual = run('arboricx', 'run', '--store', store, '--module', library_manifest, '--name', 'main', '--format', 'ternary') 296 assert actual == expected, 'published dispatch/list traversal probe' 297 report['libraryProbe'] = dict(result=actual, bundleSha256=hashlib.sha256(library_probe.read_bytes()).hexdigest()) 298 formatter, formatter_manifest = compile_unit('formatter') 299 # Independent real formatter cases, no file effects or diverging probe needed. 300 for label, before, expected in json.loads((ROOT / 'test/formatter_cases.json').read_text()): 301 input_file = root / 'formatter-input.tri'; input_file.write_text(before) 302 actual = run('format', input_file, '--formatter', formatter_manifest, '--tool-store', store, strip=False) 303 assert input_file.read_bytes() == before.encode(), label 304 assert actual == expected, (label, repr(actual), repr(expected)) 305 published('tricu-formatter', 'Conservative tricu equation alignment (not an AST printer)', formatter) 306 parser_bundle, parser_manifest = compile_unit('parser_demo') 307 goldens = [] 308 for export, text, expected in parser_cases(): 309 actual = run('arboricx', 'run', '--store', store, '--module', parser_manifest, 310 '--name', export, '--arg', text, '--format', 'ternary') 311 assert actual == record_tree(expected), (export, text, actual[:200], expected) 312 goldens.append(dict(export=export, arguments=[of_string(text)], input=None, terminal='', result=actual, record=expected)) 313 release = published('parser-demo', 'Parser combinators: complete input, prefixes and committed errors', parser_bundle, 'parser') 314 source = (HERE / 'demos/parser.tri').read_bytes() 315 preset = dict(arguments=[dict(type='string', value='red,green,blue')], output='record', mode='pure') 316 demo = dict(id='parser', preparation='native', environment='empty', category='Language & Runtime Explorations', title='Parser combinators', hash=release['hash'], source=source.decode(), 317 sourceSha256=hashlib.sha256(source).hexdigest(), export='parseNames', **preset, 318 presets={name: preset for name in ['parseNames', 'parsePrefix']}) 319 goldens = dict(id='parser', source=source.decode(), hash=release['hash'], mode='pure', compilerSha256=metadata['sha256'], 320 sourceSha256=hashlib.sha256(source).hexdigest(), haskellCases=goldens) 321 demos, vectors = [demo], [goldens] 322 lexer_bundle, lexer_manifest = compile_unit('lexer_demo') 323 cases = [] 324 for export, text, expected in lexer_cases(): 325 actual = run('arboricx', 'run', '--store', store, '--module', lexer_manifest, 326 '--name', export, '--arg', text, '--format', 'ternary') 327 assert actual == record_tree(expected), (export, text, actual[:200], expected) 328 cases.append(dict(export=export, arguments=[of_string(text)], input=None, terminal='', result=actual, record=expected)) 329 release = published('lexer-demo', 'Located scans: matched prefix, remainder and character positions', lexer_bundle, 'lexer') 330 source = (HERE / 'demos/lexer.tri').read_bytes() 331 word = dict(arguments=[dict(type='string', value='hello,world')], output='record', mode='pure') 332 whitespace = dict(arguments=[dict(type='string', value=' \n\tword')], output='record', mode='pure') 333 demos.append(dict(id='lexer', preparation='native', environment='empty', category='Language & Runtime Explorations', title='Located lexer scans', hash=release['hash'], source=source.decode(), 334 sourceSha256=hashlib.sha256(source).hexdigest(), export='scanWord', **word, 335 presets=dict(scanWord=word, scanWhitespace=whitespace))) 336 vectors.append(dict(id='lexer', source=source.decode(), hash=release['hash'], mode='pure', compilerSha256=metadata['sha256'], 337 sourceSha256=hashlib.sha256(source).hexdigest(), haskellCases=cases)) 338 report['complete'] = True; report['elapsedSeconds'] = time.monotonic()-started; save() 339 if args.stage_site: stage(HERE, artifacts, report['releases'], demos, vectors) 340 print('Accepted', len(report['releases']), 'releases in', round(report['elapsedSeconds'], 1), 'seconds') 341 342 343 if __name__ == '__main__': main()