prepare.py (18173B)
1 #!/usr/bin/env python3 2 """Opt-in native manifest-module demo compilation / Haskell goldens. No library builds. 3 Run after nix build .#: python3 ext/demo-site/prepare.py result/bin/tricu --backend stable 4 """ 5 import argparse 6 import hashlib 7 import json 8 from pathlib import Path 9 import re 10 import subprocess 11 import tempfile 12 import sys 13 14 HERE = Path(__file__).resolve().parent 15 ROOT = HERE.parent.parent 16 sys.path.insert(0, str(ROOT / 'test')) 17 from shipped_compiler import verify 18 from bundle_dag import read 19 20 21 def digest(data): 22 return hashlib.sha256(data).hexdigest() 23 24 25 def codec_vector_source(kind, value): 26 # Scalar lists keep this authoring source ASCII with the retained compiler; 27 # they test runtime string codecs, not source-file UTF-8 decoding. 28 literal = str(value) if kind == 'number' else '[' + ' '.join(str(ord(c)) for c in value) + ']' 29 return '!export value\nvalue = ' + literal + '\n' 30 31 32 def source_tree(text): 33 if text == '0': 34 return 't' 35 if text[0] == '1': 36 return '(t ' + source_tree(text[1:]) + ')' 37 left, right = pair(text) 38 return '(t ' + source_tree(left) + ' ' + source_tree(right) + ')' 39 40 41 def apply_vector_source(program, argument): 42 return '!export value\nvalue = ' + source_tree(program) + ' ' + source_tree(argument) + '\n' 43 44 45 def reference_compiler(): 46 """Reproduce the shipped historical cohort, never silently follow HEAD.""" 47 descriptor = json.loads((HERE / 'compiler.json').read_text()) 48 pin = descriptor['hash'] 49 if not re.fullmatch('[0-9a-f]{64}', pin): 50 raise ValueError('Invalid compiler pin') 51 bundle = HERE / 'bundles' / (pin + '.arboricx') 52 if digest(bundle.read_bytes()) != pin: 53 raise ValueError('Compiler bundle integrity failure') 54 return bundle 55 56 57 def end(text, at=0): 58 pending = 1 59 while pending: 60 if at >= len(text) or text[at] not in '012': 61 raise ValueError('invalid/truncated tree') 62 pending += int(text[at]) - 1 63 at += 1 64 return at 65 66 67 def pair(text): 68 assert text[0] == '2', text[:80] 69 at = end(text, 1) 70 assert end(text, at) == len(text) 71 return text[1:at], text[at:] 72 73 74 def number(text): 75 n, bit = 0, 1 76 while text != '0': 77 head, text = pair(text) 78 assert head in ('0', '10') 79 n += (head == '10') * bit 80 bit *= 2 81 return n 82 83 84 def string(text): 85 chars = [] 86 while text != '0': 87 head, text = pair(text) 88 chars.append(chr(number(head))) 89 return ''.join(chars) 90 91 92 def of_number(n): 93 return ('2' + ('10' if n % 2 else '0') + of_number(n // 2)) if n else '0' 94 95 96 def of_string(text): 97 return ''.join('2' + of_number(ord(c)) for c in text) + '0' 98 99 100 def main(): 101 import practical 102 import tutorials 103 parser = argparse.ArgumentParser(description=__doc__) 104 parser.add_argument('tricu', type=Path) 105 parser.add_argument('--backend', required=True, choices=['stable']) 106 parser.add_argument('--compiler-bundle', type=Path, default=reference_compiler()) 107 parser.add_argument('--compiler-export', default='source_unit.compileStableModuleDefinitions') 108 args = parser.parse_args() 109 if args.compiler_export.rsplit('.', 1)[-1] != 'compileStableModuleDefinitions': 110 parser.error('compiler export must explicitly select compileStableModuleDefinitions') 111 host, compiler = args.tricu.resolve(), args.compiler_bundle.resolve() 112 native_metadata = verify() 113 with tempfile.TemporaryDirectory(prefix='tricu-demo-prepare-') as temporary: 114 temp = Path(temporary) 115 store = temp / 'store' 116 117 def run(*command, input=None): 118 reply = subprocess.run([str(host), *map(str, command), '+RTS', '-N1', '-M8192m', '-RTS'], cwd=ROOT, 119 input=input, capture_output=True, timeout=120) 120 if reply.returncode: 121 raise RuntimeError(reply.stderr.decode() + reply.stdout.decode()) 122 return reply.stdout.decode() 123 124 imported = run('arboricx', 'import', '--file', compiler, '--store', store, '--indexed') 125 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', imported)[1] 126 (temp / 'empty.tree').write_bytes(b'\0') 127 generated = {} 128 129 def publish(kind, payload): 130 hashed = digest(kind.encode() + b'\0' + payload) 131 path = store / 'objects' / hashed[:3] / hashed 132 path.parent.mkdir(parents=True, exist_ok=True) 133 path.write_bytes(payload) 134 return hashed 135 136 def of_list(values): 137 return ''.join('2' + value for value in values) + '0' 138 139 # The retired intro remains an independently checked runtime fixture. 140 settings = { 141 'intro': dict(title='Start here', exports=['answer', 'identity', 'first', 'leaf', 'stem', 'fork', 'greeting', 'numbers'], 142 export='answer', args=[], output='number', mode='pure'), 143 'greet': dict(title='A conversation', exports=['main'], export='main', args=[], output='string', mode='io'), 144 'microscope': dict(title='Tree microscope', exports=['shape', 'size', 'same'], export='size', 145 args=[dict(type='tree', value='21100')], output='number', mode='pure'), 146 'rgb': dict(title='RGB validator', exports=['main'], export='main', 147 args=[dict(type='bytes', value=[64, 128, 255])], output='string', mode='pure'), 148 } 149 settings['microscope']['presets'] = { 150 'shape': dict(arguments=[dict(type='tree', value='21100')], output='string', mode='pure'), 151 'size': dict(arguments=[dict(type='tree', value='21100')], output='number', mode='pure'), 152 'same': dict(arguments=[dict(type='tree', value='21100'), dict(type='tree', value='21100')], output='bool', mode='pure'), 153 } 154 settings.update(practical.settings()) 155 settings = {**tutorials.settings(), **settings} 156 tutorial_cases = tutorials.cases() 157 practical_cases = practical.cases() 158 def compile_unit(source, environment): 159 env_file = temp / 'environment.tree' 160 env_file.write_bytes(bytes(map(int, ''.join('22' + of_string(name) + tree for name, tree in environment.items()) + '0'))) 161 reply = run('arboricx', 'run', '--module', pin, '--name', args.compiler_export, 162 '--store', store, '--arg-tree-file', env_file, 163 '--arg', source.decode(), '--format', 'ternary').strip() 164 assert end(reply) == len(reply) 165 success, body = pair(reply) 166 assert success == '10', f'compiler diagnostic {body[:200]}' 167 definitions, _ = pair(body) 168 bindings = {} 169 while definitions != '0': 170 binding, definitions = pair(definitions) 171 key, tree = pair(binding) 172 bindings[string(key)] = tree 173 return bindings 174 def imported(file, target=store): 175 output = run('arboricx', 'import', '--file', file, '--store', target, '--indexed') 176 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 177 native_compiler = imported(ROOT / native_metadata['bundle'], temp / 'tools') 178 # Source headers are authored with immutable references. Import only the 179 # indexed dependency pins they declare; native compilation parses/resolves. 180 releases = json.loads((HERE / 'packages.json').read_text())['releases'] 181 dependencies = set() 182 for name in settings: 183 for ref in re.findall(r'^!import ([0-9a-f]{64}) ', (HERE / 'demos' / (name + '.tri')).read_text(), re.M): 184 dependencies.add(ref) 185 for ref in sorted(dependencies): 186 release = next(r for r in releases if r['manifest'] == ref) 187 file = HERE / 'bundles' / (release['hash'] + '.arboricx') 188 assert digest(file.read_bytes()) == release['hash'] 189 assert imported(file) == ref 190 def encode_literal(literal): 191 kind, value = literal['type'], literal['value'] 192 if kind == 'number': return of_number(int(value)) 193 if kind == 'string': return of_string(value) 194 if kind == 'tree': return value 195 if kind == 'bool': return '10' if value else '0' 196 return of_list([encode_literal(v) for v in value]) 197 def encode_record(record): 198 fields = [] 199 for key, value in record.items(): 200 tag = 2 if isinstance(value, bool) else (1 if isinstance(value, int) else 0) 201 fields.append(of_list([of_string(key), of_number(tag), encode_literal(practical.literal(value))])) 202 return of_list(fields) 203 package_bytes = {} 204 catalogue = [] 205 for name, setting in settings.items(): 206 source = (HERE / 'demos' / (name + '.tri')).read_bytes() 207 package = temp / (name + '.arboricx') 208 source_file = temp / (name + '.tri') 209 source_file.write_bytes(source) 210 run('arboricx', 'compile', '--file', source_file, '--store', store, 211 '--toolchain', native_compiler, '--tool-store', temp / 'tools', 212 '--output', package, '--compile-timeout', '110') 213 module_pin = imported(package) 214 assert set(read(package)[3]) == set(setting['exports']), name 215 bundle = package.read_bytes() 216 bundle_hash = digest(bundle) 217 package_bytes[bundle_hash] = bundle 218 cases = [] 219 def check(export, arguments, expected, terminal='', line=None, record=None): 220 argv = [] 221 for i, argument in enumerate(arguments): 222 path = temp / f'argument{i}.tree' 223 path.write_bytes(bytes(map(int, argument))) 224 argv += ['--arg-tree-file', path] 225 output = run('arboricx', 'run', '--module', module_pin, '--name', export, 226 '--store', store, '--format', 'ternary', *argv, 227 *(['--io'] if setting['mode'] == 'io' else []), 228 input=(line + '\n').encode() if line is not None else None) 229 assert output == terminal + expected, (name, export, arguments, output) 230 captured = dict(export=export, arguments=arguments, input=line, terminal=terminal, result=expected) 231 if record is not None: captured['record'] = record 232 cases.append(captured) 233 if name in practical_cases: 234 for export, arguments, record in practical_cases[name]: 235 check(export, [encode_literal(v) for v in arguments], encode_record(record), record=record) 236 elif name in tutorial_cases: 237 for export, arguments, expected in tutorial_cases[name]: 238 check(export, [encode_literal(a) for a in arguments], encode_literal(expected)) 239 elif name == 'intro': 240 check('answer', [], of_number(42)) 241 for tree in ['0', '10', '200', of_number(7), of_string('Hello!')]: 242 check('identity', [tree], tree) 243 check('first', [of_number(7), of_number(9)], of_number(7)) 244 for export, tree in [('leaf', '0'), ('stem', '10'), ('fork', '200')]: 245 check(export, [], tree) 246 check('greeting', [], of_string('Hello from tricu!')) 247 check('numbers', [], of_list([of_number(n) for n in [1, 2, 3]])) 248 elif name == 'greet': 249 for value in ('James', '', 'λ雪🙂', '<img src=x onerror=alert(1)>'): 250 check('main', [], of_string(value), f'What is your name?\nHello, {value}!\n', value) 251 elif name == 'microscope': 252 for tree, shape, size in [('0', 'Leaf', 1), ('10', 'Stem', 2), ('200', 'Fork', 3), ('21100', 'Fork', 5)]: 253 check('shape', [tree], of_string(shape)) 254 check('size', [tree], of_number(size)) 255 check('same', [tree, tree], '10') 256 check('same', [tree, '10' if tree != '10' else '0'], '0') 257 else: 258 for values, expected in [([0, 128, 255], 'RGB triple accepted.'), 259 ([0, 256, 255], 'A channel is not a canonical byte (0-255).'), 260 ([0, 1], 'Expected exactly three channels.'), 261 ([0, 1, 2, 3], 'Expected exactly three channels.')]: 262 check('main', [of_list([of_number(n) for n in values])], of_string(expected)) 263 for invalid in ['200', '22000', '210200', '10']: 264 check('main', [of_list([invalid, of_number(128), of_number(255)])], 265 of_string('A channel is not a canonical byte (0-255).')) 266 demand_cases = [] 267 probes = { 268 'events': [('unknown-event', 'vendorA ["unknown" (q q) (q q)]', 269 encode_record(dict(status='error', message='Unknown vendor A event.'))), 270 ('invalid-boolean', 'vendorB [(q q) 2 (q q)]', 271 encode_record(dict(status='error', message='refunded must be a canonical Boolean.')))], 272 'shipping': [('invalid-subtotal', 'v1 (q q) (t t) (q q)', 273 encode_record(dict(status='error', message='Subtotal and grams must be canonical natural numbers.')))], 274 'microscope': [('shape-with-opaque-children', 'shape (t (q q) (q q))', of_string('Fork'))], 275 'rgb': [('invalid-first-byte', 'main [(t t) (q q) (q q)]', 276 of_string('A channel is not a canonical byte (0-255).'))], 277 } 278 bindings = {export: run('arboricx', 'run', '--module', module_pin, '--name', export, 279 '--store', store, '--format', 'ternary').strip() for export in setting['exports']} if name in probes else {} 280 for label, expression, expected in probes.get(name, []): 281 probe_source = ('q x = x x\nprobe ignored = ' + expression).encode() 282 probe = compile_unit(probe_source, bindings)['probe'] 283 tree_pin = publish('arboricx.tree-term.v1', bytes(map(int, probe))) 284 probe_pin = publish('arboricx.module-manifest.v1', 285 f'arboricx.module-manifest.v1\nexport\tprobe\tarboricx.tree-term.v1\t{tree_pin}\tarboricx.abi.tree.v1\n'.encode()) 286 actual = run('arboricx', 'run', '--module', probe_pin, '--name', 'probe', 287 '--store', store, '--format', 'ternary', '--arg-tree-file', temp / 'empty.tree') 288 assert actual == expected, (name, label, actual) 289 demand_cases.append(dict(label=label, source=probe_source.decode(), program=probe, result=expected)) 290 generated[name] = dict(demandCases=demand_cases, source=source.decode(), sourceSha256=digest(source), hash=bundle_hash, 291 mode=setting['mode'], haskellCases=cases) 292 if name != 'intro': 293 catalogue.append(dict(id=name, title=setting['title'], category=setting.get('category', 'Language & Runtime Explorations' if name == 'microscope' else 'Practical Examples'), hash=bundle_hash, 294 source=source.decode(), 295 sourceSha256=digest(source), export=setting['export'], arguments=setting['args'], 296 output=setting['output'], mode=setting['mode'], presets=setting.get('presets', {}))) 297 print(f'{name}: stable bundle {len(bundle)} bytes; {len(cases)} Haskell goldens') 298 299 # Portable-authored, Haskell-serialized codec vectors, independent of the JS codec. 300 vectors = [] 301 for kind, values in [('number', [0, 1, 2, 42, 255, 1114111]), 302 ('string', ['', 'Hello', 'λ雪🙂', '\0', '<script>'])]: 303 for value in values: 304 (temp / 'vector.tri').write_text(codec_vector_source(kind, value)) 305 run('tree', 'write', temp / 'vector.tri', '--name', 'value', 306 '--store', temp / 'codec-store', '--output', temp / 'vector.tree') 307 encoded = ''.join(str(b) for b in (temp / 'vector.tree').read_bytes()) 308 vectors.append(dict(kind=kind, value=value, tree=encoded)) 309 apply_vectors = [] 310 for program, argument in [('0', '10'), ('10', '200'), ('200', '10'), 311 ('2100', '0'), ('22000', '0'), ('22000', '10'), 312 ('22000', '200'), ('21100', '200')]: 313 (temp / 'vector.tri').write_text(apply_vector_source(program, argument)) 314 run('tree', 'write', temp / 'vector.tri', '--name', 'value', 315 '--store', temp / 'codec-store', '--output', temp / 'vector.tree') 316 encoded = ''.join(str(b) for b in (temp / 'vector.tree').read_bytes()) 317 apply_vectors.append(dict(program=program, argument=argument, result=encoded)) 318 319 data = dict(schema=1, backend='stable', nativeCompilerSha256=native_metadata['sha256'], compilerExport=args.compiler_export, 320 compilerBundleSha256=digest(compiler.read_bytes()), 321 compilerManifest=pin, hostSha256=digest(host.read_bytes()), 322 demos=generated, codecVectors=vectors, applyVectors=apply_vectors) 323 # Publish only after every compilation and independent host check passes. 324 directory = HERE / 'bundles' 325 directory.mkdir(exist_ok=True) 326 for bundle_hash, payload in package_bytes.items(): 327 (directory / (bundle_hash + '.arboricx')).write_bytes(payload) 328 (HERE / 'demos.json').write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') 329 previous = json.loads((HERE / 'catalogue.json').read_text()) if (HERE / 'catalogue.json').exists() else [] 330 catalogue.extend(entry for entry in previous if entry.get('preparation') == 'native' and entry['id'] not in settings) 331 (HERE / 'catalogue.json').write_text(json.dumps(catalogue, ensure_ascii=False, indent=2) + '\n') 332 print('Published bundles, catalogue and Haskell goldens (no compiler/library fixture refresh).') 333 334 335 if __name__ == '__main__': 336 main()