default_frontend.py (14286B)
1 #!/usr/bin/env python3 2 """Opt-in edited-host portable frontend controls; no generations or fixtures.""" 3 import os 4 import hashlib 5 import importlib.util 6 import json 7 from pathlib import Path 8 import subprocess 9 import sys 10 import tempfile 11 import time 12 import signal 13 from process_rss import ChildRSS 14 from native_indexed_store import indexed_bytes 15 from http_headers import number, tree_list 16 17 host = str(Path(sys.argv[1]).resolve()) 18 repo = Path(__file__).resolve().parents[1] 19 bundle = repo / 'bootstrap/build-stable.arboricx' 20 with tempfile.TemporaryDirectory(prefix='tricu-frontend-') as directory: 21 root = Path(directory) 22 store = root / 'store' 23 24 def run(*args, input=None, env=None, ok=True): 25 command = [host, *map(str, args), '+RTS', '-N1', '-M8192m', '-RTS'] 26 with tempfile.TemporaryFile() as stdin, tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: 27 stdin.write((input or '').encode()); stdin.seek(0) 28 proc = subprocess.Popen(command, cwd=root, stdin=stdin, stdout=stdout, 29 stderr=stderr, env=env, start_new_session=True) 30 deadline = time.monotonic() + 180 31 monitor = ChildRSS(proc) 32 try: 33 while proc.poll() is None: 34 rss = monitor.sample() 35 assert rss is None or rss <= 8192 * 1024, 'RSS guard exceeded' 36 assert time.monotonic() < deadline, '180-second guard exceeded' 37 time.sleep(.05) 38 finally: 39 if proc.poll() is None: 40 os.killpg(proc.pid, signal.SIGKILL) 41 proc.wait() 42 stdout.seek(0); stderr.seek(0) 43 output = stdout.read().decode() + stderr.read().decode() 44 assert (proc.returncode == 0) == ok, (args, output) 45 return output 46 47 source = root / 'example.tri' 48 source.write_text('''!export main 49 !export extra 50 copy xs = (let rec 51 walk [] = [] 52 walk [h | rest] = t h (walk rest) 53 walk _ = [] 54 in walk xs) 55 main = case (copy [42 7]) of 56 [h | _] = h 57 _ = 0 58 extra = 9 59 ''') 60 assert run('eval', source, '--store', store, '--format', 'number') == '42' 61 def receipt_for(name): 62 checkpoint = (store / 'cache/source-build' / name).read_text().split('\t') 63 payload = (store / 'objects' / checkpoint[3][:3] / checkpoint[3]).read_bytes() 64 assert hashlib.sha256(b'arboricx.build-receipt.v1\0' + payload).hexdigest() == checkpoint[3] 65 return checkpoint[3], payload 66 67 default_receipt, receipt = receipt_for('example') 68 source_hash = hashlib.sha256(b'arboricx.source.v1\0' + source.read_bytes()).hexdigest().encode() 69 assert b'source\t' + source_hash + b'\n' in receipt 70 assert b'compiler\t' + hashlib.sha256(bundle.read_bytes()).hexdigest().encode() + b'\n' in receipt 71 assert run('eval', '--store', store, '--format', 'number', 72 input='!export main\nmain = 17\n') == '17' 73 unicode_source = root / 'unicode.tri' 74 text = 'é€😀e\u0301' 75 unicode_source.write_bytes(('!export main\n-- Unicode comment: ' + text + '\nmain = "' + text + '"\n').encode('utf-8')) 76 expected_tree = ''.join(map(str, tree_list([number(ord(c)) for c in text]))) 77 assert run('eval', unicode_source, '--store', store, '--format', 'ternary') == expected_tree 78 _, unicode_receipt = receipt_for('unicode') 79 assert ('source\t' + hashlib.sha256(b'arboricx.source.v1\0' + unicode_source.read_bytes()).hexdigest() + '\n').encode() in unicode_receipt 80 assert run('eval', '--store', store, '--format', 'ternary', 81 input='!export main\nmain = "' + text + '"\n') == expected_tree 82 for raw, diagnostic in [(b'!export main\nmain = "\xc0\xaf"\n', 'invalid UTF-8 leading byte at byte 21'), 83 (b'\xef\xbb\xbf!export main\nmain = t\n', 'initial source BOM is not permitted')]: 84 bad_unicode = root / 'badunicode.tri' 85 bad_unicode.write_bytes(raw) 86 diagnostic_output = run('eval', bad_unicode, '--store', store, ok=False) 87 assert diagnostic in diagnostic_output, (diagnostic, diagnostic_output) 88 assert bad_unicode.read_bytes() == raw 89 assert not (store / 'cache/source-build/badunicode').exists() 90 91 output = root / 'example.arboricx' 92 run('arboricx', 'compile', '--file', source, '--store', store, '--output', output) 93 imported = run('arboricx', 'import', '--file', output, '--store', store) 94 manifest = imported.strip().split()[-1] 95 assert 'Imported 2 root(s)' in imported 96 assert run('arboricx', 'run', '--module', manifest, '--name', 'extra', 97 '--store', store, '--format', 'number') == '9' 98 tools = root / 'tools' 99 tool_import = run('arboricx', 'import', '--file', bundle, '--store', tools) 100 tool_manifest = tool_import.strip().split()[-1] 101 tool_object = next(line.split()[-1] for line in tool_import.splitlines() if 'sourceBuildGraph ->' in line) 102 broken = dict(os.environ, tricu_datadir=str(root / 'missing')) 103 assert 'Packaged stable compiler unavailable' in run('eval', source, '--store', store, env=broken, ok=False) 104 assert 'Packaged stable compiler unavailable' in run('--store', store, input='!exit\n', env=broken, ok=False) 105 assert run('eval', source, '--store', store, '--toolchain', tool_manifest, 106 '--tool-store', tools, '--format', 'number', env=broken) == '42' 107 override_receipt, receipt_bytes = receipt_for('example') 108 assert override_receipt != default_receipt, 'selection cache identities must be distinct' 109 assert ('compiler\t' + tool_object + '\n').encode() in receipt_bytes 110 assert ('compiler\t' + tool_manifest + '\n').encode() not in receipt_bytes 111 assert tool_object != hashlib.sha256(bundle.read_bytes()).hexdigest() 112 corrupt = root / 'corrupt' / 'bootstrap' 113 corrupt.mkdir(parents=True) 114 (corrupt / bundle.name).write_bytes(b'not a compiler') 115 assert 'SHA256 mismatch' in run('eval', source, '--store', store, 116 env=dict(os.environ, tricu_datadir=str(corrupt.parent)), ok=False) 117 assert '--tool-store requires --toolchain' in run('eval', source, '--store', store, 118 '--tool-store', tools, ok=False) 119 bad = root / 'bad.tri' 120 bad.write_text('!export main\nmain = unknown\n') 121 assert 'Portable compilation failed' in run('eval', bad, '--store', store, ok=False) 122 lazy = root / 'lazy.tri' 123 lazy.write_text('!export main\nloop x = loop x\nmain = case (loop t) of\n _ = 29\n') 124 assert run('eval', lazy, '--store', store, '--format', 'number') == '29' 125 run('tree', 'write', source, '--name', 'extra', '--output', root / 'extra.tree', '--store', store) 126 assert (root / 'extra.tree').is_file() 127 exact = root / 'exact.tri' 128 exact.write_text('!export chosen\n!export other\nchosen = t (t t) (t t t)\nother = t\n') 129 payload = root / 'exact.tree' 130 run('tree', 'write', exact, '--name', 'chosen', '--output', payload, '--store', store) 131 assert payload.read_bytes() == bytes([2, 1, 0, 2, 0, 0]), 'constructor shape or selected definition changed' 132 run('tree', 'write', exact, '--name', 'absent', '--output', payload, '--store', store, ok=False) 133 assert payload.read_bytes() == bytes([2, 1, 0, 2, 0, 0]), 'failed selection replaced output' 134 run('arboricx', 'compile', '--file', exact, '--output', root / 'exact.arboricx', '--store', store) 135 exact_pin = run('arboricx', 'import', '--file', root / 'exact.arboricx', '--store', store).strip().split()[-1] 136 assert run('arboricx', 'run', '--module', exact_pin, '--name', 'chosen', '--format', 'ternary', '--store', store) == '210200' 137 # Exercise the actual migrated vector-authoring helpers against unchanged 138 # published independent expectations; never run preparation/publication. 139 spec = importlib.util.spec_from_file_location('demo_prepare', repo / 'ext/demo-site/prepare.py') 140 prepare = importlib.util.module_from_spec(spec) 141 spec.loader.exec_module(prepare) 142 vectors = json.loads((repo / 'ext/demo-site/demos.json').read_text()) 143 for vector in vectors['codecVectors'] + vectors['applyVectors']: 144 if 'kind' in vector: 145 text = prepare.codec_vector_source(vector['kind'], vector['value']) 146 expected = vector['tree'] 147 else: 148 text = prepare.apply_vector_source(vector['program'], vector['argument']) 149 expected = vector['result'] 150 exact.write_text(text) 151 run('tree', 'write', exact, '--name', 'value', '--output', payload, '--store', store) 152 assert payload.read_bytes() == bytes(map(int, expected)), vector 153 for script, arguments in [ 154 ('native_indexed_store.py', ['--generation', str(root / 'missing-generation'), 155 '--compiler', str(bundle), '--directory', str(root / 'retired-v1')]), 156 ('verified_tree_loading.py', [str(root / 'missing-previous'), str(root / 'missing-generation'), 157 '--compiler-bundle', str(bundle), '--directory', str(root / 'retired-v1')])]: 158 reply = subprocess.run([sys.executable, str(repo / 'test' / script), host, *arguments], 159 capture_output=True, text=True, timeout=10) 160 assert reply.returncode and 'Retired v1 source-splicing probe' in reply.stderr 161 assert not (root / 'retired-v1').exists() 162 for script, arguments in [('stable_bundle.py', ['--compiler-bundle', str(bundle), '--directory', str(root / 'retired')]), 163 ('stable_costs.py', [str(root / 'retired')])]: 164 reply = subprocess.run([sys.executable, str(repo / 'test' / script), host, *arguments], 165 capture_output=True, text=True, timeout=10) 166 assert reply.returncode and 'Retired implicit recovery preparation' in reply.stderr 167 assert not (root / 'retired').exists() 168 formatter_input = root / 'formatter.tri' 169 for raw, diagnostic in [(b'\xef\xbb\xbfmain = t\n', 'initial UTF-8 BOM'), 170 (b'\xc0\xaf', 'Invalid UTF-8')]: 171 formatter_input.write_bytes(raw) 172 assert diagnostic in run('format', formatter_input, '--formatter', '0' * 64, 173 '--store', store, '--write', ok=False) 174 assert formatter_input.read_bytes() == raw 175 176 # The artifact-only fast gate exercises the stored loader. These separate 177 # opt-in controls retain the native source-import equivalents. 178 def put_object(domain, data): 179 pin = hashlib.sha256(domain.encode() + b'\0' + data).hexdigest() 180 path = store / 'objects' / pin[:3] / pin 181 path.parent.mkdir(parents=True, exist_ok=True) 182 path.write_bytes(data) 183 return pin, path 184 185 tree_bytes = indexed_bytes(b'\0') 186 tree_pin, tree_path = put_object('arboricx.indexed-tree.v1', tree_bytes) 187 row = f'export\tvalue\tarboricx.indexed-tree.v1\t{tree_pin}\tarboricx.abi.tree.v1\n'.encode() 188 manifest_bytes = b'arboricx.module-manifest.v1\n' + row 189 provider_pin, provider_path = put_object('arboricx.module-manifest.v1', manifest_bytes) 190 importing = root / 'importing.tri' 191 192 def import_source(pin, ok=True): 193 importing.write_text(f'!import {pin} as D\n!export main\nmain = D.value\n') 194 return run('eval', importing, '--store', store, '--format', 'fsl', ok=ok) 195 196 assert import_source(provider_pin) == 'Leaf' 197 provider_path.write_bytes(manifest_bytes + b'corrupt') 198 assert 'CAS digest mismatch' in import_source(provider_pin, ok=False) 199 provider_path.write_bytes(manifest_bytes) 200 tree_path.write_bytes(b'\1\0') 201 assert 'CAS digest mismatch' in import_source(provider_pin, ok=False) 202 tree_path.write_bytes(tree_bytes) 203 outside = root / 'outside.tree' 204 outside.write_bytes(b'\0') 205 for data, message in [(manifest_bytes.replace(b'arboricx.indexed-tree.v1', b'arboricx.tree-term.v1'), 'unsupported export kind'), 206 (manifest_bytes + row, 'duplicate module export'), 207 (manifest_bytes.replace(b'arboricx.abi.tree.v1', b'wrong.abi'), 'unsupported export ABI'), 208 (manifest_bytes.replace(tree_pin.encode(), str(outside).encode()), 'invalid export name or digest')]: 209 invalid, _ = put_object('arboricx.module-manifest.v1', data) 210 assert message in import_source(invalid, ok=False) 211 assert outside.read_bytes() == b'\0' 212 213 (root / 'dep.tri').write_text('!export value\nvalue = 23\n') 214 transcript = run('--store', root / 'interactive', input='''!format number 215 x = 7 216 x 217 f ignored = x 218 f t 219 f "a=b" 220 x = 8 221 x 222 f t 223 x = unknown 224 x 225 f t 226 !import dep as D 227 D.value 228 !load dep.tri 229 value 230 !def first 231 first xs = case xs of 232 [h | _] = h 233 _ = 0 234 !end 235 first [31] 236 !expr 237 (let rec 238 walk [] = 19 239 walk [h | rest] = walk rest 240 walk _ = 0 241 in walk [1 2]) 242 !end 243 !import missing * 244 x 245 !load bad.tri 246 x 247 !reset 248 x 249 !exit 250 ''') 251 for expected in ['tricu < 7\n', 'tricu < 8\n', 'tricu < 23\n', 'tricu < 31\n', '19\n']: 252 assert expected in transcript, (expected, transcript) 253 assert transcript.count('tricu < 8\n') == 6, transcript 254 assert transcript.count('Error:') == 4, transcript 255 assert not list(root.glob('tricu_repl*.tri')), 'session temporary source leaked' 256 block_transcript = run('--store', root / 'blockstore', input='''!format number 257 x = 3 258 !def x 259 y = 7 260 !end 261 f ignored = x 262 x 263 f t 264 !env 265 !def x 266 x = 8 267 extra = 9 268 !end 269 x 270 f t 271 !env 272 !def x 273 -- Leading comments belong to the submitted source. 274 x = 11 275 !end 276 x 277 f t 278 !env 279 !exit 280 ''') 281 assert block_transcript.count('Definition block must define exactly its named equation group') == 2, block_transcript 282 assert block_transcript.count('Error:') == 2, block_transcript 283 assert block_transcript.count('tricu < 3\n') == 4, block_transcript 284 assert block_transcript.count('tricu < 11\n') == 2, block_transcript 285 assert block_transcript.count('tricu < f\nx\n') == 3, block_transcript 286 assert 'tricu < y\n' not in block_transcript and '\nextra\n' not in block_transcript 287 overridden = run('--store', store, 'repl', '--toolchain', tool_manifest, '--tool-store', tools, 288 input='!format number\n42\n!exit\n', env=broken) 289 assert '42\n' in overridden 290 print('default portable frontend: CLI, packaging, overrides, errors and transactional REPL passed')