native_source_utf8.py (7105B)
1 #!/usr/bin/env python3 2 """Current IO source-boundary check with independent byte/token oracles. 3 4 Compile complete workspace modules through the selected portable compiler, not 5 source-spliced consumers bound to retained v1 support. No fixture publication. 6 """ 7 import argparse 8 import hashlib 9 import json 10 from pathlib import Path 11 import re 12 13 from collections_batch import ProbeRunner 14 from http_headers import LEAF, byte_tree, pair, result, number, tree_list 15 16 ROOT = Path(__file__).resolve().parent.parent 17 18 19 def main(): 20 p = argparse.ArgumentParser(description=__doc__) 21 p.add_argument('tricu', type=Path) 22 p.add_argument('--compiler', type=Path, required=True) 23 p.add_argument('--directory', type=Path, required=True) 24 args = p.parse_args() 25 root = args.directory.resolve() 26 root.mkdir(parents=True) 27 runner = ProbeRunner(args.tricu, root, 1500, memory_mib=8192) 28 source = """!import prelude * 29 !import compiler_io * 30 !import source_build * 31 !import source_map * 32 !import source_lexer * 33 !export readProbe 34 !export mapProbe 35 !export lexProbe 36 readProbe path = io (sourceBuildReadTriPath path) 37 mapProbe sourceRoot overlayRoot row = io (sourceMapStageOne sourceRoot overlayRoot [] row) 38 lexProbe path = io (onOk_ (readSourceFile path) (source : pure (lexGateSource (snd source)))) 39 """ 40 (root / 'driver.tri').write_text(source) 41 compiler = runner.imported(args.compiler, 'compiler-import') 42 runner.run('compile', ['compile', '--file', root / 'driver.tri', '--store', runner.store, 43 '--toolchain', compiler, '--tool-store', runner.store, 44 '--output', root / 'driver.arboricx', '--compile-timeout', '900'], 900) 45 pin = runner.imported(root / 'driver.arboricx', 'driver-import') 46 source_root, overlay = root / 'input', root / 'overlay' 47 source_root.mkdir(); overlay.mkdir() 48 checks = [] 49 50 def invoke(label, name, values): 51 argv = ['--io'] 52 for i, value in enumerate(values): 53 path = root / f'arg-{i}.tree' 54 path.write_bytes(value) 55 argv += ['--arg-tree-file', path] 56 # ProbeRunner's default is intentionally overridden to the smoke budget. 57 output = runner.run(label, ['run', '--module', pin, '--name', name, 58 '--store', runner.store, '--format', 'ternary', *argv], 180) 59 from source_order import end 60 raw = bytes(int(c) for c in re.search(r'(?m)^([012]+)', output)[1]) 61 return raw[:end(raw)] 62 63 def chars(text): 64 return tree_list([number(ord(c)) for c in text]) 65 66 def path_tree(path): 67 return chars(str(path)) 68 69 for label, text in [('ascii', '!export main\nmain = 42\n'), 70 ('unicode', '!export main\n-- é€😀\nmain = "é€😀"\n'), 71 ('normalization', '!export main\nmain = "e\u0301é\ufeff"\r\n')]: 72 raw = text.encode('utf-8') 73 path = source_root / 'sample.tri' 74 path.write_bytes(raw) 75 # read/stage's public result: raw bytes, dependencies, refs, scalar source. 76 expected = result(True, pair(byte_tree(raw), pair(LEAF, pair(LEAF, chars(text))))) 77 assert invoke(label + '-read', 'readProbe', [path_tree(path)]) == expected 78 row = pair(chars('sample'), pair(chars('sample'), pair(chars('sample.tri'), pair(LEAF, chars('native'))))) 79 actual = invoke(label + '-map', 'mapProbe', [path_tree(source_root), path_tree(overlay), row]) 80 digest = hashlib.sha256(b'arboricx.source.v1\0' + raw).hexdigest() 81 assert actual == result(True, pair(chars('sample'), pair(chars(digest), chars(digest)))) 82 assert (overlay / 'sample.tri').read_bytes() == raw 83 checks.append(label + ': raw read, native map bytes and domain fingerprints') 84 85 # Legacy mapped headers are transformed as scalar text then encoded once. 86 text = '-- é€😀\nmain = "é€😀"\n' 87 path.write_bytes(text.encode()) 88 row = pair(chars('sample'), pair(chars('sample'), pair(chars('sample.tri'), pair(LEAF, chars('unit'))))) 89 actual = invoke('legacy-map', 'mapProbe', [path_tree(source_root), path_tree(overlay), row]) 90 # Existing legacy lines/unlines preserves the final empty line with an extra LF. 91 expected_overlay = ('!export *\n' + text + '\n').encode() 92 raw_digest = hashlib.sha256(b'arboricx.source.v1\0' + text.encode()).hexdigest() 93 overlay_digest = hashlib.sha256(b'arboricx.source.v1\0' + expected_overlay).hexdigest() 94 assert actual == result(True, pair(chars('sample'), pair(chars(raw_digest), chars(overlay_digest)))) 95 assert (overlay / 'sample.tri').read_bytes() == expected_overlay 96 checks.append('legacy scalar overlay re-encoded as UTF-8') 97 98 # Token coordinates after multibyte string and comment count scalars, not octets. 99 text = '"é€😀" -- é€😀\nt' 100 path.write_bytes(text.encode()) 101 def pos(line, column, offset): 102 return pair(number(line), pair(number(column), number(offset))) 103 def token(kind, payload, start, stop): 104 return pair(number(kind), pair(payload, pair(start, stop))) 105 tokens = tree_list([token(8, chars('é€😀'), pos(1, 1, 0), pos(1, 6, 5)), 106 token(6, LEAF, pos(1, 13, 12), pos(2, 1, 13)), 107 token(1, chars('t'), pos(2, 1, 13), pos(2, 2, 14))]) 108 expected = result(True, tokens, pair(LEAF, pair(b'\1\0', LEAF))) 109 assert invoke('scalar-locations', 'lexProbe', [path_tree(path)]) == expected 110 checks.append('decoded string/comment scalar token coordinates') 111 112 # A non-ASCII identifier must still be rejected by the ASCII grammar. 113 path.write_bytes('é'.encode()) 114 assert invoke('ascii-identifiers', 'lexProbe', [path_tree(path)]).startswith(b'\2\0\2') 115 checks.append('identifier grammar remains ASCII') 116 117 for label, raw in [('overlong', b'\xc0\xaf'), ('truncated', b'\xf0\x9f\x98'), 118 ('surrogate', b'\xed\xa0\x80'), ('range', b'\xf4\x90\x80\x80'), 119 ('bom', b'\xef\xbb\xbf!export main\nmain = 0\n')]: 120 path.write_bytes(raw) 121 before = (overlay / 'sample.tri').read_bytes() 122 for name, values in [('readProbe', [path_tree(path)]), 123 ('mapProbe', [path_tree(source_root), path_tree(overlay), row])]: 124 actual = invoke(label + '-' + name, name, values) 125 assert actual.startswith(b'\2\0\2'), (label, name, actual) 126 assert (overlay / 'sample.tri').read_bytes() == before 127 checks.append(label + ': read and map rejected without overlay writes') 128 report = dict(complete=True, checks=checks, compiler_sha256=hashlib.sha256(args.compiler.read_bytes()).hexdigest(), 129 source_sha256=hashlib.sha256(source.encode()).hexdigest(), io_version=2, 130 limits=dict(memory_mib=8192, job_seconds=1500, compile_seconds=900, smoke_seconds=180), 131 scope='candidate read/stage/map boundary; not generation acceptance') 132 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 133 print(json.dumps(checks, indent=2)) 134 135 136 if __name__ == '__main__': 137 main()