native_source_utf8.py (10485B)
1 #!/usr/bin/env python3 2 """Opt-in source-boundary candidate check; never publishes fixtures/generations. 3 4 Compile current build/map decoding consumers and IO constructors against explicitly 5 selected retained support. The support is executable rollback, not current-source 6 compiler acceptance. Every smoke uses the edited Haskell host and fresh files. 7 """ 8 import argparse 9 import hashlib 10 import json 11 from pathlib import Path 12 import re 13 14 from collections_batch import ProbeRunner 15 from http_headers import LEAF, byte_tree, pair, result, number, tree_list 16 17 ROOT = Path(__file__).resolve().parent.parent 18 19 20 def main(): 21 p = argparse.ArgumentParser(description=__doc__) 22 p.add_argument('tricu', type=Path) 23 p.add_argument('--generation', type=Path, required=True) 24 p.add_argument('--compiler', type=Path, required=True) 25 p.add_argument('--directory', type=Path, required=True) 26 args = p.parse_args() 27 root = args.directory.resolve() 28 root.mkdir(parents=True) 29 runner = ProbeRunner(args.tricu, root, 1500, memory_mib=8192) 30 old = args.generation.resolve() / 'cas' 31 parts = [] 32 for name in ['build', 'map']: 33 text = (ROOT / 'lib/source' / (name + '.tri')).read_text() 34 if name == 'build': 35 text = text[text.index('sourceBuildStage source'):text.index('-- Lock rewriting')] + text[text.index('sourceBuildReadTriPath path'):text.index('sourceBuildReadTri workspace')] 36 else: 37 text = text[:text.index('sourceMapStageRows rows')] 38 parts.append('\n'.join(line for line in text.splitlines() 39 if not line.startswith(('!import', '!export')))) 40 body = '\n'.join(parts) 41 constructors = '\n'.join(line for line in (ROOT / 'lib/io.tri').read_text().splitlines() 42 if line.startswith(('readSourceFile =', 'encodeSourceText ='))) 43 body = constructors + '\n' + body 44 local = set(re.findall(r'^([A-Za-z_][A-Za-z0-9_?\-]*)[^\n=]*=', body, re.M)) 45 wrapper = ''' 46 readProbe path = io (sourceBuildReadTriPath path) 47 mapProbe sourceRoot overlayRoot row = io (sourceMapStageOne sourceRoot overlayRoot [] row) 48 lexProbe path = io (onOk_ (readSourceFile path) (source : pure (lexGateSource (snd source)))) 49 ''' 50 words = set(re.findall(r'[A-Za-z_][A-Za-z0-9_?\-]*', body + wrapper)) 51 selected = {} 52 modules = ['prelude', 'io', 'source_object', 'source_module', 'source_lock', 53 'source_bundle', 'source_resolve', 'source_unit', 'source_lexer', 'parser'] 54 for module in modules: 55 pin = (old / 'cache/source-build' / module).read_text().split('\t')[2] 56 raw = (old / 'objects' / pin[:3] / pin).read_bytes() 57 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + raw).hexdigest() == pin 58 for row in raw.decode().splitlines()[1:]: 59 _, name, _, tree, _ = row.split('\t') 60 if name in words and name not in local: 61 selected[name] = tree 62 command = ['export', '--store', old, '--output', root / 'support.arboricx'] 63 for name, tree in selected.items(): 64 command += ['--target', tree, '--name', name] 65 runner.run('support-export', command, 180) 66 from bundle_dag import read 67 _, _, nodes, exports, _ = read(root / 'support.arboricx') 68 assert set(exports) == set(selected), 'complete support export set' 69 sizes = [] 70 for node in nodes: 71 sizes.append(1 + sum(sizes[child] for child in node[1:])) 72 expanded_bytes = sum(sizes[index] for index in exports.values()) 73 support_reply = runner.run('support-import', ['import', '--file', root / 'support.arboricx', '--store', runner.store, '--indexed'], 180) 74 support = re.search(r'Created module alias .* -> ([0-9a-f]{64})', support_reply)[1] 75 manifest = (runner.store / 'objects' / support[:3] / support).read_bytes() 76 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + manifest).hexdigest() == support 77 entries = [line.split('\t') for line in manifest.decode().splitlines()[1:]] 78 assert {fields[1]: fields[3] for fields in entries} == selected 79 assert len(entries) == len(selected) 80 from executable_object import verify_payload 81 for _, name, kind, tree, abi in entries: 82 raw = (runner.store / 'objects' / tree[:3] / tree).read_bytes() 83 verify_payload(kind, raw) 84 assert hashlib.sha256(kind.encode() + b'\0' + raw).hexdigest() == tree 85 (root / 'support.tri').write_text(f'!export * from {support}\n') 86 source = '!import support *\n!export readProbe\n!export mapProbe\n!export lexProbe\n' + body + wrapper 87 (root / 'driver.tri').write_text(source) 88 compiler = runner.imported(args.compiler, 'compiler-import') 89 runner.run('compile', ['compile', '--file', root / 'driver.tri', '--store', runner.store, 90 '--toolchain', compiler, '--tool-store', runner.store, 91 '--output', root / 'driver.arboricx', '--compile-timeout', '900'], 900) 92 pin = runner.imported(root / 'driver.arboricx', 'driver-import') 93 source_root, overlay = root / 'input', root / 'overlay' 94 source_root.mkdir(); overlay.mkdir() 95 checks = [] 96 97 def invoke(label, name, values): 98 argv = ['--io', '--allow-read', root, '--allow-write', root] 99 for i, value in enumerate(values): 100 path = root / f'arg-{i}.tree' 101 path.write_bytes(value) 102 argv += ['--arg-tree-file', path] 103 # ProbeRunner's default is intentionally overridden to the smoke budget. 104 output = runner.run(label, ['run', '--module', pin, '--name', name, 105 '--store', runner.store, '--format', 'ternary', *argv], 180) 106 from source_order import end 107 raw = bytes(int(c) for c in re.search(r'(?m)^([012]+)', output)[1]) 108 return raw[:end(raw)] 109 110 def chars(text): 111 return tree_list([number(ord(c)) for c in text]) 112 113 def path_tree(path): 114 return chars(str(path)) 115 116 for label, text in [('ascii', '!export main\nmain = 42\n'), 117 ('unicode', '!export main\n-- é€😀\nmain = "é€😀"\n'), 118 ('normalization', '!export main\nmain = "e\u0301é\ufeff"\r\n')]: 119 raw = text.encode('utf-8') 120 path = source_root / 'sample.tri' 121 path.write_bytes(raw) 122 # read/stage's public result: raw bytes, dependencies, refs, scalar source. 123 expected = result(True, pair(byte_tree(raw), pair(LEAF, pair(LEAF, chars(text))))) 124 assert invoke(label + '-read', 'readProbe', [path_tree(path)]) == expected 125 row = pair(chars('sample'), pair(chars('sample'), pair(chars('sample.tri'), pair(LEAF, chars('native'))))) 126 actual = invoke(label + '-map', 'mapProbe', [path_tree(source_root), path_tree(overlay), row]) 127 digest = hashlib.sha256(b'arboricx.source.v1\0' + raw).hexdigest() 128 assert actual == result(True, pair(chars('sample'), pair(chars(digest), chars(digest)))) 129 assert (overlay / 'sample.tri').read_bytes() == raw 130 checks.append(label + ': raw read, native map bytes and domain fingerprints') 131 132 # Legacy mapped headers are transformed as scalar text then encoded once. 133 text = '-- é€😀\nmain = "é€😀"\n' 134 path.write_bytes(text.encode()) 135 row = pair(chars('sample'), pair(chars('sample'), pair(chars('sample.tri'), pair(LEAF, chars('unit'))))) 136 actual = invoke('legacy-map', 'mapProbe', [path_tree(source_root), path_tree(overlay), row]) 137 # Existing legacy lines/unlines preserves the final empty line with an extra LF. 138 expected_overlay = ('!export *\n' + text + '\n').encode() 139 raw_digest = hashlib.sha256(b'arboricx.source.v1\0' + text.encode()).hexdigest() 140 overlay_digest = hashlib.sha256(b'arboricx.source.v1\0' + expected_overlay).hexdigest() 141 assert actual == result(True, pair(chars('sample'), pair(chars(raw_digest), chars(overlay_digest)))) 142 assert (overlay / 'sample.tri').read_bytes() == expected_overlay 143 checks.append('legacy scalar overlay re-encoded as UTF-8') 144 145 # Token coordinates after multibyte string and comment count scalars, not octets. 146 text = '"é€😀" -- é€😀\nt' 147 path.write_bytes(text.encode()) 148 def pos(line, column, offset): 149 return pair(number(line), pair(number(column), number(offset))) 150 def token(kind, payload, start, stop): 151 return pair(number(kind), pair(payload, pair(start, stop))) 152 tokens = tree_list([token(8, chars('é€😀'), pos(1, 1, 0), pos(1, 6, 5)), 153 token(6, LEAF, pos(1, 13, 12), pos(2, 1, 13)), 154 token(1, chars('t'), pos(2, 1, 13), pos(2, 2, 14))]) 155 expected = result(True, tokens, pair(LEAF, pair(b'\1\0', LEAF))) 156 assert invoke('scalar-locations', 'lexProbe', [path_tree(path)]) == expected 157 checks.append('decoded string/comment scalar token coordinates') 158 159 # A non-ASCII identifier must still be rejected by the ASCII grammar. 160 path.write_bytes('é'.encode()) 161 assert invoke('ascii-identifiers', 'lexProbe', [path_tree(path)]).startswith(b'\2\0\2') 162 checks.append('identifier grammar remains ASCII') 163 164 for label, raw in [('overlong', b'\xc0\xaf'), ('truncated', b'\xf0\x9f\x98'), 165 ('surrogate', b'\xed\xa0\x80'), ('range', b'\xf4\x90\x80\x80'), 166 ('bom', b'\xef\xbb\xbf!export main\nmain = 0\n')]: 167 path.write_bytes(raw) 168 before = (overlay / 'sample.tri').read_bytes() 169 for name, values in [('readProbe', [path_tree(path)]), 170 ('mapProbe', [path_tree(source_root), path_tree(overlay), row])]: 171 actual = invoke(label + '-' + name, name, values) 172 assert actual.startswith(b'\2\0\2'), (label, name, actual) 173 assert (overlay / 'sample.tri').read_bytes() == before 174 checks.append(label + ': read and map rejected without overlay writes') 175 report = dict(complete=True, checks=checks, compiler_sha256=hashlib.sha256(args.compiler.read_bytes()).hexdigest(), 176 source_sha256=hashlib.sha256(source.encode()).hexdigest(), support=selected, 177 support_expanded_bytes=expanded_bytes, support_manifest=support, 178 limits=dict(memory_mib=8192, job_seconds=1500, compile_seconds=900, smoke_seconds=180), 179 scope='candidate read/stage/map boundary; not generation acceptance') 180 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 181 print(json.dumps(checks, indent=2)) 182 183 184 if __name__ == '__main__': 185 main()