source_order.py (12437B)
1 #!/usr/bin/env python3 2 """Opt-in artifact-only ordering regression. No compilation or generations. 3 4 Select full source_order and relinked source_stable candidate bundles explicitly. 5 Optionally select relinked source_unit to compare complete compiled outputs too. 6 Compares complete result trees (including diagnostic locations/remainders) with 7 retained reference workers. Every timed invocation is a separate process, never 8 --repeat. This checks selected artifacts, not their source freshness. 9 """ 10 import argparse 11 import hashlib 12 import json 13 from pathlib import Path 14 import re 15 import time 16 17 from bundle_dag import merge_bundles 18 from fixture_sources import verify 19 from process_rss import run_guarded 20 from rts_metrics import parse_rts_stats 21 from stable_bundle import byte_tree 22 23 ROOT = Path(__file__).resolve().parent.parent 24 25 26 def end(tree, at=0): 27 slots = 1 28 while slots: 29 tag = tree[at] 30 assert tag in (0, 1, 2), tag 31 slots += tag - 1 32 at += 1 33 return at 34 35 36 def field(tree, *path): 37 at = 0 38 for side in path: 39 assert tree[at] == 2 40 at += 1 41 if side: 42 at = end(tree, at) 43 return tree[at:end(tree, at)] 44 45 46 def number(tree): 47 value, bit = 0, 1 48 while tree != b'\0': 49 value += bit * (field(tree, 0) == b'\1\0') 50 bit *= 2 51 tree = field(tree, 1) 52 return value 53 54 55 def names(tree): 56 values = [] 57 while tree != b'\0': 58 text, chars = field(tree, 0), [] 59 while text != b'\0': 60 chars.append(chr(number(field(text, 0)))) 61 text = field(text, 1) 62 values.append(''.join(chars)) 63 tree = field(tree, 1) 64 return values 65 66 67 def cases(): 68 # name, source, expected gate error code (0 = success), stable error code, 69 # optional independently specified successful ordering. 70 return [ 71 ('empty', '', 0, 0, []), 72 ('independent', 'z = t\na = t\nm = t', 0, 0, ['z', 'a', 'm']), 73 ('forward', 'main = f\nf x = x', 0, 0, ['f', 'main']), 74 ('ready-ties', 'a = c\nb = t\nc = t\nd = b', 0, 0, ['b', 'c', 'a', 'd']), 75 ('repeated-edge', 'a = [b b b]\nb = t', 0, 0, ['b', 'a']), 76 ('external', 'main = outside', 0, 0, ['main']), 77 ('local-over-external', 'a = x\nx = t', 0, 0, ['x', 'a']), 78 ('lambda-shadow', 'x = t\nf x = x', 0, 0, ['x', 'f']), 79 ('let-shadow', 'main = let x = t in x\nx = t', 0, 0, ['main', 'x']), 80 ('let-value-scope', 'main = let x = x in x\nx = t', 0, 0, ['x', 'main']), 81 ('where', 'main = f t where f x = x', 0, 0, ['main']), 82 ('pipe', 'main = t |> f\nf x = x', 0, 0, ['f', 'main']), 83 ('do-scope', 'main = do bind\n x <- t\n x\nbind a f = f a', 0, 0, ['bind', 'main']), 84 ('pattern-body', 'f (t x) = helper x\nf _ = t\nhelper x = x', 0, 0, ['helper', 'f']), 85 ('whole-capture', 'f whole{t x y} = [whole x y]\nf _ = t', 0, 0, ['f']), 86 ('pin-dependency', 'f ^wanted = t\nf _ = t\nwanted = t', 0, 0, ['wanted', 'f']), 87 ('nested-pin', 'f (t x) = let\n g ^x = t\n g _ = t\nin g x\nf _ = t', 0, 0, ['f']), 88 ('unused-missing', 'main = t\nbroken = missing', 20, 20, None), 89 ('first-missing', 'a = [missing later]\nb = absent', 20, 20, None), 90 ('final-missing', 'a = t\nmissing', 20, 20, None), 91 ('definition-before-final', 'a = missing\nabsent', 20, 20, None), 92 ('duplicate-before-missing', 'a = missing\na = t', 30, 30, None), 93 ('duplicate-before-equations', 'f x x = x\na = t\na = t', 30, 30, None), 94 ('capture-before-missing', 'f x x = missing', 48, 48, None), 95 ('arity-before-missing', 'f t x = missing\nf _ = t', 48, 48, None), 96 ('catchall-before-missing', 'f t = missing', 48, 48, None), 97 ('earlier-missing-before-equations', 'a = missing\nf x x = x', 20, 20, None), 98 ('pin-collision', 'x = t\nf x ^x = x\nf _ _ = t', 48, 48, None), 99 ('missing-pin', 'f ^missing = t\nf _ = t', 20, 20, None), 100 ('wildcard-not-binder', 'f _ = _', 20, 20, None), 101 ('mutual-cycle', 'a = b\nb = a', 31, 31, None), 102 ('cycle-dependent', 'start = b\na = b\nb = a', 31, 31, None), 103 ('ordered-cycle-edges', 'start = [b a b]\na = b\nb = a', 31, 31, None), 104 ('final-before-cycle', 'a = a\nmissing', 20, 48, None), 105 ('recursive-value', 'a = a', 31, 48, None), 106 ('recursive-function', 'f x = f x', 31, 0, ['f']), 107 ('recursive-pattern', 'f t = t\nf (t x) = f x\nf _ = t', 31, 0, ['f']), 108 ('self-shadow', 'f f = f', 0, 0, ['f']), 109 ] 110 111 112 class Runner: 113 def __init__(self, executable, directory, *, memory_mib=8192): 114 if not 0 < memory_mib <= 65536: 115 raise ValueError("memory guard must be within 1..65536 MiB") 116 self.memory_mib = memory_mib 117 self.exe, self.root = executable.resolve(), directory.resolve() 118 self.store = self.root / 'store' 119 self.records = [] 120 self.started = time.monotonic() 121 122 def run(self, label, args, seconds=180): 123 assert time.monotonic() - self.started < 1800, '30-minute whole-run guard' 124 command = [str(self.exe), 'arboricx', *map(str, args), 125 '+RTS', '-N1', f'-M{self.memory_mib}m', '-T', '-s', '-RTS'] 126 start = time.monotonic() 127 try: 128 output, peak = run_guarded(command, memory_mib=self.memory_mib, seconds=seconds) 129 except Exception as error: 130 (self.root / (label + '.log')).write_text(str(getattr(error, 'output', error))) 131 raise 132 (self.root / (label + '.log')).write_text(output) 133 row = dict(label=label, seconds=time.monotonic()-start, peak_kib=peak, command=command) 134 samples = [json.loads(line) for line in output.splitlines() if line.startswith('{')] 135 intervals = [sample for sample in samples if sample.get('profile') in ('invoke-begin-1', 'observe-end-1')] 136 if len(intervals) == 2: 137 before, after = intervals 138 row['measurement_scope'] = 'invocation-through-rendered-output-v1' 139 row['invoke_seconds'] = (after['wall_ns'] - before['wall_ns']) / 1e9 140 row['allocated_bytes'] = after['allocated_bytes'] - before['allocated_bytes'] 141 row['rts_process'] = parse_rts_stats(output) 142 self.records.append(row) 143 (self.root / 'samples.json').write_text(json.dumps(self.records, indent=2) + '\n') 144 return output 145 146 def imported(self, path, label): 147 output = self.run(label, ['import', '--file', path, '--store', self.store, '--indexed']) 148 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 149 150 def invoke(self, label, pin, name, args): 151 output = self.run(label, ['run', '--module', pin, '--name', name, '--store', self.store, 152 '--format', 'ternary', '--profile', *args]) 153 text = next(re.match(r'^([012]+)', line)[1] for line in output.splitlines() 154 if re.match(r'^[012]+', line)) 155 raw = bytes(int(c) for c in text) 156 # Rendering has no trailing newline; RTS stderr can touch its last digit. 157 result = raw[:end(raw)] 158 (self.root / (label + '.tree')).write_bytes(result) 159 return result 160 161 162 def main(): 163 parser = argparse.ArgumentParser(description=__doc__) 164 parser.add_argument('tricu', type=Path) 165 parser.add_argument('--order-bundle', type=Path, required=True) 166 parser.add_argument('--stable-bundle', type=Path, required=True) 167 parser.add_argument('--unit-bundle', type=Path) 168 parser.add_argument('--directory', type=Path, required=True) 169 args = parser.parse_args() 170 root = args.directory.resolve() 171 root.mkdir(parents=True, exist_ok=False) 172 verify(ROOT / 'test/fixtures') 173 reference = ROOT / 'test/fixtures/compiler-library.arboricx' 174 bundle = root / 'workers.arboricx' 175 selections = [ 176 (reference, [('source_order.orderGateProgram', 'refProgram'), 177 ('source_order.orderGateSource', 'refSource'), 178 ('source_order.sourceCheckDefinitionNames', 'refNames'), 179 ('source_order.sourceDefinitionGraph', 'refGraph'), 180 ('source_stable.orderStableProgram', 'refStable'), 181 ('source_program.parseGateProgram', 'parse')]), 182 (args.order_bundle, [('orderGateProgram', 'newProgram'), ('orderGateSource', 'newSource'), 183 ('sourceAnalyzeDefinitions', 'newAnalysis')]), 184 (args.stable_bundle, [('orderStableProgram', 'newStable')]), 185 ] 186 if args.unit_bundle: 187 selections += [(reference, [('source_unit.compileStableModuleDefinitions', 'refCompile')]), 188 (args.unit_bundle, [('compileStableModuleDefinitions', 'newCompile')])] 189 merge_bundles(bundle, selections, prune_unreachable=True) 190 runner = Runner(args.tricu, root) 191 pin = runner.imported(bundle, 'import') 192 env = root / 'environment.tree' 193 env.write_bytes(b''.join(b'\2\2' + byte_tree(name) + b'\0' for name in (b'outside', b'x')) + b'\0') 194 passed = [] 195 for label, source, gate_code, stable_code, ordered in cases(): 196 source_file = root / (label + '-source.tree') 197 source_file.write_bytes(byte_tree(source.encode())) 198 source_args = ['--arg-tree-file', env, '--arg-tree-file', source_file] 199 before = runner.invoke(label + '-reference', pin, 'refSource', source_args) 200 after = runner.invoke(label + '-candidate', pin, 'newSource', source_args) 201 assert before == after, (label, 'complete source-order result differs') 202 def check(result, code): 203 assert (field(result, 0) == b'\1\0') == (code == 0), (label, code) 204 if code: 205 assert number(field(result, 1, 0, 0)) == code, label 206 elif ordered is not None: 207 assert names(field(result, 1, 0)) == ordered, label 208 check(after, gate_code) 209 parsed = runner.invoke(label + '-parse', pin, 'parse', ['--arg-tree-file', source_file]) 210 assert field(parsed, 0) == b'\1\0', (label, 'test input did not parse') 211 program = root / (label + '-program.tree') 212 program.write_bytes(field(parsed, 1, 0)) 213 defs = root / (label + '-definitions.tree') 214 defs.write_bytes(field(program.read_bytes(), 0)) 215 program_args = ['--arg-tree-file', env, '--arg-tree-file', program] 216 before = runner.invoke(label + '-stable-reference', pin, 'refStable', program_args) 217 after = runner.invoke(label + '-stable-candidate', pin, 'newStable', program_args) 218 assert before == after, (label, 'complete stable-order result differs') 219 check(after, stable_code) 220 definition_args = ['--arg-tree-file', defs, '--arg-tree-file', env] 221 checked = runner.invoke(label + '-names', pin, 'refNames', definition_args) 222 analyzed = runner.invoke(label + '-analysis', pin, 'newAnalysis', definition_args) 223 if field(checked, 0) == b'\1\0': 224 graph = runner.invoke(label + '-graph', pin, 'refGraph', ['--arg-tree-file', defs]) 225 assert field(analyzed, 0) == b'\1\0' and field(analyzed, 1, 0) == graph, label 226 else: 227 assert analyzed == checked, (label, 'analysis diagnostic/remainder differs') 228 if args.unit_bundle: 229 before = runner.invoke(label + '-compile-reference', pin, 'refCompile', source_args) 230 after = runner.invoke(label + '-compile-candidate', pin, 'newCompile', source_args) 231 assert before == after, (label, 'complete stable compilation result differs') 232 passed.append(label) 233 report = dict(complete=True, cases=passed, seconds=time.monotonic()-runner.started, 234 reference_sha256=hashlib.sha256(reference.read_bytes()).hexdigest(), 235 order_sha256=hashlib.sha256(args.order_bundle.read_bytes()).hexdigest(), 236 stable_sha256=hashlib.sha256(args.stable_bundle.read_bytes()).hexdigest(), 237 host_sha256=hashlib.sha256(runner.exe.read_bytes()).hexdigest()) 238 if args.unit_bundle: 239 report['unit_sha256'] = hashlib.sha256(args.unit_bundle.read_bytes()).hexdigest() 240 report['compiled_result_comparisons'] = len(passed) 241 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 242 print(f'PASS: {len(passed)} complete gate/stable/analysis result comparisons' 243 + (' and compiled-result comparisons' if args.unit_bundle else '')) 244 245 246 if __name__ == '__main__': 247 main()