source_order.py (12344B)
1 #!/usr/bin/env python3 2 """Opt-in artifact-only ordering regression. No compilation or generations. 3 4 Select full source_order and relinked source_stable candidate bundles explicitly. 5 Optionally select relinked source_unit to compare complete compiled outputs too. 6 Compares complete result trees (including diagnostic locations/remainders) with 7 retained reference workers. Every timed invocation is a separate process, never 8 --repeat. This checks selected artifacts, not their source freshness. 9 """ 10 import argparse 11 import hashlib 12 import json 13 from pathlib import Path 14 import re 15 import time 16 17 from bundle_dag import merge_bundles 18 from fixture_sources import verify 19 from process_rss import run_guarded 20 from stable_bundle import byte_tree 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def end(tree, at=0): 26 slots = 1 27 while slots: 28 tag = tree[at] 29 assert tag in (0, 1, 2), tag 30 slots += tag - 1 31 at += 1 32 return at 33 34 35 def field(tree, *path): 36 at = 0 37 for side in path: 38 assert tree[at] == 2 39 at += 1 40 if side: 41 at = end(tree, at) 42 return tree[at:end(tree, at)] 43 44 45 def number(tree): 46 value, bit = 0, 1 47 while tree != b'\0': 48 value += bit * (field(tree, 0) == b'\1\0') 49 bit *= 2 50 tree = field(tree, 1) 51 return value 52 53 54 def names(tree): 55 values = [] 56 while tree != b'\0': 57 text, chars = field(tree, 0), [] 58 while text != b'\0': 59 chars.append(chr(number(field(text, 0)))) 60 text = field(text, 1) 61 values.append(''.join(chars)) 62 tree = field(tree, 1) 63 return values 64 65 66 def cases(): 67 # name, source, expected gate error code (0 = success), stable error code, 68 # optional independently specified successful ordering. 69 return [ 70 ('empty', '', 0, 0, []), 71 ('independent', 'z = t\na = t\nm = t', 0, 0, ['z', 'a', 'm']), 72 ('forward', 'main = f\nf x = x', 0, 0, ['f', 'main']), 73 ('ready-ties', 'a = c\nb = t\nc = t\nd = b', 0, 0, ['b', 'c', 'a', 'd']), 74 ('repeated-edge', 'a = [b b b]\nb = t', 0, 0, ['b', 'a']), 75 ('external', 'main = outside', 0, 0, ['main']), 76 ('local-over-external', 'a = x\nx = t', 0, 0, ['x', 'a']), 77 ('lambda-shadow', 'x = t\nf x = x', 0, 0, ['x', 'f']), 78 ('let-shadow', 'main = let x = t in x\nx = t', 0, 0, ['main', 'x']), 79 ('let-value-scope', 'main = let x = x in x\nx = t', 0, 0, ['x', 'main']), 80 ('where', 'main = f t where f x = x', 0, 0, ['main']), 81 ('pipe', 'main = t |> f\nf x = x', 0, 0, ['f', 'main']), 82 ('do-scope', 'main = do bind\n x <- t\n x\nbind a f = f a', 0, 0, ['bind', 'main']), 83 ('pattern-body', 'f (t x) = helper x\nf _ = t\nhelper x = x', 0, 0, ['helper', 'f']), 84 ('whole-capture', 'f whole{t x y} = [whole x y]\nf _ = t', 0, 0, ['f']), 85 ('pin-dependency', 'f ^wanted = t\nf _ = t\nwanted = t', 0, 0, ['wanted', 'f']), 86 ('nested-pin', 'f (t x) = let\n g ^x = t\n g _ = t\nin g x\nf _ = t', 0, 0, ['f']), 87 ('unused-missing', 'main = t\nbroken = missing', 20, 20, None), 88 ('first-missing', 'a = [missing later]\nb = absent', 20, 20, None), 89 ('final-missing', 'a = t\nmissing', 20, 20, None), 90 ('definition-before-final', 'a = missing\nabsent', 20, 20, None), 91 ('duplicate-before-missing', 'a = missing\na = t', 30, 30, None), 92 ('duplicate-before-equations', 'f x x = x\na = t\na = t', 30, 30, None), 93 ('capture-before-missing', 'f x x = missing', 48, 48, None), 94 ('arity-before-missing', 'f t x = missing\nf _ = t', 48, 48, None), 95 ('catchall-before-missing', 'f t = missing', 48, 48, None), 96 ('earlier-missing-before-equations', 'a = missing\nf x x = x', 20, 20, None), 97 ('pin-collision', 'x = t\nf x ^x = x\nf _ _ = t', 48, 48, None), 98 ('missing-pin', 'f ^missing = t\nf _ = t', 20, 20, None), 99 ('wildcard-not-binder', 'f _ = _', 20, 20, None), 100 ('mutual-cycle', 'a = b\nb = a', 31, 31, None), 101 ('cycle-dependent', 'start = b\na = b\nb = a', 31, 31, None), 102 ('ordered-cycle-edges', 'start = [b a b]\na = b\nb = a', 31, 31, None), 103 ('final-before-cycle', 'a = a\nmissing', 20, 48, None), 104 ('recursive-value', 'a = a', 31, 48, None), 105 ('recursive-function', 'f x = f x', 31, 0, ['f']), 106 ('recursive-pattern', 'f t = t\nf (t x) = f x\nf _ = t', 31, 0, ['f']), 107 ('self-shadow', 'f f = f', 0, 0, ['f']), 108 ] 109 110 111 class Runner: 112 def __init__(self, executable, directory, *, memory_mib=8192): 113 if not 0 < memory_mib <= 65536: 114 raise ValueError("memory guard must be within 1..65536 MiB") 115 self.memory_mib = memory_mib 116 self.exe, self.root = executable.resolve(), directory.resolve() 117 self.store = self.root / 'store' 118 self.records = [] 119 self.started = time.monotonic() 120 121 def run(self, label, args, seconds=180): 122 assert time.monotonic() - self.started < 1800, '30-minute whole-run guard' 123 command = [str(self.exe), 'arboricx', *map(str, args), 124 '+RTS', '-N1', f'-M{self.memory_mib}m', '-T', '-s', '-RTS'] 125 start = time.monotonic() 126 try: 127 output, peak = run_guarded(command, memory_mib=self.memory_mib, seconds=seconds) 128 except Exception as error: 129 (self.root / (label + '.log')).write_text(str(getattr(error, 'output', error))) 130 raise 131 (self.root / (label + '.log')).write_text(output) 132 row = dict(label=label, seconds=time.monotonic()-start, peak_kib=peak, command=command) 133 samples = [json.loads(line) for line in output.splitlines() if line.startswith('{')] 134 intervals = [sample for sample in samples if sample.get('profile') in ('invoke-begin-1', 'observe-end-1')] 135 if len(intervals) == 2: 136 before, after = intervals 137 row['measurement_scope'] = 'invocation-through-rendered-output-v1' 138 row['invoke_seconds'] = (after['wall_ns'] - before['wall_ns']) / 1e9 139 row['allocated_bytes'] = after['allocated_bytes'] - before['allocated_bytes'] 140 self.records.append(row) 141 (self.root / 'samples.json').write_text(json.dumps(self.records, indent=2) + '\n') 142 return output 143 144 def imported(self, path, label): 145 output = self.run(label, ['import', '--file', path, '--store', self.store, '--indexed']) 146 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 147 148 def invoke(self, label, pin, name, args): 149 output = self.run(label, ['run', '--module', pin, '--name', name, '--store', self.store, 150 '--format', 'ternary', '--profile', *args]) 151 text = next(re.match(r'^([012]+)', line)[1] for line in output.splitlines() 152 if re.match(r'^[012]+', line)) 153 raw = bytes(int(c) for c in text) 154 # Rendering has no trailing newline; RTS stderr can touch its last digit. 155 result = raw[:end(raw)] 156 (self.root / (label + '.tree')).write_bytes(result) 157 return result 158 159 160 def main(): 161 parser = argparse.ArgumentParser(description=__doc__) 162 parser.add_argument('tricu', type=Path) 163 parser.add_argument('--order-bundle', type=Path, required=True) 164 parser.add_argument('--stable-bundle', type=Path, required=True) 165 parser.add_argument('--unit-bundle', type=Path) 166 parser.add_argument('--directory', type=Path, required=True) 167 args = parser.parse_args() 168 root = args.directory.resolve() 169 root.mkdir(parents=True, exist_ok=False) 170 verify(ROOT / 'test/fixtures') 171 reference = ROOT / 'test/fixtures/compiler-library.arboricx' 172 bundle = root / 'workers.arboricx' 173 selections = [ 174 (reference, [('source_order.orderGateProgram', 'refProgram'), 175 ('source_order.orderGateSource', 'refSource'), 176 ('source_order.sourceCheckDefinitionNames', 'refNames'), 177 ('source_order.sourceDefinitionGraph', 'refGraph'), 178 ('source_stable.orderStableProgram', 'refStable'), 179 ('source_program.parseGateProgram', 'parse')]), 180 (args.order_bundle, [('orderGateProgram', 'newProgram'), ('orderGateSource', 'newSource'), 181 ('sourceAnalyzeDefinitions', 'newAnalysis')]), 182 (args.stable_bundle, [('orderStableProgram', 'newStable')]), 183 ] 184 if args.unit_bundle: 185 selections += [(reference, [('source_unit.compileStableModuleDefinitions', 'refCompile')]), 186 (args.unit_bundle, [('compileStableModuleDefinitions', 'newCompile')])] 187 merge_bundles(bundle, selections, prune_unreachable=True) 188 runner = Runner(args.tricu, root) 189 pin = runner.imported(bundle, 'import') 190 env = root / 'environment.tree' 191 env.write_bytes(b''.join(b'\2\2' + byte_tree(name) + b'\0' for name in (b'outside', b'x')) + b'\0') 192 passed = [] 193 for label, source, gate_code, stable_code, ordered in cases(): 194 source_file = root / (label + '-source.tree') 195 source_file.write_bytes(byte_tree(source.encode())) 196 source_args = ['--arg-tree-file', env, '--arg-tree-file', source_file] 197 before = runner.invoke(label + '-reference', pin, 'refSource', source_args) 198 after = runner.invoke(label + '-candidate', pin, 'newSource', source_args) 199 assert before == after, (label, 'complete source-order result differs') 200 def check(result, code): 201 assert (field(result, 0) == b'\1\0') == (code == 0), (label, code) 202 if code: 203 assert number(field(result, 1, 0, 0)) == code, label 204 elif ordered is not None: 205 assert names(field(result, 1, 0)) == ordered, label 206 check(after, gate_code) 207 parsed = runner.invoke(label + '-parse', pin, 'parse', ['--arg-tree-file', source_file]) 208 assert field(parsed, 0) == b'\1\0', (label, 'test input did not parse') 209 program = root / (label + '-program.tree') 210 program.write_bytes(field(parsed, 1, 0)) 211 defs = root / (label + '-definitions.tree') 212 defs.write_bytes(field(program.read_bytes(), 0)) 213 program_args = ['--arg-tree-file', env, '--arg-tree-file', program] 214 before = runner.invoke(label + '-stable-reference', pin, 'refStable', program_args) 215 after = runner.invoke(label + '-stable-candidate', pin, 'newStable', program_args) 216 assert before == after, (label, 'complete stable-order result differs') 217 check(after, stable_code) 218 definition_args = ['--arg-tree-file', defs, '--arg-tree-file', env] 219 checked = runner.invoke(label + '-names', pin, 'refNames', definition_args) 220 analyzed = runner.invoke(label + '-analysis', pin, 'newAnalysis', definition_args) 221 if field(checked, 0) == b'\1\0': 222 graph = runner.invoke(label + '-graph', pin, 'refGraph', ['--arg-tree-file', defs]) 223 assert field(analyzed, 0) == b'\1\0' and field(analyzed, 1, 0) == graph, label 224 else: 225 assert analyzed == checked, (label, 'analysis diagnostic/remainder differs') 226 if args.unit_bundle: 227 before = runner.invoke(label + '-compile-reference', pin, 'refCompile', source_args) 228 after = runner.invoke(label + '-compile-candidate', pin, 'newCompile', source_args) 229 assert before == after, (label, 'complete stable compilation result differs') 230 passed.append(label) 231 report = dict(complete=True, cases=passed, seconds=time.monotonic()-runner.started, 232 reference_sha256=hashlib.sha256(reference.read_bytes()).hexdigest(), 233 order_sha256=hashlib.sha256(args.order_bundle.read_bytes()).hexdigest(), 234 stable_sha256=hashlib.sha256(args.stable_bundle.read_bytes()).hexdigest(), 235 host_sha256=hashlib.sha256(runner.exe.read_bytes()).hexdigest()) 236 if args.unit_bundle: 237 report['unit_sha256'] = hashlib.sha256(args.unit_bundle.read_bytes()).hexdigest() 238 report['compiled_result_comparisons'] = len(passed) 239 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 240 print(f'PASS: {len(passed)} complete gate/stable/analysis result comparisons' 241 + (' and compiled-result comparisons' if args.unit_bundle else '')) 242 243 244 if __name__ == '__main__': 245 main()