native_math_performance.py (10632B)
1 """Bounded performance/consumer phase of native_math.py; not a standalone runner.""" 2 import hashlib 3 import json 4 import re 5 import statistics 6 7 8 def probe(root, repo, generation, tool, candidate, run, import_bundle, number, boolean, tree_list, counter=None): 9 summaries = [] 10 counts = [] 11 cas = generation / 'cas' 12 if (cas / 'cache/source-build/native_math').exists(): 13 raise ValueError('performance probes require a pre-adoption baseline generation; ' 14 'the canonical source lexer already uses native math') 15 tools = root / 'tools' 16 app = root / 'application' 17 18 def exports(module): 19 manifest = (cas / 'cache/source-build' / module).read_text().split('\t')[2] 20 data = (cas / 'objects' / manifest[:3] / manifest).read_bytes() 21 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + data).hexdigest() == manifest 22 return {f[1]: f[3] for f in (line.split('\t') for line in data.decode().splitlines()[1:])} 23 24 def pack(name, entries): 25 artifact = root / (name + '.arboricx') 26 run(['arboricx', 'export', '--store', cas, '--output', artifact, 27 *[arg for key, value in entries.items() for arg in ['--target', value, '--name', key]], 28 '+RTS', '-N1', '-RTS']) 29 return import_bundle(artifact, tools) 30 31 def invoke(label, manifest, name, arguments, expected=None, counted=False): 32 command = ['arboricx', 'run', '--store', tools, '--module', manifest, '--name', name, 33 '--format', 'ternary', '--profile'] 34 for i, payload in enumerate(arguments): 35 path = root / f'{label}-{i}.tree' 36 path.write_bytes(payload) 37 command += ['--arg-tree-file', path] 38 # Fresh process each time: --repeat would reuse the already evaluated application. 39 output = run([*command, '+RTS', '-N1', '-T', '-RTS'], executable=counter if counted else None) 40 if expected is not None: 41 assert output == ''.join(str(b) for b in expected), label 42 measurements = json.loads((root / 'measurements.json').read_text()) 43 if counted: 44 applications = json.loads(next(line.removeprefix('REDUCER_COUNTS ') for line in 45 measurements[-1]['stderr'].splitlines() if line.startswith('REDUCER_COUNTS '))) 46 counts.append(dict(label=label, cases=applications, total=sum(applications), 47 output_sha256=hashlib.sha256(output.encode()).hexdigest())) 48 (root / 'counts.json').write_text(json.dumps(counts, indent=2) + '\n') 49 return output 50 samples = [json.loads(line) for line in measurements[-1]['stderr'].splitlines() if line.startswith('{')] 51 begin = next(s for s in samples if s['profile'] == 'invoke-begin-1') 52 end = next(s for s in samples if s['profile'] == 'observe-end-1') 53 summaries.append(dict(measurement_scope='invocation-through-rendered-output-v1', label=label, name=name, wall_ns=end['wall_ns'] - begin['wall_ns'], 54 allocated_bytes=end['allocated_bytes'] - begin['allocated_bytes'], 55 output_sha256=hashlib.sha256(output.encode()).hexdigest())) 56 (root / 'performance.json').write_text(json.dumps(summaries, indent=2) + '\n') 57 return output 58 59 base = exports('prelude') 60 lexer = exports('source_lexer') 61 baseline = pack('baseline', {name: (lexer if name.startswith('source') else base)[name] for name in 62 ['sourceAsciiLetter?', 'sourceAsciiDigit?', 'lte_?', 'add', 'sub', 'mul', 'succ', 'pred', 'equal?']}) 63 cases = [ 64 ('letter-a', 'sourceAsciiLetter?', 'letter', [number(97)], boolean(True)), 65 ('digit-a', 'sourceAsciiDigit?', 'digit', [number(97)], boolean(False)), 66 ('lte-255', 'lte_?', 'lte', [number(255), number(255)], boolean(True)), 67 ('add-255', 'add', 'add', [number(255), number(255)], number(510)), 68 ('sub-255', 'sub', 'subtract', [number(255), number(127)], number(128)), 69 ('mul-31', 'mul', 'multiply', [number(31), number(10)], number(310)), 70 ('succ-255', 'succ', 'successor', [number(255)], number(256)), 71 ('pred-256', 'pred', 'predecessor', [number(256)], number(255)), 72 ('equal-255', 'equal?', 'equal', [number(255), number(255)], boolean(True)), 73 ] 74 for label, old, new, arguments, expected in cases: 75 for repetition in range(3): 76 variants = [('baseline', baseline, old), ('candidate', candidate, new)] 77 if repetition & 1: 78 variants.reverse() 79 for variant, manifest, name in variants: 80 invoke(f'{label}-{variant}', manifest, name, arguments, expected) 81 if counter: 82 for variant, manifest, name in variants: 83 invoke(f'{label}-{variant}', manifest, name, arguments, expected, counted=True) 84 85 # Compile an actual copied source lexer. Only classifiers/decimal accumulation 86 # differ; helpers and compiler artifact are identical in the two variants. 87 original = (repo / 'lib/source/lexer.tri').read_text() 88 body = '\n'.join(line for line in original.splitlines() if not line.startswith('!')) 89 # Retain the explicit decrement-based baseline after canonical lexer wiring. 90 # Never accidentally benchmark the new math against itself. 91 if 'N.letter' in body: 92 start = body.index('sourceAsciiLetter? character =') 93 end = body.index('sourceNameStart? character =') 94 body = (body[:start] + 95 'sourceAsciiLetter? character =\n' 96 ' or? (and? (not? (lt_? character 65)) (lte_? character 90))\n' 97 ' (and? (not? (lt_? character 97)) (lte_? character 122))\n' 98 'sourceAsciiDigit? character =\n' 99 ' and? (not? (lt_? character 48)) (lte_? character 57)\n\n' + body[end:]) 100 body = body.replace('(N.subtract first 48)', '(sub first 48)').replace( 101 '(value character : N.decimal value (N.subtract character 48))', 102 '(value character : add (mul value 10) (sub character 48))') 103 names = set(re.findall(r"[A-Za-z_][A-Za-z0-9_?'$%.-]*", body)) 104 support = {} 105 for module in ['prelude', 'parser', 'lexer']: 106 support.update({name: target for name, target in exports(module).items() if name in names}) 107 pack('lexer-support', support) 108 # Native imports require flat tree terms, not indexed executable objects. 109 support_bundle = root / 'lexer-support.arboricx' 110 reply = run(['arboricx', 'import', '--file', support_bundle, '--store', app]) 111 flat_support = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 112 (root / 'lexer_support.tri').write_text(f'!export * from {flat_support}\n') 113 reply = run(['arboricx', 'import', '--file', root / 'math.arboricx', '--store', app]) 114 flat_math = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 115 (root / 'math_support.tri').write_text(f'!export * from {flat_math}\n') 116 header = '!import lexer_support *\n!export lexGateSource\n' 117 changed = body 118 start = changed.index('sourceAsciiLetter? character =') 119 end = changed.index('sourceNameStart? character =') 120 changed = (changed[:start] + 'sourceAsciiLetter? character = N.letter character\n' 121 + 'sourceAsciiDigit? character = N.digit character\n\n' + changed[end:]) 122 changed = changed.replace('(sub first 48)', '(N.subtract first 48)').replace( 123 '(value character : add (mul value 10) (sub character 48))', 124 '(value character : N.decimal value (N.subtract character 48))') 125 assert changed != body and 'N.decimal' in changed 126 variants = {} 127 for variant, text in [('baseline', header + body), 128 ('candidate', '!import math_support as N\n!import lexer_support *\n!export lexGateSource\n' + changed)]: 129 path = root / ('lexer_' + variant + '.tri') 130 path.write_text(text) 131 artifact = root / ('lexer-' + variant + '.arboricx') 132 run(['arboricx', 'compile', '--file', path, '--toolchain', tool, 133 '--tool-store', tools, '--store', app, '--output', artifact, 134 '--compile-timeout', '120', '+RTS', '-N1', '-RTS']) 135 variants[variant] = import_bundle(artifact, tools) 136 # Independent located-token golden for the one-character name; other cases 137 # require complete baseline/candidate reply equality, including diagnostics. 138 def pair(left, right): 139 return b'\2' + left + right 140 141 position_start = pair(number(1), pair(number(1), number(0))) 142 position_end = pair(number(1), pair(number(2), number(1))) 143 token = pair(number(2), pair(tree_list([number(97)]), pair(position_start, position_end))) 144 name_golden = pair(boolean(True), pair(tree_list([token]), pair(b'\0', pair(boolean(True), boolean(False))))) 145 specimens = ['a', 'pick (t x) = x\npick _ = t\nmain z = pick (t z)\n', 146 'main = 12345\n', 'name_1 = "a\\n" -- comment\nmain = [name_1 09]\n', 147 'main = é\n'] 148 for index, text in enumerate(specimens): 149 argument = tree_list([number(ord(c)) for c in text]) 150 for repetition in range(3): 151 outputs = {} 152 order = ['baseline', 'candidate'] if repetition % 2 == 0 else ['candidate', 'baseline'] 153 for variant in order: 154 outputs[variant] = invoke(f'lexer-{index}-{variant}', variants[variant], 155 'lexGateSource', [argument], name_golden if index == 0 else None) 156 assert outputs['baseline'] == outputs['candidate'], ('lexer mismatch', text) 157 if counter: 158 for variant in ['baseline', 'candidate']: 159 counted_output = invoke(f'lexer-{index}-{variant}', variants[variant], 'lexGateSource', 160 [argument], name_golden if index == 0 else None, counted=True) 161 assert counted_output == outputs[variant], ('counted host mismatch', text) 162 for label in sorted({s['label'].removesuffix('-baseline').removesuffix('-candidate') for s in summaries}): 163 old = [s for s in summaries if s['label'] == label + '-baseline'] 164 new = [s for s in summaries if s['label'] == label + '-candidate'] 165 if old and new: 166 speed = statistics.median(s['wall_ns'] for s in old) / statistics.median(s['wall_ns'] for s in new) 167 allocation = statistics.median(s['allocated_bytes'] for s in old) / statistics.median(s['allocated_bytes'] for s in new) 168 print(f'{label}: {speed:.2f}x time, {allocation:.2f}x allocation reduction', flush=True)