base_workloads.py (15441B)
1 #!/usr/bin/env python3 2 """Opt-in stable/Haskell base-list comparison; no production source or fixture edits. 3 4 Generate exact manual workers from base.tri and isolated alternative workers. 5 Use fresh processes (never --repeat), full-result Python oracles and --profile 6 observation samples; report load and invocation-through-output separately. 7 This is not compiler acceptance. 8 """ 9 import argparse 10 import hashlib 11 import json 12 import os 13 import platform 14 from pathlib import Path 15 import random 16 import re 17 import statistics 18 import time 19 20 from bundle_dag import read, merge_bundles, Intern 21 from fixture_sources import verify, snapshot_bytes 22 from http_headers import number, tree_list, expect 23 from source_order import Runner 24 25 ROOT = Path(__file__).resolve().parent.parent 26 # Immutable manual-worker source; never regenerate a baseline from edited base.tri. 27 BASE_REFERENCE = 'fc210e5cdd97d60ce7290b4ac4227adc51119e2786bcacb671b7ad9cb6cda427' 28 29 30 def reference_source(fixtures=None): 31 return snapshot_bytes(fixtures or ROOT / 'test/fixtures', BASE_REFERENCE).decode() 32 33 34 WORKERS = { 35 'map': ('map_', ['l', 'f'], ['f']), 36 'filter': ('filter_', ['l', 'f'], ['f']), 37 'foldl': ('foldl_', ['l', 'f', 'acc'], ['f']), 38 'reverse': ('reverse_', ['xs', 'acc'], []), 39 'takeWhile': ('takeWhile_', ['xs', 'f'], ['f']), 40 'splitOnByte': ('splitByte_', ['str', 'byte', 'acc', 'current'], ['byte']), 41 } 42 HELPERS = ['y', 'matchList', 'lazyList', 'lazyBool', 'matchBool', 'pair', 43 'succ', 'equal?', 'emptyList?', 'not?'] 44 PATTERNS = '''map_ [] f = t 45 map_ [h | r] f = pair (f h) (map_ r f) 46 map_ other f = other 47 filter_ [] f = t 48 filter_ [h | r] f = matchBool (pair h (filter_ r f)) (filter_ r f) (f h) 49 filter_ other f = other 50 foldl_ [] f acc = acc 51 foldl_ [h | r] f acc = foldl_ r f (f acc h) 52 foldl_ other f acc = other 53 reverse_ [] acc = acc 54 reverse_ [h | r] acc = reverse_ r (pair h acc) 55 reverse_ other acc = other 56 takeWhile_ [] f = t 57 takeWhile_ [h | r] f = lazyBool (_ : pair h (takeWhile_ r f)) (_ : t) (f h) 58 takeWhile_ other f = other t 59 splitByte_ [] byte acc current = map reverse (reverse (pair current acc)) 60 splitByte_ [h | r] byte acc current = (lazyBool 61 (_ : splitByte_ r byte (pair current acc) t) 62 (_ : splitByte_ r byte acc (pair h current)) 63 (equal? h byte)) 64 splitByte_ other byte acc current = other t 65 ''' 66 DRIVER = '''lowerByte b = lazyBool (_ : natAdd b 32) (_ : b) (inRange b 65 90) 67 lower xs = map lowerByte xs 68 digits xs = filter digit xs 69 reverseBytes xs = reverse xs 70 sumBytes xs = foldl natAdd 0 xs 71 splitLines xs = splitOnByte 10 xs 72 prefixDigits xs = takeWhile digit xs 73 normalizeLines xs = map lower (filter (x : not? (emptyList? x)) (splitOnByte 10 xs)) 74 sumRows rows = map (foldl natAdd 0) rows 75 spin x = spin x 76 lazyMapIgnored x = map (_ : 7) (pair (spin x) t) 77 lazyFoldIgnored x = foldl (_ _ : 7) 0 (pair (spin x) t) 78 lazyTakeEarly x = takeWhile (_ : t) (pair (spin x) (spin x)) 79 ''' 80 EXPORTS = ['lower', 'digits', 'reverseBytes', 'sumBytes', 'splitLines', 81 'prefixDigits', 'normalizeLines', 'sumRows', 'lazyMapIgnored', 'lazyFoldIgnored', 'lazyTakeEarly'] 82 83 84 def definition(source, name): 85 match = re.search(r'^' + re.escape(name) + r' .*=.*(?:\n[ \t].*|\n(?=\n))*', source, re.M) 86 if not match: 87 raise ValueError('missing definition ' + name) 88 return match[0].rstrip() + '\n' 89 90 91 def candidate(source, variant): 92 if variant == 'published': 93 return '' 94 result = [] 95 for public, (worker, parameters, fixed) in WORKERS.items(): 96 raw = definition(source, worker) 97 wrapper = definition(source, public) 98 if variant == 'manual': 99 result.extend([raw, wrapper]) 100 elif variant == 'direct': 101 result.extend([raw.replace(worker + ' self ', worker + ' ', 1).replace('self ', worker + ' '), 102 wrapper.replace('y ' + worker, worker)]) 103 elif variant == 'patterned': 104 result.append(wrapper.replace('y ' + worker, worker)) 105 elif variant == 'captured': 106 body = raw.split('=', 1)[1].strip() 107 dynamic = [p for p in parameters if p not in fixed] 108 # Known recursive calls are atom-only; assert every self call matched. 109 call = r'\bself' + ''.join(r' (\([^\n]*?\)|[A-Za-z]+)' for _ in parameters) 110 def recur(m): 111 return 'walk ' + ' '.join(m[i + 1] for i, p in enumerate(parameters) if p not in fixed) 112 body = re.sub(call, recur, body) 113 assert not re.search(r'\bself\b', body), (public, body) 114 head, application = wrapper.split('=', 1) 115 arguments, application = application.split(':', 1) 116 head = public + ' ' + arguments.strip() + ' ' 117 args = application.strip().removeprefix('y ' + worker + ' ').split() 118 assert len(args) == len(parameters), wrapper 119 entry = 'walk ' + ' '.join(v for p, v in zip(parameters, args) if p not in fixed) 120 result.append(head + '= (let rec\n walk ' + ' '.join(dynamic) + ' = (' + 121 '\n'.join(' ' + line for line in body.splitlines()) + 122 ')\n in ' + entry + ')\n') 123 else: 124 raise ValueError(variant) 125 if variant == 'patterned': 126 result.insert(0, PATTERNS) 127 return '\n'.join(result) 128 129 130 def workloads(size): 131 # ASCII protocol/log-style data, including empty records and mixed casing. 132 seed = b'GET /Api/Items/42 HTTP/1.1\nHost: Example.TEST\n\nX-Request-ID: abc123\n' 133 data = (seed * (size // len(seed) + 1))[:size] 134 lines = data.split(b'\n') 135 values = list(data) 136 rows = [values[i:i + 64] for i in range(0, len(values), 64)] 137 expected_prefix = b'12345678901234567890'[:min(size, 20)] 138 prefix = expected_prefix + b'x' + data 139 return [ 140 ('lower', values, list(data.lower())), 141 ('digits', values, [x for x in values if 48 <= x <= 57]), 142 ('reverseBytes', values, list(reversed(values))), 143 ('sumBytes', values, sum(values)), 144 ('splitLines', values, [list(x) for x in lines]), 145 ('prefixDigits', list(prefix), list(expected_prefix)), 146 ('normalizeLines', values, [list(x.lower()) for x in lines if x]), 147 ('sumRows', rows, [sum(x) for x in rows]), 148 ] 149 150 151 def encode(value): 152 return number(value) if isinstance(value, int) else tree_list([encode(x) for x in value]) 153 154 155 def profiles(path): 156 return {v['profile']: v for line in path.read_text().splitlines() if line.startswith('{') 157 for v in [json.loads(line)] if 'profile' in v} 158 159 160 def measurements(path): 161 p = profiles(path) 162 result = {'measurement_scope': 'invocation-through-rendered-output-v1'} 163 for phase, begin, end in [('load', 'load-begin', 'load-end'), 164 ('invoke', 'invoke-begin-1', 'observe-end-1')]: 165 for key in ['wall_ns', 'allocated_bytes', 'gc_elapsed_ns']: 166 result[phase + '_' + key] = p[end][key] - p[begin][key] 167 residency = re.search(r'^\s*([0-9,]+) bytes maximum residency', path.read_text(), re.M) 168 assert residency, 'RTS maximum residency missing' 169 result['rts_max_residency_bytes'] = int(residency[1].replace(',', '')) 170 result['post_invoke_live_bytes'] = p['observe-end-1']['live_bytes'] 171 return result 172 173 174 def summarize(rows): 175 groups = {} 176 for row in rows: 177 if row['warmup'] or row.get('failed'): 178 continue 179 key = (row['variant'], row['workload'], row['size']) 180 groups.setdefault(key, []).append(row) 181 output = [] 182 for (variant, workload, size), samples in sorted(groups.items()): 183 entry = dict(variant=variant, workload=workload, size=size, repetitions=len(samples)) 184 for field in ['invoke_wall_ns', 'invoke_allocated_bytes', 'invoke_gc_elapsed_ns', 185 'load_wall_ns', 'load_allocated_bytes', 'post_invoke_live_bytes', 'rts_max_residency_bytes', 'peak_kib', 'seconds']: 186 values = [v[field] for v in samples] 187 entry[field] = dict(median=statistics.median(values), minimum=min(values), maximum=max(values)) 188 output.append(entry) 189 return output 190 191 192 def main(): 193 p = argparse.ArgumentParser(description=__doc__) 194 p.add_argument('tricu', type=Path) 195 p.add_argument('--compiler', type=Path, required=True, help='accepted build-stable.arboricx') 196 p.add_argument('--directory', type=Path, required=True) 197 p.add_argument('--reference-fixtures', type=Path, default=ROOT / 'test/fixtures', 198 help='authenticated pre-adoption fixture cohort for published/manual comparison') 199 p.add_argument('--sizes', type=int, nargs='+', default=[64, 1024, 8192]) 200 p.add_argument('--repetitions', type=int, default=5) 201 p.add_argument('--variants', nargs='+', choices=['published', 'manual', 'direct', 'patterned', 'captured'], 202 default=['published', 'manual', 'direct', 'patterned', 'captured']) 203 args = p.parse_args() 204 assert args.repetitions >= 3 and all(0 < x <= 65536 for x in args.sizes) 205 root = args.directory.resolve();root.mkdir(parents=True, exist_ok=False) 206 (root / 'harness.py').write_bytes(Path(__file__).read_bytes()) 207 verify(args.reference_fixtures) 208 source = reference_source(args.reference_fixtures) 209 (root / 'base-reference.tri').write_text(source) 210 runner = Runner(args.tricu, root) 211 tool = runner.imported(args.compiler, 'compiler') 212 fixture = args.reference_fixtures / 'compiler-library.arboricx' 213 selected = [('base.' + n, n) for n in HELPERS] 214 selected += [('native_math.' + old, new) for old, new in 215 [('add', 'natAdd'), ('inRange', 'inRange'), ('digit', 'digit')]] 216 pins = {}; sizes = {}; kernels = {}; intern = Intern() 217 for variant in args.variants: 218 support = root / (variant + '-support.arboricx') 219 entries = selected + ([('base.' + n, n) for n in WORKERS] if variant == 'published' else []) 220 merge_bundles(support, [(fixture, entries)], prune_unreachable=True) 221 # Native dependency parsing needs flat terms, not merely indexed roots. 222 text = runner.run(variant + '-support', ['import', '--file', support, '--store', runner.store]) 223 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', text)[1] 224 path = root / (variant + '.tri') 225 path.write_text('!import ' + pin + ' *\n' + ''.join('!export ' + n + '\n' for n in EXPORTS) + 226 candidate(source, variant) + '\n' + DRIVER) 227 bundle = root / (variant + '.arboricx') 228 runner.run(variant + '-compile', ['compile', '--file', path, '--toolchain', tool, 229 '--tool-store', runner.store, '--store', runner.store, '--output', bundle, 230 '--compile-timeout', '180'], seconds=240) 231 pins[variant] = runner.imported(bundle, variant + '-import') 232 graph = read(bundle); expanded = []; ids = intern.graph(graph[2]) 233 kernels[variant] = {name: ids[node] for name, node in graph[3].items()} 234 for tag, *children in graph[2]: 235 expanded.append(1 + sum(expanded[x] for x in children)) 236 sizes[variant] = dict(bundle_bytes=bundle.stat().st_size, dag_nodes=len(graph[2]), 237 expanded_export_nodes={k: expanded[v] for k, v in graph[3].items()}, 238 source_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), 239 bundle_sha256=hashlib.sha256(bundle.read_bytes()).hexdigest(), manifest=pins[variant]) 240 (root / 'subjects.json').write_text(json.dumps(dict(compiler_sha256=hashlib.sha256(args.compiler.read_bytes()).hexdigest(), 241 fixture_sha256=hashlib.sha256(fixture.read_bytes()).hexdigest(), 242 base_sha256=hashlib.sha256(source.encode()).hexdigest(), 243 harness_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), subjects=sizes), indent=2) + '\n') 244 if 'manual' in kernels and 'published' in kernels: 245 assert kernels['manual'] == kernels['published'], 'manual baseline differs from published kernels' 246 (root / 'environment.json').write_text(json.dumps(dict(platform=platform.platform(), 247 python=platform.python_version(), cpu_affinity=sorted(os.sched_getaffinity(0)), 248 runtime=str(args.tricu.resolve()), runtime_sha256=hashlib.sha256(args.tricu.read_bytes()).hexdigest(), 249 repetitions=args.repetitions, seed=20261009, source_date=time.time(), 250 force_profile_tree=True, profile_samples_force_gc=True, 251 manual_kernels_equal_published=(kernels['manual'] == kernels['published'] 252 if 'manual' in kernels and 'published' in kernels else None)), indent=2) + '\n') 253 # Small independent edge cases, including historical Stem behavior and demand. 254 for variant in args.variants: 255 for name, value in [('lazyMapIgnored', [7]), ('lazyFoldIgnored', 7), ('lazyTakeEarly', [])]: 256 expect(runner.invoke(variant + '-' + name, pins[variant], name, ['--arg', '0']), encode(value), name) 257 stem = root / 'stem.tree';stem.write_bytes(b'\1\0') 258 for name in ['lower', 'digits', 'sumBytes', 'reverseBytes']: 259 expect(runner.invoke(variant + '-stem-' + name, pins[variant], name, ['--arg-tree-file', stem]), b'\1\0', name) 260 for name in ['splitLines', 'prefixDigits']: 261 expect(runner.invoke(variant + '-stem-' + name, pins[variant], name, ['--arg-tree-file', stem]), b'\2\0\0', name) 262 for index, data in enumerate([b'', b'\n', b'\n\n', b'a\n', b'\na', b'A09z']): 263 edge = root / f'edge-{index}.tree';edge.write_bytes(encode(list(data))) 264 expect(runner.invoke(f'{variant}-split-edge-{index}', pins[variant], 'splitLines', ['--arg-tree-file', edge]), 265 encode([list(x) for x in data.split(b'\n')]), 'split edge') 266 rows = []; randomizer = random.Random(20261009) 267 # Warmup is discarded and orders are interleaved, not all old then all new. 268 for size in args.sizes: 269 for workload, argument, expected in workloads(size): 270 path = root / f'{workload}-{size}.tree';path.write_bytes(encode(argument)) 271 for repetition in range(args.repetitions + 1): 272 order = args.variants.copy();randomizer.shuffle(order) 273 for variant in order: 274 label = f'{workload}-{size}-{repetition}-{variant}' 275 actual = runner.invoke(label, pins[variant], workload, ['--arg-tree-file', path]) 276 expect(actual, encode(expected), label) 277 row = dict(variant=variant, workload=workload, size=size, repetition=repetition, 278 warmup=repetition == 0, **measurements(root / (label + '.log')), 279 seconds=runner.records[-1]['seconds'], peak_kib=runner.records[-1]['peak_kib']) 280 rows.append(row) 281 (root / 'results.json').write_text(json.dumps(rows, indent=2) + '\n') 282 print('PASS', workload, size, flush=True) 283 # Wrong subject: same-arity reverse must not pass the lowercase oracle. 284 path = root / 'wrong-subject.tree';path.write_bytes(encode(list(b'ABCdef'))) 285 wrong = runner.invoke('wrong-subject', pins[args.variants[0]], 'reverseBytes', ['--arg-tree-file', path]) 286 assert wrong != encode(list(b'abcdef')) 287 (root / 'summary.json').write_text(json.dumps(summarize(rows), indent=2) + '\n') 288 print('PASS', len(rows), 'full-result invocations; wrong-subject control rejected') 289 290 291 if __name__ == '__main__': 292 main()