measure.py (2817B)
1 #!/usr/bin/env python3 2 """Fresh-process smoke measurements, not compiler or browser benchmarks.""" 3 import argparse 4 import gzip 5 import hashlib 6 import json 7 import os 8 from pathlib import Path 9 import platform 10 import statistics 11 import subprocess 12 import tempfile 13 import time 14 15 p = argparse.ArgumentParser() 16 p.add_argument('package', type=Path) 17 p.add_argument('output', type=Path) 18 p.add_argument('--samples', type=int, default=5) 19 a = p.parse_args() 20 assert a.samples > 0 21 root = a.package.resolve() 22 report = {'versions': (root / 'versions.txt').read_text(), 23 'platform': platform.platform(), 'samples': [], 'artifacts': {}, 24 'limits': {'timeout_s': 60, 'haskell_heap': '512m', 'haskell_stack': '8m'}, 25 'scope': 'Synthetic reducer/codec smoke only; no bundles, compiler, browser, or JS transfer. Native and WASM GHC versions differ. External time includes process startup and engine compilation. No peak-memory measurement.'} 26 for name in ['probe-native', 'probe.wasm', 'hello.wasm']: 27 data = (root / 'bin' / name).read_bytes() 28 report['artifacts'][name] = {'bytes': len(data), 'gzip_bytes': len(gzip.compress(data, mtime=0)), 29 'sha256': hashlib.sha256(data).hexdigest()} 30 with tempfile.TemporaryDirectory(prefix='tricu-haskell-wasm-') as tmp: 31 env = dict(os.environ, XDG_CACHE_HOME=tmp) 32 # Disable engine caching explicitly: compilation remains in external timings. 33 hosts = {'native': [str(root / 'bin/probe-native')], 34 'wasmtime': [str(root / 'bin/wasmtime'), 'run', '-C', 'cache=n', str(root / 'bin/probe.wasm')]} 35 for sample in range(a.samples): 36 for kind, n, rounds in [('codec', 65536, 16), ('deep', 100000, 16), ('distribution', 100000, 16)]: 37 for host in (list(hosts) if sample % 2 == 0 else list(reversed(hosts))): 38 command = hosts[host] + [kind, str(n), str(rounds), '+RTS', '-M512m', '-K8m', '-RTS'] 39 start = time.monotonic() 40 result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=60, check=True) 41 elapsed = time.monotonic() - start 42 report['samples'].append(dict(json.loads(result.stdout), host=host, sample=sample, process_s=elapsed)) 43 report['medians'] = [] 44 for kind in ['codec', 'deep', 'distribution']: 45 for host in hosts: 46 rows = [r for r in report['samples'] if r['kind'] == kind and r['host'] == host] 47 report['medians'].append(dict(kind=kind, host=host, **{ 48 key: statistics.median(r[key] for r in rows) for key in ['build_s', 'reduce_s', 'observe_s', 'process_s']}, 49 evaluation_observation_s=statistics.median(r['reduce_s']+r['observe_s'] for r in rows))) 50 a.output.write_text(json.dumps(report, indent=2) + '\n') 51 print(json.dumps(report['medians'], indent=2))