measure-compiler.py (4061B)
1 #!/usr/bin/env python3 2 """Compare the site's pinned compiler on native Haskell and Wasmtime.""" 3 import argparse 4 import gzip 5 import hashlib 6 import json 7 import os 8 from pathlib import Path 9 import platform 10 import statistics 11 import subprocess 12 import tempfile 13 import time 14 15 p = argparse.ArgumentParser() 16 p.add_argument('package', type=Path) 17 p.add_argument('output', type=Path) 18 p.add_argument('--samples', type=int, default=5) 19 a = p.parse_args() 20 assert a.samples > 0 21 root = a.package.resolve() 22 data = root / 'compiler-data' 23 manifest = json.loads((data / 'manifest.json').read_text()) 24 assert hashlib.sha256((data / 'compiler.dag').read_bytes()).hexdigest() == manifest['dagSha256'] 25 report = dict(package=str(root), manifest=manifest, versions=(root/'versions.txt').read_text(), platform=platform.platform(), 26 samples=[], artifacts={}, limits={'timeout_s':120, 'heap':'512m', 'haskell_stack':'8m', 'ts_graph_nodes':6000000, 'ts_graph_stack':100000}, 27 scope='Trusted prepared DAG, not bundle authentication in WASM. Node/V8 command execution is not browser validation; no JS tree transfer/IO. TS loads/authenticates the original bundle; Haskell loads a prepared DAG, so loading times are not like-for-like. RTS memory is not total engine RSS. Names and full-normal-form boundaries both measured; goldens checked in every process. Native and WASM GHC versions differ.') 28 for name in ['compiler-native','compiler.wasm']: 29 b=(root/'bin'/name).read_bytes() 30 report['artifacts'][name]=dict(bytes=len(b), gzip_bytes=len(gzip.compress(b,mtime=0)),sha256=hashlib.sha256(b).hexdigest()) 31 with tempfile.TemporaryDirectory(prefix='tricu-haskell-compiler-') as tmp: 32 env=dict(os.environ,XDG_CACHE_HOME=tmp) 33 for sample in range(a.samples): 34 for case in manifest['cases']: 35 i=str(case['id']) 36 hosts={'native':[str(root/'bin/compiler-native'),str(data),str(data/i)], 37 'wasmtime':[str(root/'bin/wasmtime'),'run','-C','cache=n','--dir',f'{data}::/data',str(root/'bin/compiler.wasm'),'/data','/data/'+i], 38 'node-wasm':[str(root/'bin/node'),str(root/'bin/node-wasi.mjs'),str(root/'bin/compiler.wasm'),str(data),i], 39 'node-ts':[str(root/'bin/node'),str(root/'bin/ts-compiler.mjs'),str(root/'site'),i]} 40 for host in list(hosts) if sample%2==0 else list(reversed(hosts)): 41 start=time.monotonic() 42 result=subprocess.run(hosts[host]+([] if host=='node-ts' else ['+RTS','-T','-M512m','-K8m','-RTS']),capture_output=True,text=True,env=env,timeout=120) 43 elapsed=time.monotonic()-start 44 row=dict(host=host,case=case['id'],sample=sample,process_s=elapsed,returncode=result.returncode) 45 if result.returncode: 46 row.update(stdout=result.stdout,stderr=result.stderr) 47 else: 48 row.update(json.loads(result.stdout)) 49 if host!='node-ts': 50 row['compile_names_s']=row['whnf_s']+row['names_s'] 51 row['compile_complete_s']=row['compile_names_s']+row['complete_s'] 52 report['samples'].append(row) 53 a.output.write_text(json.dumps(report,indent=2)+'\n') 54 print(host,i,sample, result.returncode, row.get('compile_complete_s'),flush=True) 55 report['medians']=[] 56 for case in manifest['cases']: 57 for host in ['native','wasmtime','node-wasm','node-ts']: 58 rows=[r for r in report['samples'] if r['case']==case['id'] and r['host']==host and r['returncode']==0] 59 if len(rows)==a.samples: 60 report['medians'].append(dict(case=case['id'],host=host,**{k:statistics.median(r[k] for r in rows) for k in 61 (['load_s','compile_names_s','compile_complete_s','run_s','process_s'] + ([] if host=='node-ts' else ['allocated_bytes','max_live_bytes','max_mem_in_use_bytes']))})) 62 a.output.write_text(json.dumps(report,indent=2)+'\n') 63 assert all(r['returncode']==0 for r in report['samples']), 'workload failures recorded in report'