compiler_probe.py (6162B)
1 #!/usr/bin/env python3 2 """Bounded two-module compiler job via ctypes, checked against the native CLI. 3 4 Reports fresh-process end-to-end timings, not isolated reducer timings. The CLI 5 also imports packaged compiler data and packages the application bundle. 6 """ 7 import argparse 8 import hashlib 9 import json 10 import os 11 from pathlib import Path 12 import re 13 import resource 14 import subprocess 15 import sys 16 import time 17 from tricu import Library 18 from test_embedding import envelope 19 20 21 def embedded(args): 22 # A subprocess is the resource boundary, independently of the embedding API. 23 resource.setrlimit(resource.RLIMIT_AS, (1024*1024*1024, 1024*1024*1024)) 24 resource.setrlimit(resource.RLIMIT_CPU, (120, 120)) 25 start = time.perf_counter() 26 data = args.compiler.read_bytes() 27 identity = hashlib.sha256(data).hexdigest() 28 library = Library(args.library) 29 with library.context() as ctx: 30 worker = ctx.load_bundle(data, 'sourceBuildGraph') 31 for text in (str(args.source.resolve()), str(args.store.resolve()), identity, 'app'): 32 argument = ctx.string(text) 33 applied = ctx.apply(worker, argument) 34 ctx.release(worker) 35 ctx.release(argument) 36 worker = applied 37 program = envelope(ctx, worker) 38 outcome, result, cleanup = ctx.run_io(program) 39 assert outcome == 0 and ctx.kind(cleanup) == 0 40 assert ctx.kind(ctx.child(result, 0)) == 1, 'compiler returned error' 41 pair = ctx.child(ctx.child(result, 1), 0) 42 manifest = ctx.to_string(ctx.child(pair, 0)) 43 receipt = ctx.to_string(ctx.child(pair, 1)) 44 assert re.fullmatch('[0-9a-f]{64}', manifest) 45 assert re.fullmatch('[0-9a-f]{64}', receipt) 46 report = dict(seconds=time.perf_counter()-start, manifest=manifest, receipt=receipt, 47 peak_rss_native_units=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) 48 print(json.dumps(report)) 49 50 51 def main(): 52 parser = argparse.ArgumentParser(description=__doc__) 53 parser.add_argument('library', type=Path) 54 parser.add_argument('compiler', type=Path) 55 parser.add_argument('--tricu', type=Path) 56 parser.add_argument('--directory', type=Path) 57 parser.add_argument('--child', action='store_true') 58 parser.add_argument('--source', type=Path) 59 parser.add_argument('--store', type=Path) 60 args = parser.parse_args() 61 if args.child: 62 embedded(args) 63 return 64 if args.tricu is None or args.directory is None: 65 parser.error('--tricu and --directory are required') 66 directory = args.directory.resolve() 67 directory.mkdir(parents=True, exist_ok=False) 68 source = directory/'source' 69 source.mkdir() 70 (source/'dep.tri').write_text('!export answer\nanswer = 41\n') 71 (source/'app.tri').write_text('!import dep as D\n!export main\nmain = D.answer\n') 72 ffi_store, cli_store = directory/'ffi-store', directory/'cli-store' 73 ffi_store.mkdir() 74 cli_bundle, ffi_bundle = directory/'cli.arboricx', directory/'ffi.arboricx' 75 reports = {'allocator': 'MALLOC_ARENA_MAX=2 for both subprocesses; FFI RLIMIT_AS=1 GiB, RLIMIT_CPU=120 seconds'} 76 for name in ('cli', 'ffi'): 77 start = time.perf_counter() 78 if name == 'cli': 79 command = [args.tricu, 'arboricx', 'compile', '--file', source/'app.tri', 80 '--store', cli_store, '--output', cli_bundle, 81 '+RTS', '-N1', '-M1024m', '-RTS'] 82 else: 83 command = [sys.executable, Path(__file__).resolve(), args.library, args.compiler, 84 '--child', '--source', source, '--store', ffi_store] 85 # Bound glibc's per-thread arena reservations in this supervised job; 86 # GHC and Python share the child's unchanged 1-GiB address-space limit. 87 environment = dict(os.environ, MALLOC_ARENA_MAX='2') 88 completed = subprocess.run(list(map(str, command)), text=True, 89 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 90 timeout=120, check=False, env=environment) 91 (directory/(name+'.log')).write_text(completed.stdout+completed.stderr) 92 if completed.returncode: 93 print(f'{name} compiler subprocess failed:\n{completed.stdout}{completed.stderr}', file=sys.stderr) 94 completed.check_returncode() 95 reports[name] = dict(process_seconds=time.perf_counter()-start) 96 if name == 'ffi': 97 reports[name].update(json.loads(completed.stdout)) 98 manifest, receipt = reports['ffi']['manifest'], reports['ffi']['receipt'] 99 # Every source-owned published object must match the CLI job byte for byte. 100 # CLI also imports its packaged compiler, so its inventory is a superset. 101 objects = list((ffi_store/'objects').glob('*/*')) 102 assert objects, 'no compiler publications' 103 for path in objects: 104 relative = path.relative_to(ffi_store) 105 assert path.read_bytes() == (cli_store/relative).read_bytes(), relative 106 receipt_bytes = (ffi_store/'objects'/receipt[:3]/receipt).read_bytes() 107 assert hashlib.sha256(b'arboricx.build-receipt.v1\0'+receipt_bytes).hexdigest() == receipt 108 subprocess.run(list(map(str, [args.tricu, 'arboricx', 'export', '--module', manifest, 109 '--store', ffi_store, '--output', ffi_bundle])), check=True, timeout=30) 110 assert ffi_bundle.read_bytes() == cli_bundle.read_bytes(), 'program bundle mismatch' 111 with Library(args.library).context() as ctx: 112 worker = ctx.load_bundle(ffi_bundle.read_bytes(), 'main') 113 assert ctx.to_number(ctx.reduce(worker)) == 41 114 reports['published_objects_compared'] = len(objects) 115 reports['bundle_sha256'] = hashlib.sha256(ffi_bundle.read_bytes()).hexdigest() 116 reports['compiler_sha256'] = hashlib.sha256(args.compiler.read_bytes()).hexdigest() 117 reports['library_sha256'] = hashlib.sha256(args.library.read_bytes()).hexdigest() 118 reports['scope'] = 'cold two-module application build; not a compiler generation' 119 (directory/'report.json').write_text(json.dumps(reports, indent=2)+'\n') 120 print(json.dumps(reports, sort_keys=True)) 121 122 123 if __name__ == '__main__': 124 main()