refresh_compiler_fixture.py (6213B)
1 #!/usr/bin/env python3 2 """Opt-in native packing of a verified self-built compiler-library unit fixture. 3 Never compiles source or invokes the recovery frontend. A complete generation 4 with current mapped inputs is required; the default gate only verifies/loads it. 5 """ 6 import argparse 7 import hashlib 8 import json 9 from pathlib import Path 10 import subprocess 11 import os 12 import signal 13 import sys 14 import tempfile 15 import time 16 from process_rss import ChildRSS 17 18 REPO = Path(__file__).resolve().parent.parent 19 p = argparse.ArgumentParser(description=__doc__) 20 p.add_argument('tricu', type=Path) 21 p.add_argument('generation', type=Path) 22 p.add_argument('--output-directory', type=Path, default=REPO / 'test/fixtures', 23 help='stage a candidate fixture without replacing published references') 24 p.add_argument('--timeout', type=float, default=600) 25 p.add_argument('--memory-mib', type=int, default=1024, help='sampled Linux process RSS guard') 26 p.add_argument('--heap-mib', type=int, default=512, help='hard GHC heap ceiling') 27 p.add_argument('--native-header-equivalence', action='store_true', 28 help='explicitly verify unit/reexport to native changes against retained staged bytes; not a new generation') 29 args = p.parse_args() 30 if args.timeout <= 0 or args.memory_mib <= 0 or args.heap_mib <= 0: 31 p.error('resource guards must be positive') 32 if not Path('/proc/self/status').is_file(): 33 p.error('fixture refresh requires Linux RSS monitoring; refusing an unguarded run') 34 generation = args.generation.resolve() 35 status = json.loads((generation / 'status.json').read_text()) 36 assert status['complete'] and status['root'] == 'source_map' 37 provenance = json.loads((generation / 'inputs.json').read_text()) 38 rows = [line.split('\t') for line in (generation / 'source-map.v1').read_text().splitlines()[1:]] 39 equivalence = None 40 if args.native_header_equivalence: 41 from native_header_equivalence import native_header_equivalence 42 rows, equivalence = native_header_equivalence(generation, REPO, REPO / 'bootstrap/compiler-source-map.v1') 43 output = args.output_directory.resolve() / 'compiler-library.arboricx' 44 output.parent.mkdir(parents=True, exist_ok=True) 45 command = [str(args.tricu.resolve()), 'arboricx', 'export', '--store', str(generation / 'cas')] 46 metadata = ['tricu.compiler-library-fixture.v1'] 47 source_snapshot = {} 48 for row in rows: 49 _, alias, legacy, path, _ = row 50 payload = (REPO / path).read_bytes() 51 source_snapshot[path] = hashlib.sha256(payload).hexdigest() 52 if equivalence is None: 53 assert hashlib.sha256(payload).hexdigest() == provenance['sources'][alias], ('stale generation input', alias) 54 digest = hashlib.sha256(b'arboricx.source.v1\0' + payload).hexdigest() 55 metadata.append('\t'.join(['module', legacy, alias, path, digest])) 56 cas = generation / 'cas' 57 manifest = (cas / 'cache/source-build' / alias).read_text().split('\t')[2] 58 payload = (cas / 'objects' / manifest[:3] / manifest).read_bytes() 59 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + payload).hexdigest() == manifest 60 for entry in payload.decode().splitlines()[1:]: 61 _, name, _, tree, _ = entry.split('\t') 62 command += ['--target', tree, '--name', alias + '.' + name] 63 if equivalence is not None: 64 proof = json.dumps(equivalence, sort_keys=True, separators=(',', ':')) 65 metadata.append('native-header-equivalence\t' + proof) 66 print('Verified byte-identical native headers:', ', '.join(equivalence['changed']), flush=True) 67 # Publish only after bounded packing succeeds. A failed/killed exporter must not 68 # overwrite the retained fixture. The bundle/metadata pair still fails loudly if 69 # interruption occurs between the two atomic replacements. 70 with tempfile.TemporaryDirectory(prefix='.fixture-refresh-', dir=output.parent) as temporary: 71 candidate = Path(temporary) / output.name 72 invocation = [*command, '--output', str(candidate), '+RTS', '-N1', 73 '-M' + str(args.heap_mib) + 'm', '-RTS'] 74 peak, started = 0, time.monotonic() 75 with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: 76 proc = subprocess.Popen(invocation, stdout=stdout, stderr=stderr, start_new_session=True) 77 try: 78 monitor = ChildRSS(proc) 79 while proc.poll() is None: 80 rss = monitor.sample() 81 if rss is not None: 82 peak = max(peak, rss) 83 if peak > args.memory_mib * 1024: 84 raise RuntimeError('fixture packing RSS guard exceeded') 85 if time.monotonic() - started > args.timeout: 86 raise RuntimeError('fixture packing time guard exceeded') 87 time.sleep(0.05) 88 if proc.returncode: 89 raise RuntimeError(f'fixture exporter failed with code {proc.returncode}') 90 finally: 91 if proc.poll() is None: 92 os.killpg(proc.pid, signal.SIGTERM) 93 try: 94 proc.wait(timeout=2) 95 except subprocess.TimeoutExpired: 96 os.killpg(proc.pid, signal.SIGKILL) 97 proc.wait() 98 stdout.seek(0) 99 stderr.seek(0) 100 sys.stdout.write(stdout.read().decode()) 101 sys.stderr.write(stderr.read().decode()) 102 print(f'Packing elapsed={time.monotonic() - started:.2f}s peak-RSS={peak} KiB', flush=True) 103 for path, expected in source_snapshot.items(): 104 if hashlib.sha256((REPO / path).read_bytes()).hexdigest() != expected: 105 raise RuntimeError('source changed during fixture packing: ' + path) 106 if equivalence is not None and hashlib.sha256((REPO / 'bootstrap/compiler-source-map.v1').read_bytes()).hexdigest() != equivalence['current_map_sha256']: 107 raise RuntimeError('source map changed during fixture packing') 108 metadata.append('bundle\t' + hashlib.sha256(b'tricu.compiler-library-fixture.v1\0' + candidate.read_bytes()).hexdigest()) 109 candidate_metadata = Path(temporary) / output.with_suffix('.v1').name 110 candidate_metadata.write_text('\n'.join(metadata) + '\n') 111 os.replace(candidate, output) 112 os.replace(candidate_metadata, output.with_suffix('.v1')) 113 print('Refreshed', output)