bootstrap_generations.py (22448B)
1 #!/usr/bin/env python3 2 """Opt-in artifact-only compiler rebuild; native packing, no source frontend.""" 3 import argparse 4 import fcntl 5 import hashlib 6 import json 7 import os 8 from pathlib import Path 9 import re 10 import signal 11 import subprocess 12 import sys 13 import time 14 from generation_backend import BACKENDS, DEFAULT_BACKEND 15 from bootstrap_inputs import load_kit 16 from process_rss import ChildRSS, run_guarded 17 18 REPO = Path(__file__).resolve().parent.parent 19 20 21 def source_map(path, source_root, root): 22 """Fail early on stale headers, incomplete edges, cycles and unsafe paths.""" 23 text = path.read_text() 24 lines = text.splitlines() 25 if not lines or lines[0] != 'arboricx.source-map.v1': 26 raise ValueError('unsupported source map') 27 entries = {} 28 for line in lines[1:]: 29 kind, alias, legacy, relative, deps = line.split('\t') 30 if kind not in ('unit', 'reexport', 'native') or alias in entries: 31 raise ValueError('duplicate alias or invalid map kind') 32 if not re.fullmatch(r'[A-Za-z0-9_-]+', alias): 33 raise ValueError('invalid alias') 34 relative_path = Path(relative) 35 if relative_path.is_absolute() or any(p in ('.', '..') for p in relative.split('/')): 36 raise ValueError('unsafe source path') 37 if not (source_root / relative_path).resolve().is_relative_to(source_root.resolve()): 38 raise ValueError('source path escapes root') 39 entries[alias] = (legacy, relative, [] if deps == '-' else deps.split(',')) 40 required, active = set(), set() 41 def visit(alias): 42 if alias in active: 43 raise ValueError('source map dependency cycle: ' + alias) 44 if alias in required: 45 return 46 if alias not in entries: 47 raise ValueError('undeclared map dependency: ' + alias) 48 active.add(alias) 49 for dep in entries[alias][2]: 50 visit(dep) 51 active.remove(alias) 52 required.add(alias) 53 visit(root) 54 # Staging reads every row, including entries outside the selected closure. 55 kinds = {row.split('\t')[1]: row.split('\t')[0] for row in lines[1:]} 56 for alias, (_, relative, deps) in entries.items(): 57 if any(dep not in entries for dep in deps): 58 raise ValueError('undeclared dependency: ' + alias) 59 native = kinds[alias] == 'native' 60 rows = (source_root / relative).read_text().splitlines() 61 if native: 62 from native_header import native_header 63 actual, _ = native_header('\n'.join(rows)) 64 if actual != deps: 65 raise ValueError('source map header mismatch: ' + alias) 66 else: 67 # Explicit recovery-map translation, never native import dispatch. 68 expected = [['!import', '"' + entries[dep][0] + '"', '!Local'] for dep in deps] 69 actual = [row.split() for row in rows[:len(deps)]] 70 if actual != expected or (len(rows) > len(deps) and rows[len(deps)].split()[:1] == ['!import']): 71 raise ValueError('source map header mismatch: ' + alias) 72 return text, entries, required 73 74 75 def snapshot(map_text, entries, source_root): 76 return dict(map_sha256=hashlib.sha256(map_text.encode()).hexdigest(), 77 sources={alias: hashlib.sha256((source_root / row[1]).read_bytes()).hexdigest() 78 for alias, row in entries.items()}) 79 80 81 def main(): 82 parser = argparse.ArgumentParser(description=__doc__) 83 parser.add_argument('previous', type=Path, nargs='?', help='completed generation; omit when selecting --bootstrap-kit') 84 parser.add_argument('output', type=Path) 85 parser.add_argument('--tricu', default='result/bin/tricu') 86 parser.add_argument('--execution-host', choices=('haskell', 'zig'), default='haskell') 87 parser.add_argument('--zig-library', type=Path, help='explicit Nix-built owned-ABI library; required for Zig execution') 88 parser.add_argument('--zig-gc-policy', type=int, nargs=2, metavar=('MIN_ALLOCATIONS', 'LIVE_MULTIPLIER'), 89 help='explicit positive u32 allocation headroom (not an RSS limit); Zig only') 90 parser.add_argument('--backend', choices=BACKENDS, default=DEFAULT_BACKEND, 91 help='generation lowering backend (default: %(default)s); never inferred from previous') 92 parser.add_argument('--worker-bundle', type=Path, help='explicit pinned transition worker exporting the selected backend entrypoint; no fallback') 93 parser.add_argument('--adapter', type=Path, help='precompiled sourceBootstrapNativeAdapter bundle; otherwise reuse previous/io-adapter.arboricx') 94 parser.add_argument('--bootstrap-kit', type=Path, help='explicit verified compiler/adapter kit; no previous store access') 95 parser.add_argument('--map', type=Path, default=REPO / 'bootstrap/compiler-source-map.v1') 96 parser.add_argument('--source-root', type=Path, default=REPO) 97 parser.add_argument('--root', default='source_map') 98 parser.add_argument('--memory-mib', type=int, default=32768) 99 parser.add_argument('--capabilities', type=int, default=1, help='GHC capabilities; explicit to avoid machine-wide -N defaults') 100 parser.add_argument('--wall-seconds', type=float, default=3600, 101 help='whole-job budget including preparation/audit/packing; max 7200 seconds') 102 parser.add_argument('--initial-seconds', type=float, default=600) 103 parser.add_argument('--extended-seconds', type=float, default=1800) 104 parser.add_argument('--expect-identical', action='store_true', help='require identical inputs, executable manifests and packaged artifacts') 105 parser.add_argument('--fresh', action='store_true', help='require a nonexistent output directory; no checkpoint reuse') 106 parser.add_argument('--validate-only', action='store_true') 107 args = parser.parse_args() 108 if not 0 < args.memory_mib <= 65536 or args.capabilities <= 0 or not 0 < args.wall_seconds <= 7200 or not 0 < args.initial_seconds <= args.extended_seconds: 109 parser.error('invalid resource boundaries') 110 if (args.execution_host == 'zig') != (args.zig_library is not None): 111 parser.error('--zig-library is required exactly when --execution-host zig is selected') 112 if args.zig_library and not args.zig_library.is_file(): 113 parser.error('selected Zig library is unavailable; no fallback') 114 if args.zig_gc_policy and (args.execution_host != 'zig' or 115 any(not 0 < n <= 0xffffffff for n in args.zig_gc_policy)): 116 parser.error('--zig-gc-policy requires Zig and two positive u32 values') 117 previous = args.previous.resolve() if args.previous else None 118 output = args.output.resolve() 119 from bootstrap_retention_lock import protect_generations 120 retention_locks = protect_generations([previous, output]) 121 for selected in (previous, output): 122 if selected and any((selected / store / 'PRUNED-CAS.json').exists() 123 for store in ('cas', 'compiler-cas')): 124 parser.error('pruned generations are historical evidence, not resumable stores; ' 125 'select a retained input and a fresh output directory') 126 if args.bootstrap_kit: 127 if previous or args.adapter or args.worker_bundle or args.expect_identical: 128 parser.error('--bootstrap-kit replaces previous/adapter/worker selection; reproduction uses the first generation') 129 elif previous is None: 130 parser.error('select a previous generation or --bootstrap-kit') 131 source_root = args.source_root.resolve() 132 map_text, entries, required = source_map(args.map.resolve(), source_root, args.root) 133 inputs = snapshot(map_text, entries, source_root) 134 worker_export, backend_options = BACKENDS[args.backend] 135 if args.expect_identical and not args.validate_only: 136 prior_inputs = json.loads((previous / 'inputs.json').read_text()) 137 if (any(prior_inputs.get(key) != value for key, value in inputs.items()) 138 or prior_inputs.get('backend', 'optimized') != args.backend 139 or prior_inputs.get('root') != args.root): 140 parser.error('reproduction requires identical source/map inputs, root and backend') 141 if args.validate_only: 142 print(json.dumps(dict(root=args.root, backend=args.backend, worker_export=worker_export, 143 backend_options=backend_options, modules=sorted(required), **inputs), indent=2)) 144 return 145 if output == previous: 146 parser.error('output must differ from previous generation') 147 if args.fresh and output.exists(): 148 parser.error('--fresh requires a nonexistent output directory') 149 fresh_store = not (output / 'cas').exists() 150 kit_digest, kit_manifest, kit_payloads = None, None, None 151 old_cas, target = None, None 152 if args.bootstrap_kit: 153 kit_manifest, kit_payloads, kit_digest = load_kit(args.bootstrap_kit, args.backend) 154 adapter_bytes = kit_payloads['adapter'] 155 else: 156 adapter_input = (args.adapter or previous / 'io-adapter.arboricx').resolve() 157 if not adapter_input.is_file(): 158 parser.error('precompiled adapter required; use --adapter (no automatic bootstrap compilation)') 159 adapter_bytes = adapter_input.read_bytes() 160 old_cas = previous / 'cas' 161 _, _, manifest, _ = (old_cas / 'cache/source-build/source_map').read_text().split('\t') 162 manifest_bytes = (old_cas / 'objects' / manifest[:3] / manifest).read_bytes() 163 if hashlib.sha256(b'arboricx.module-manifest.v1\0' + manifest_bytes).hexdigest() != manifest: 164 raise ValueError('invalid previous compiler manifest') 165 rows = manifest_bytes.decode().splitlines()[1:] 166 target = next((row.split('\t')[3] for row in rows if row.split('\t')[1] == worker_export), None) 167 if target is None and not args.worker_bundle: 168 parser.error('previous compiler lacks selected entrypoint ' + worker_export + 169 '; supply an explicit --worker-bundle (no backend fallback)') 170 tricu = str(Path(args.tricu).resolve()) 171 output.mkdir(parents=True, exist_ok=True) 172 lock = (output / '.generation.lock').open('a') 173 try: 174 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) 175 except BlockingIOError: 176 parser.error('another generation is using this output directory') 177 # Do not let a failed/restarted invocation retain a stale success status. 178 (output / 'status.json').write_text(json.dumps(dict(complete=False, reason='preparation'))) 179 job_start = time.monotonic() 180 def remaining(): 181 budget = args.wall_seconds - (time.monotonic() - job_start) 182 if budget <= 0: 183 raise TimeoutError('whole-job wall-time guard') 184 return budget 185 def transport(command): 186 # Native packing/import should need far less than a full compilation. 187 reply, _ = run_guarded([tricu, *command, '+RTS', 188 '-N' + str(args.capabilities), '-M' + str(args.memory_mib) + 'm', '-RTS'], 189 memory_mib=args.memory_mib, seconds=min(600, remaining())) 190 return reply 191 bundle = output / 'compiler.arboricx' 192 if kit_payloads: 193 bundle.write_bytes(kit_payloads['compiler']) 194 (output / 'bootstrap-kit.json').write_bytes(kit_manifest) 195 elif args.worker_bundle: 196 bundle.write_bytes(args.worker_bundle.resolve().read_bytes()) 197 else: 198 print(transport(['arboricx', 'export', '--store', str(old_cas), '--target', target, 199 '--name', worker_export, '--output', str(bundle)]), end='') 200 compiler = hashlib.sha256(bundle.read_bytes()).hexdigest() 201 adapter = output / 'io-adapter.arboricx' 202 adapter.write_bytes(adapter_bytes) 203 def imported(path, alias): 204 reply = transport(['arboricx', 'import', '--store', str(output / 'compiler-cas'), 205 '--file', str(path), '--module', alias, '--indexed']) 206 return reply.split('Created module alias ' + alias + ' -> ')[1].strip() 207 compiler_manifest = imported(bundle, 'compiler') 208 payload = (output / 'compiler-cas/objects' / compiler_manifest[:3] / compiler_manifest).read_text() 209 argument_export = next((row.split('\t') for row in payload.splitlines()[1:] 210 if row.split('\t')[1] == worker_export), None) 211 if argument_export is None: 212 parser.error('compiler bundle lacks selected entrypoint ' + worker_export + 213 ' (no backend fallback)') 214 # Transport object identity is distinct from the bundle identity used in receipts. 215 argument_kind, argument_hash = argument_export[2:4] 216 adapter_manifest = imported(adapter, 'io-adapter') 217 provenance = dict(source_root=str(source_root), root=args.root, backend=args.backend, 218 fresh_store=fresh_store, 219 worker_export=worker_export, backend_options=backend_options, 220 compiler_tree=argument_hash, compiler_tree_kind=argument_kind, 221 compiler_source_tree=None if args.worker_bundle else target, 222 bootstrap_kit_sha256=kit_digest, 223 compiler_bundle_sha256=compiler, compiler_manifest=compiler_manifest, 224 adapter_sha256=hashlib.sha256(adapter_bytes).hexdigest(), 225 execution_host=args.execution_host, 226 runtime_path=str(args.zig_library.resolve()) if args.zig_library else tricu, 227 runtime_sha256=hashlib.sha256((args.zig_library or Path(tricu)).read_bytes()).hexdigest(), 228 transport_host='haskell', transport_runtime_path=tricu, 229 transport_runtime_sha256=hashlib.sha256(Path(tricu).read_bytes()).hexdigest(), 230 executor_sha256=hashlib.sha256((REPO / 'test/zig_generation_executor.py').read_bytes()).hexdigest() if args.zig_library else None, 231 **inputs) 232 if args.zig_gc_policy: 233 provenance['zig_gc_policy'] = args.zig_gc_policy 234 (output / 'inputs.json').write_text(json.dumps(provenance, indent=2) + '\n') 235 (output / 'source-map.v1').write_text(map_text) 236 cmd = [tricu, 'arboricx', 'run', '--store', str(output / 'compiler-cas'), 237 '--module', adapter_manifest, '--name', 'main', 238 '--arg-export', compiler_manifest + ':' + worker_export, 239 '--arg', str(source_root), '--arg', str(output / 'overlay'), '--arg', str(output / 'cas'), 240 '--arg', compiler, '--arg', map_text, '--arg', args.root, 241 '--io', '--unsafe-io', '--result', '--format', 'decode', 242 '+RTS', '-N' + str(args.capabilities), '-M' + str(args.memory_mib) + 'm', '-T', '-s', '-RTS'] 243 if args.execution_host == 'zig': 244 request = dict(worker_export=worker_export, compiler_bundle_sha256=compiler, 245 adapter_sha256=provenance['adapter_sha256'], 246 arguments=[str(source_root), str(output / 'overlay'), str(output / 'cas'), 247 compiler, map_text, args.root]) 248 (output / 'execution-request.json').write_text(json.dumps(request, indent=2) + '\n') 249 import math 250 cmd = [sys.executable, str(REPO / 'test/zig_generation_executor.py'), 251 '--library', str(args.zig_library.resolve()), '--output', str(output), 252 '--memory-mib', str(args.memory_mib), '--seconds', str(math.ceil(remaining()))] 253 if args.zig_gc_policy: 254 cmd += ['--gc-policy', *map(str, args.zig_gc_policy)] 255 (output / 'invocation.json').write_text(json.dumps(cmd, indent=2)) 256 log = output / f'native-{time.time_ns()}.log' 257 (output / 'status.json').write_text(json.dumps(dict(complete=False, 258 reason='running or externally interrupted', log=str(log), **provenance), indent=2) + '\n') 259 start = last = time.monotonic() 260 seen, object_count, peak = {}, 0, 0 261 extended, reason = False, 'completed' 262 with log.open('w') as stdout: 263 proc = subprocess.Popen(cmd, stdout=stdout, stderr=stdout, start_new_session=True) 264 try: 265 monitor = ChildRSS(proc) 266 while proc.poll() is None: 267 for checkpoint in (output / 'cas/cache/source-build').glob('*'): 268 current = checkpoint.read_bytes() 269 if seen.get(checkpoint.name) != current: 270 seen[checkpoint.name] = current 271 last = time.monotonic(); extended = False 272 print(f'[{last-start:.1f}s] module-checkpoint={checkpoint.name}', flush=True) 273 objects = list((output / 'cas/objects').glob('*/*')) 274 if len(objects) != object_count: 275 object_count = len(objects) 276 last = time.monotonic(); extended = False 277 print(f'[{last-start:.1f}s] objects={object_count}', flush=True) 278 rss = monitor.sample() 279 if rss is not None: 280 peak = max(peak, rss) 281 if time.monotonic() - job_start > args.wall_seconds: 282 reason = 'wall-time guard'; break 283 if rss is not None and rss > args.memory_mib * 1024: 284 reason = 'RSS guard'; break 285 idle = time.monotonic() - last 286 if idle > args.extended_seconds: 287 reason = 'no-publication boundary'; break 288 if idle > args.initial_seconds and not extended: 289 extended = True 290 print(f'EXTEND publication boundary to {args.extended_seconds}s; peak={peak} KiB', flush=True) 291 time.sleep(0.25) 292 except (OSError, RuntimeError) as error: 293 reason = 'process monitoring failure: ' + str(error) 294 finally: 295 if proc.poll() is None: 296 os.killpg(proc.pid, signal.SIGTERM) 297 try: proc.wait(timeout=10) 298 except subprocess.TimeoutExpired: 299 os.killpg(proc.pid, signal.SIGKILL); proc.wait() 300 comparisons = {} 301 for name in required: 302 new = output / 'cas/cache/source-build' / name 303 if new.exists() and old_cas is not None: 304 old = old_cas / 'cache/source-build' / name 305 comparisons[name] = (old.exists() and old.read_text().split('\t')[2] == new.read_text().split('\t')[2]) 306 stable_inputs = inputs == snapshot(map_text, entries, source_root) and args.map.read_text() == map_text 307 success = (reason == 'completed' and proc.returncode == 0 308 and log.read_text().startswith('ok:') and stable_inputs) 309 audit = None 310 status = dict(code=proc.returncode, reason=reason, elapsed_seconds=time.monotonic()-start, 311 peak_kib=peak, complete=False, manifests_equal=comparisons, 312 inputs_unchanged=stable_inputs, log=str(log), **provenance) 313 (output / 'status.json').write_text(json.dumps(status, indent=2) + '\n') 314 preparation_seconds = start - job_start 315 compilation_seconds = time.monotonic() - start 316 audit_start = time.monotonic() 317 if success: 318 try: 319 reply, _ = run_guarded([sys.executable, str(REPO / 'test/bootstrap_store_check.py'), str(output)], 320 memory_mib=args.memory_mib, seconds=remaining()) 321 status['audit'] = reply 322 except (subprocess.CalledProcessError, RuntimeError, OSError) as error: 323 success = False 324 status['audit'] = str(error) + '\n' + getattr(error, 'output', '') 325 if not success: 326 status['reason'] = reason if reason != 'completed' else 'execution, input consistency or audit failure' 327 if args.expect_identical: 328 reproduced = len(comparisons) == len(required) and all(comparisons.values()) 329 success = success and reproduced 330 if not reproduced: 331 status['reason'] = 'manifest reproduction failure' 332 audit_seconds = time.monotonic() - audit_start 333 packing_start = time.monotonic() 334 if success and args.root == 'source_map': 335 # Publish ready-to-use next-generation artifacts using the native packer. 336 artifacts = {} 337 for module, export, name, filename in [ 338 ('source_map', worker_export, worker_export, 'rebuilt-compiler.arboricx'), 339 ('source_build', 'sourceBuildGraph', 'sourceBuildGraph', 'build.arboricx'), 340 ('source_build', 'sourceBuildGraphStable', 'sourceBuildGraph', 'build-stable.arboricx')]: 341 manifest = (output / 'cas/cache/source-build' / module).read_text().split('\t')[2] 342 rows = (output / 'cas/objects' / manifest[:3] / manifest).read_text().splitlines()[1:] 343 tree = next(row.split('\t')[3] for row in rows if row.split('\t')[1] == export) 344 destination = output / filename 345 print(transport(['arboricx', 'export', '--store', str(output / 'cas'), 346 '--target', tree, '--name', name, '--output', str(destination)]), end='') 347 artifacts[filename] = dict(tree=tree, sha256=hashlib.sha256(destination.read_bytes()).hexdigest()) 348 status['artifacts'] = artifacts 349 if args.expect_identical: 350 identical = {name: ((previous / name).is_file() and 351 hashlib.sha256((previous / name).read_bytes()).hexdigest() == item['sha256']) 352 for name, item in artifacts.items()} 353 status['artifacts_equal'] = identical 354 if not all(identical.values()): 355 success = False 356 status['reason'] = 'artifact reproduction failure' 357 if time.monotonic() - job_start > args.wall_seconds: 358 success = False 359 status['reason'] = 'whole-job wall-time guard' 360 status['timings'] = dict(preparation_seconds=preparation_seconds, 361 compilation_seconds=compilation_seconds, audit_seconds=audit_seconds, 362 packing_seconds=time.monotonic()-packing_start, 363 total_seconds=time.monotonic()-job_start) 364 status['limits'] = dict(memory_mib=args.memory_mib, wall_seconds=args.wall_seconds) 365 status['complete'] = success 366 (output / 'status.json').write_text(json.dumps(status, indent=2) + '\n') 367 print(json.dumps(status, indent=2), flush=True) 368 if not success: 369 print(log.read_text()[-3000:], flush=True) 370 raise SystemExit('STOP generation did not establish the selected verified closure') 371 372 373 if __name__ == '__main__': 374 main()