bootstrap_generations.py (25553B)
1 #!/usr/bin/env python3 2 """Opt-in artifact-only compiler rebuild; native packing, no source frontend.""" 3 import argparse 4 import fcntl 5 import hashlib 6 import json 7 import os 8 from pathlib import Path 9 import re 10 import signal 11 import subprocess 12 import sys 13 import time 14 from generation_backend import BACKENDS, DEFAULT_BACKEND 15 from bootstrap_inputs import load_kit, kit_io_version 16 from generation_io import versions, produced_adapter, execution_flags, cli_version 17 from process_rss import ChildRSS, run_guarded 18 19 REPO = Path(__file__).resolve().parent.parent 20 21 22 def source_map(path, source_root, root): 23 """Fail early on stale headers, incomplete edges, cycles and unsafe paths.""" 24 text = path.read_text() 25 lines = text.splitlines() 26 if not lines or lines[0] != 'arboricx.source-map.v1': 27 raise ValueError('unsupported source map') 28 entries = {} 29 for line in lines[1:]: 30 kind, alias, legacy, relative, deps = line.split('\t') 31 if kind not in ('unit', 'reexport', 'native') or alias in entries: 32 raise ValueError('duplicate alias or invalid map kind') 33 if not re.fullmatch(r'[A-Za-z0-9_-]+', alias): 34 raise ValueError('invalid alias') 35 relative_path = Path(relative) 36 if relative_path.is_absolute() or any(p in ('.', '..') for p in relative.split('/')): 37 raise ValueError('unsafe source path') 38 if not (source_root / relative_path).resolve().is_relative_to(source_root.resolve()): 39 raise ValueError('source path escapes root') 40 entries[alias] = (legacy, relative, [] if deps == '-' else deps.split(',')) 41 required, active = set(), set() 42 def visit(alias): 43 if alias in active: 44 raise ValueError('source map dependency cycle: ' + alias) 45 if alias in required: 46 return 47 if alias not in entries: 48 raise ValueError('undeclared map dependency: ' + alias) 49 active.add(alias) 50 for dep in entries[alias][2]: 51 visit(dep) 52 active.remove(alias) 53 required.add(alias) 54 visit(root) 55 # Staging reads every row, including entries outside the selected closure. 56 kinds = {row.split('\t')[1]: row.split('\t')[0] for row in lines[1:]} 57 for alias, (_, relative, deps) in entries.items(): 58 if any(dep not in entries for dep in deps): 59 raise ValueError('undeclared dependency: ' + alias) 60 native = kinds[alias] == 'native' 61 rows = (source_root / relative).read_text().splitlines() 62 if native: 63 from native_header import native_header 64 actual, _ = native_header('\n'.join(rows)) 65 if actual != deps: 66 raise ValueError('source map header mismatch: ' + alias) 67 else: 68 # Explicit recovery-map translation, never native import dispatch. 69 expected = [['!import', '"' + entries[dep][0] + '"', '!Local'] for dep in deps] 70 actual = [row.split() for row in rows[:len(deps)]] 71 if actual != expected or (len(rows) > len(deps) and rows[len(deps)].split()[:1] == ['!import']): 72 raise ValueError('source map header mismatch: ' + alias) 73 return text, entries, required 74 75 76 def snapshot(map_text, entries, source_root): 77 return dict(map_sha256=hashlib.sha256(map_text.encode()).hexdigest(), 78 sources={alias: hashlib.sha256((source_root / row[1]).read_bytes()).hexdigest() 79 for alias, row in entries.items()}) 80 81 82 def published_checkpoints(output, required): 83 """Observe only committed aliases, never same-directory private stages.""" 84 for name in required: 85 checkpoint = output / 'cas/cache/source-build' / name 86 try: 87 yield name, checkpoint.read_bytes() 88 except FileNotFoundError: 89 continue 90 91 92 def main(): 93 parser = argparse.ArgumentParser(description=__doc__) 94 parser.add_argument('previous', type=Path, nargs='?', help='completed generation; omit when selecting --bootstrap-kit') 95 parser.add_argument('output', type=Path) 96 parser.add_argument('--tricu', default='result/bin/tricu') 97 parser.add_argument('--execution-host', choices=('haskell', 'haskell-ffi'), default='haskell') 98 parser.add_argument('--haskell-library', type=Path, help='explicit full Haskell embedding library; required for haskell-ffi execution') 99 parser.add_argument('--backend', choices=BACKENDS, default=DEFAULT_BACKEND, 100 help='generation lowering backend (default: %(default)s); never inferred from previous') 101 parser.add_argument('--worker-bundle', type=Path, help='explicit pinned transition worker exporting the selected backend entrypoint; no fallback') 102 parser.add_argument('--execution-io-version', type=cli_version, choices=(1, 2), help='explicit transition-worker framing; otherwise selected kit/previous produced metadata') 103 parser.add_argument('--produced-io-version', type=cli_version, choices=(1, 2), help='declared output cohort framing; defaults to execution version') 104 parser.add_argument('--produced-adapter', type=Path, help='explicit output-cohort adapter; required when changing IO version') 105 parser.add_argument('--adapter', type=Path, help='precompiled standard six-argument IO adapter bundle; otherwise select previous produced adapter') 106 parser.add_argument('--bootstrap-kit', type=Path, help='explicit verified compiler/adapter kit; no previous store access') 107 parser.add_argument('--map', type=Path, default=REPO / 'bootstrap/compiler-source-map.v1') 108 parser.add_argument('--source-root', type=Path, default=REPO) 109 parser.add_argument('--root', default='source_map') 110 parser.add_argument('--memory-mib', type=int, default=32768) 111 parser.add_argument('--capabilities', type=int, default=1, help='GHC capabilities; explicit to avoid machine-wide -N defaults') 112 parser.add_argument('--wall-seconds', type=float, default=3600, 113 help='whole-job budget including preparation/audit/packing; max 7200 seconds') 114 parser.add_argument('--initial-seconds', type=float, default=600) 115 parser.add_argument('--extended-seconds', type=float, default=1800) 116 parser.add_argument('--expect-identical', action='store_true', help='require identical inputs, executable manifests and packaged artifacts') 117 parser.add_argument('--fresh', action='store_true', help='require a nonexistent output directory; no checkpoint reuse') 118 parser.add_argument('--validate-only', action='store_true') 119 args = parser.parse_args() 120 if not 0 < args.memory_mib <= 65536 or args.capabilities <= 0 or not 0 < args.wall_seconds <= 7200 or not 0 < args.initial_seconds <= args.extended_seconds: 121 parser.error('invalid resource boundaries') 122 if (args.execution_host == 'haskell-ffi') != (args.haskell_library is not None): 123 parser.error('--haskell-library is required exactly when --execution-host haskell-ffi is selected') 124 if args.haskell_library and not args.haskell_library.is_file(): 125 parser.error('selected Haskell embedding library is unavailable; no fallback') 126 previous = args.previous.resolve() if args.previous else None 127 output = args.output.resolve() 128 from bootstrap_retention_lock import protect_generations 129 retention_locks = protect_generations([previous, output]) 130 for selected in (previous, output): 131 if selected and any((selected / store / 'PRUNED-CAS.json').exists() 132 for store in ('cas', 'compiler-cas')): 133 parser.error('pruned generations are historical evidence, not resumable stores; ' 134 'select a retained input and a fresh output directory') 135 if args.bootstrap_kit: 136 if previous or args.adapter or args.worker_bundle or args.expect_identical: 137 parser.error('--bootstrap-kit replaces previous/adapter/worker selection; reproduction uses the first generation') 138 elif previous is None: 139 parser.error('select a previous generation or --bootstrap-kit') 140 source_root = args.source_root.resolve() 141 map_text, entries, required = source_map(args.map.resolve(), source_root, args.root) 142 inputs = snapshot(map_text, entries, source_root) 143 worker_export, backend_options = BACKENDS[args.backend] 144 prior_inputs = json.loads((previous / 'inputs.json').read_text()) if previous and (previous / 'inputs.json').exists() else {} 145 versions(prior_inputs) 146 if args.expect_identical and not args.validate_only: 147 prior_inputs = json.loads((previous / 'inputs.json').read_text()) 148 if (any(prior_inputs.get(key) != value for key, value in inputs.items()) 149 or prior_inputs.get('backend', 'optimized') != args.backend 150 or prior_inputs.get('root') != args.root): 151 parser.error('reproduction requires identical source/map inputs, root and backend') 152 if args.validate_only: 153 print(json.dumps(dict(root=args.root, backend=args.backend, worker_export=worker_export, 154 backend_options=backend_options, modules=sorted(required), **inputs), indent=2)) 155 return 156 if output == previous: 157 parser.error('output must differ from previous generation') 158 if args.fresh and output.exists(): 159 parser.error('--fresh requires a nonexistent output directory') 160 fresh_store = not (output / 'cas').exists() 161 kit_digest, kit_manifest, kit_payloads = None, None, None 162 old_cas, target = None, None 163 if args.bootstrap_kit: 164 kit_manifest, kit_payloads, kit_digest = load_kit(args.bootstrap_kit, args.backend) 165 adapter_bytes = kit_payloads['adapter'] 166 selected_io = kit_io_version(json.loads(kit_manifest)) 167 else: 168 selected_io = versions(prior_inputs)[1] 169 adapter_bytes = args.adapter.resolve().read_bytes() if args.adapter else produced_adapter(previous, prior_inputs) 170 old_cas = previous / 'cas' 171 _, _, manifest, _ = (old_cas / 'cache/source-build/source_map').read_text().split('\t') 172 manifest_bytes = (old_cas / 'objects' / manifest[:3] / manifest).read_bytes() 173 if hashlib.sha256(b'arboricx.module-manifest.v1\0' + manifest_bytes).hexdigest() != manifest: 174 raise ValueError('invalid previous compiler manifest') 175 rows = manifest_bytes.decode().splitlines()[1:] 176 target = next((row.split('\t')[3] for row in rows if row.split('\t')[1] == worker_export), None) 177 if target is None and not args.worker_bundle: 178 parser.error('previous compiler lacks selected entrypoint ' + worker_export + 179 '; supply an explicit --worker-bundle (no backend fallback)') 180 execution_io = args.execution_io_version if args.execution_io_version is not None else selected_io 181 if execution_io != selected_io and (args.bootstrap_kit or not args.worker_bundle): 182 parser.error('execution IO version disagrees with selected compiler metadata; select an explicit transition worker and adapter') 183 if execution_io != selected_io and not args.adapter: 184 parser.error('transition execution version requires an explicit matching adapter') 185 produced_io = args.produced_io_version if args.produced_io_version is not None else execution_io 186 grants = execution_flags(execution_io, args.execution_host) 187 if args.execution_host == 'haskell-ffi': 188 from haskell_generation_executor import validate_library 189 validate_library(args.haskell_library.resolve(),execution_io) 190 if produced_io != execution_io and not args.produced_adapter: 191 parser.error('changed produced IO version requires --produced-adapter') 192 produced_adapter_bytes = args.produced_adapter.resolve().read_bytes() if args.produced_adapter else adapter_bytes 193 produced_pin = hashlib.sha256(produced_adapter_bytes).hexdigest() 194 if args.expect_identical: 195 if produced_io != versions(prior_inputs)[1] or execution_io != versions(prior_inputs)[1]: 196 parser.error('reproduction requires the previous produced IO version') 197 if produced_adapter_bytes != produced_adapter(previous, prior_inputs): 198 parser.error('reproduction requires the previous produced adapter') 199 tricu = str(Path(args.tricu).resolve()) 200 output.mkdir(parents=True, exist_ok=True) 201 lock = (output / '.generation.lock').open('a') 202 try: 203 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) 204 except BlockingIOError: 205 parser.error('another generation is using this output directory') 206 # Do not let a failed/restarted invocation retain a stale success status. 207 (output / 'status.json').write_text(json.dumps(dict(complete=False, reason='preparation'))) 208 job_start = time.monotonic() 209 def remaining(): 210 budget = args.wall_seconds - (time.monotonic() - job_start) 211 if budget <= 0: 212 raise TimeoutError('whole-job wall-time guard') 213 return budget 214 def transport(command): 215 # Native packing/import should need far less than a full compilation. 216 reply, _ = run_guarded([tricu, *command, '+RTS', 217 '-N' + str(args.capabilities), '-M' + str(args.memory_mib) + 'm', '-RTS'], 218 memory_mib=args.memory_mib, seconds=min(600, remaining())) 219 return reply 220 bundle = output / 'compiler.arboricx' 221 if kit_payloads: 222 bundle.write_bytes(kit_payloads['compiler']) 223 (output / 'bootstrap-kit.json').write_bytes(kit_manifest) 224 elif args.worker_bundle: 225 bundle.write_bytes(args.worker_bundle.resolve().read_bytes()) 226 else: 227 print(transport(['arboricx', 'export', '--store', str(old_cas), '--target', target, 228 '--name', worker_export, '--output', str(bundle)]), end='') 229 compiler = hashlib.sha256(bundle.read_bytes()).hexdigest() 230 adapter = output / 'io-adapter.arboricx' 231 adapter.write_bytes(adapter_bytes) 232 (output / 'produced-io-adapter.arboricx').write_bytes(produced_adapter_bytes) 233 def imported(path, alias): 234 reply = transport(['arboricx', 'import', '--store', str(output / 'compiler-cas'), 235 '--file', str(path), '--module', alias, '--indexed']) 236 return reply.split('Created module alias ' + alias + ' -> ')[1].strip() 237 compiler_manifest = imported(bundle, 'compiler') 238 payload = (output / 'compiler-cas/objects' / compiler_manifest[:3] / compiler_manifest).read_text() 239 argument_export = next((row.split('\t') for row in payload.splitlines()[1:] 240 if row.split('\t')[1] == worker_export), None) 241 if argument_export is None: 242 parser.error('compiler bundle lacks selected entrypoint ' + worker_export + 243 ' (no backend fallback)') 244 # Transport object identity is distinct from the bundle identity used in receipts. 245 argument_kind, argument_hash = argument_export[2:4] 246 adapter_manifest = imported(adapter, 'io-adapter') 247 def require_adapter_entry(manifest): 248 data = (output / 'compiler-cas/objects' / manifest[:3] / manifest).read_text() 249 if not any(row.split('\t')[1] == 'main' for row in data.splitlines()[1:]): 250 parser.error('selected adapter lacks main export') 251 require_adapter_entry(adapter_manifest) 252 if produced_adapter_bytes != adapter_bytes: 253 require_adapter_entry(imported(output / 'produced-io-adapter.arboricx', 'produced-io-adapter')) 254 provenance = dict(execution_io_version=execution_io, produced_io_version=produced_io, 255 produced_adapter_sha256=produced_pin, source_root=str(source_root), root=args.root, backend=args.backend, 256 fresh_store=fresh_store, 257 worker_export=worker_export, backend_options=backend_options, 258 compiler_tree=argument_hash, compiler_tree_kind=argument_kind, 259 compiler_source_tree=None if args.worker_bundle else target, 260 bootstrap_kit_sha256=kit_digest, 261 compiler_bundle_sha256=compiler, compiler_manifest=compiler_manifest, 262 adapter_sha256=hashlib.sha256(adapter_bytes).hexdigest(), 263 execution_host=args.execution_host, 264 runtime_path=str(args.haskell_library.resolve()) if args.haskell_library else tricu, 265 runtime_sha256=hashlib.sha256((args.haskell_library or Path(tricu)).read_bytes()).hexdigest(), 266 transport_host='haskell', transport_runtime_path=tricu, 267 transport_runtime_sha256=hashlib.sha256(Path(tricu).read_bytes()).hexdigest(), 268 execution_capabilities=1 if args.haskell_library else args.capabilities, 269 binding_sha256=hashlib.sha256((REPO / 'ext/haskell/python/tricu.py').read_bytes()).hexdigest() if args.haskell_library else None, 270 executor_sha256=hashlib.sha256((REPO / 'test/haskell_generation_executor.py').read_bytes()).hexdigest() if args.haskell_library else None, 271 **inputs) 272 (output / 'inputs.json').write_text(json.dumps(provenance, indent=2) + '\n') 273 (output / 'source-map.v1').write_text(map_text) 274 cmd = [tricu, 'arboricx', 'run', '--store', str(output / 'compiler-cas'), 275 '--module', adapter_manifest, '--name', 'main', 276 '--arg-export', compiler_manifest + ':' + worker_export, 277 '--arg', str(source_root), '--arg', str(output / 'overlay'), '--arg', str(output / 'cas'), 278 '--arg', compiler, '--arg', map_text, '--arg', args.root, 279 '--io', *grants, '--result', '--format', 'decode', 280 '+RTS', '-N' + str(args.capabilities), '-M' + str(args.memory_mib) + 'm', '-T', '-s', '-RTS'] 281 if args.execution_host == 'haskell-ffi': 282 request = dict(execution_io_version=execution_io, worker_export=worker_export, compiler_bundle_sha256=compiler, 283 adapter_sha256=provenance['adapter_sha256'], 284 arguments=[str(source_root), str(output / 'overlay'), str(output / 'cas'), 285 compiler, map_text, args.root]) 286 (output / 'execution-request.json').write_text(json.dumps(request, indent=2) + '\n') 287 import math 288 cmd = [sys.executable, str(REPO / 'test/haskell_generation_executor.py'), 289 '--library', str(args.haskell_library.resolve()), '--output', str(output), 290 '--memory-mib', str(args.memory_mib), '--seconds', str(math.ceil(remaining()))] 291 (output / 'invocation.json').write_text(json.dumps(cmd, indent=2)) 292 log = output / f'native-{time.time_ns()}.log' 293 (output / 'status.json').write_text(json.dumps(dict(complete=False, 294 reason='running or externally interrupted', log=str(log), **provenance), indent=2) + '\n') 295 start = last = time.monotonic() 296 seen, object_count, peak = {}, 0, 0 297 extended, reason = False, 'completed' 298 with log.open('w') as stdout: 299 environment = dict(os.environ, MALLOC_ARENA_MAX='2') if args.haskell_library else None 300 proc = subprocess.Popen(cmd, stdout=stdout, stderr=stdout, start_new_session=True, env=environment) 301 try: 302 monitor = ChildRSS(proc) 303 while proc.poll() is None: 304 for name, current in published_checkpoints(output, required): 305 if seen.get(name) != current: 306 seen[name] = current 307 last = time.monotonic(); extended = False 308 print(f'[{last-start:.1f}s] module-checkpoint={name}', flush=True) 309 objects = [path for path in (output / 'cas/objects').glob('*/*') 310 if len(path.name) == 64 and all(c in '0123456789abcdef' for c in path.name)] 311 if len(objects) != object_count: 312 object_count = len(objects) 313 last = time.monotonic(); extended = False 314 print(f'[{last-start:.1f}s] objects={object_count}', flush=True) 315 rss = monitor.sample() 316 if rss is not None: 317 peak = max(peak, rss) 318 if time.monotonic() - job_start > args.wall_seconds: 319 reason = 'wall-time guard'; break 320 if rss is not None and rss > args.memory_mib * 1024: 321 reason = 'RSS guard'; break 322 idle = time.monotonic() - last 323 if idle > args.extended_seconds: 324 reason = 'no-publication boundary'; break 325 if idle > args.initial_seconds and not extended: 326 extended = True 327 print(f'EXTEND publication boundary to {args.extended_seconds}s; peak={peak} KiB', flush=True) 328 time.sleep(0.25) 329 except (OSError, RuntimeError) as error: 330 reason = 'process monitoring failure: ' + str(error) 331 finally: 332 if proc.poll() is None: 333 os.killpg(proc.pid, signal.SIGTERM) 334 try: proc.wait(timeout=10) 335 except subprocess.TimeoutExpired: 336 os.killpg(proc.pid, signal.SIGKILL); proc.wait() 337 comparisons = {} 338 for name in required: 339 new = output / 'cas/cache/source-build' / name 340 if new.exists() and old_cas is not None: 341 old = old_cas / 'cache/source-build' / name 342 comparisons[name] = (old.exists() and old.read_text().split('\t')[2] == new.read_text().split('\t')[2]) 343 stable_inputs = inputs == snapshot(map_text, entries, source_root) and args.map.read_text() == map_text 344 success = (reason == 'completed' and proc.returncode == 0 345 and log.read_text().startswith('ok:') and stable_inputs) 346 audit = None 347 status = dict(code=proc.returncode, reason=reason, elapsed_seconds=time.monotonic()-start, 348 peak_kib=peak, complete=False, manifests_equal=comparisons, 349 inputs_unchanged=stable_inputs, log=str(log), **provenance) 350 (output / 'status.json').write_text(json.dumps(status, indent=2) + '\n') 351 preparation_seconds = start - job_start 352 compilation_seconds = time.monotonic() - start 353 audit_start = time.monotonic() 354 if success: 355 try: 356 reply, _ = run_guarded([sys.executable, str(REPO / 'test/bootstrap_store_check.py'), str(output)], 357 memory_mib=args.memory_mib, seconds=remaining()) 358 status['audit'] = reply 359 except (subprocess.CalledProcessError, RuntimeError, OSError) as error: 360 success = False 361 status['audit'] = str(error) + '\n' + getattr(error, 'output', '') 362 if not success: 363 status['reason'] = reason if reason != 'completed' else 'execution, input consistency or audit failure' 364 if args.expect_identical: 365 reproduced = len(comparisons) == len(required) and all(comparisons.values()) 366 if success and not reproduced: 367 status['reason'] = 'manifest reproduction failure' 368 success = success and reproduced 369 audit_seconds = time.monotonic() - audit_start 370 packing_start = time.monotonic() 371 if success and args.root == 'source_map': 372 # Publish ready-to-use next-generation artifacts using the native packer. 373 artifacts = {} 374 for module, export, name, filename in [ 375 ('source_map', worker_export, worker_export, 'rebuilt-compiler.arboricx'), 376 ('source_build', 'sourceBuildGraph', 'sourceBuildGraph', 'build.arboricx'), 377 ('source_build', 'sourceBuildGraphStable', 'sourceBuildGraph', 'build-stable.arboricx')]: 378 manifest = (output / 'cas/cache/source-build' / module).read_text().split('\t')[2] 379 rows = (output / 'cas/objects' / manifest[:3] / manifest).read_text().splitlines()[1:] 380 tree = next(row.split('\t')[3] for row in rows if row.split('\t')[1] == export) 381 destination = output / filename 382 print(transport(['arboricx', 'export', '--store', str(output / 'cas'), 383 '--target', tree, '--name', name, '--output', str(destination)]), end='') 384 artifacts[filename] = dict(tree=tree, sha256=hashlib.sha256(destination.read_bytes()).hexdigest()) 385 status['artifacts'] = artifacts 386 if args.expect_identical: 387 identical = {name: ((previous / name).is_file() and 388 hashlib.sha256((previous / name).read_bytes()).hexdigest() == item['sha256']) 389 for name, item in artifacts.items()} 390 status['artifacts_equal'] = identical 391 if not all(identical.values()): 392 success = False 393 status['reason'] = 'artifact reproduction failure' 394 if time.monotonic() - job_start > args.wall_seconds: 395 success = False 396 status['reason'] = 'whole-job wall-time guard' 397 status['timings'] = dict(preparation_seconds=preparation_seconds, 398 compilation_seconds=compilation_seconds, audit_seconds=audit_seconds, 399 packing_seconds=time.monotonic()-packing_start, 400 total_seconds=time.monotonic()-job_start) 401 status['limits'] = dict(memory_mib=args.memory_mib, wall_seconds=args.wall_seconds) 402 status['complete'] = success 403 (output / 'status.json').write_text(json.dumps(status, indent=2) + '\n') 404 print(json.dumps(status, indent=2), flush=True) 405 if not success: 406 print(log.read_text()[-3000:], flush=True) 407 raise SystemExit('STOP generation did not establish the selected verified closure') 408 409 410 if __name__ == '__main__': 411 main()