native_library_migration.py (23951B)
1 #!/usr/bin/env python3 2 """Compile library source candidates with immutable dependency manifests. 3 4 Prepared sources preserve bodies, import order and namespaces. Compilation uses 5 complete dependency manifests and writes candidates and receipts to the selected 6 output directory for review. 7 """ 8 import argparse 9 import hashlib 10 import json 11 import os 12 from pathlib import Path 13 import re 14 import signal 15 import subprocess 16 import time 17 from process_rss import ChildRSS 18 from generation_io import cli_version 19 from generation_backend import BACKENDS, DEFAULT_BACKEND 20 21 REPO = Path(__file__).resolve().parent.parent 22 LIBRARIES = { 23 'io': 'lib/io.tri', 24 'socket': 'lib/socket.tri', 'binary': 'lib/binary.tri', 25 'parser': 'lib/parser.tri', 'patterns': 'lib/patterns.tri', 26 'intensional': 'lib/intensionalContracts.tri', 'http': 'lib/http.tri', 27 'arboricx.common': 'lib/arboricx/common.tri', 28 'arboricx.nodes': 'lib/arboricx/nodes.tri', 29 'arboricx.manifest': 'lib/arboricx/manifest.tri', 30 'arboricx': 'lib/arboricx/arboricx.tri', 31 'arboricx.dispatch': 'lib/arboricx/dispatch.tri', 32 'arboricx.server': 'lib/arboricx/server.tri', 33 } 34 for path in sorted((REPO / 'lib/source').glob('*.tri')): 35 LIBRARIES['source.' + path.stem] = str(path.relative_to(REPO)) 36 LIBRARIES['source.build-stable'] = 'lib/source/build-stable.tri' 37 LIBRARIES['source.export-cache'] = 'lib/source/export-cache.tri' 38 for name in ['staging-experiment', 'staged-list-experiment', 'staged-conditional-experiment']: 39 LIBRARIES['source.' + name] = 'lib/source/' + name + '.tri' 40 LIBRARIES['tests.httpEffects'] = 'test/httpEffects.tri' 41 LIBRARIES['tests.registryEffects'] = 'test/registryEffects.tri' 42 LIBRARIES['tests.localControlEffects'] = 'test/localControlEffects.tri' 43 LIBRARIES['tests.parserControl'] = 'test/parserControl.tri' 44 LIBRARIES['tests.stagingExperiment'] = 'test/stagingExperiment.tri' 45 LIBRARIES['tests.stagedConditionalRunner'] = 'test/stagedConditionalRunner.tri' 46 # Standard applications. 47 for path in sorted((REPO / 'demos').glob('*.tri')) + sorted((REPO / 'demos/interactionTrees').glob('*.tri')): 48 LIBRARIES['demos.' + '.'.join(path.relative_to(REPO / 'demos').with_suffix('').parts)] = str(path.relative_to(REPO)) 49 50 51 # Explicit IO support-cohort additions; aliases match authored workspace imports. 52 IO_SUPPORT = { 53 'arboricx_handlers': 'lib/arboricx/handlers.tri', 54 'arboricx_streaming': 'lib/arboricx/streaming.tri', 55 'handler_artifacts': 'lib/handlers/artifacts.tri', 56 'handler_log': 'lib/handlers/log.tri', 57 'handlers': 'lib/handlers.tri', 58 'io': 'lib/io.tri', 59 'io_artifacts': 'lib/io/artifacts.tri', 60 'io_http': 'lib/io/http-transport.tri', 61 'io_storage': 'lib/io/storage.tri', 62 'io_text': 'lib/io/text.tri', 63 } 64 LIBRARIES.update(IO_SUPPORT) 65 66 # Closed binary64 support cohort; public facade, private providers and demo. 67 BINARY64_SUPPORT = {'binary64': 'lib/math/binary64.tri'} 68 for name in ['adders', 'chains', 'fast', 'finite', 'ieee', 'kernels', 'layouts', 69 'near', 'product_quotient', 'quantize', 'working']: 70 BINARY64_SUPPORT['binary64_' + name] = 'lib/math/binary64/' + name + '.tri' 71 BINARY64_SUPPORT['demos.binary64'] = 'demos/binary64.tri' 72 LIBRARIES.update(BINARY64_SUPPORT) 73 74 75 def alias(name): 76 return name.replace('.', '_').replace('-', '_') 77 78 79 def native_candidate(raw, names, extra_imports=()): 80 rows = raw.splitlines(keepends=True) 81 if any(row.startswith(b'!export ') for row in rows): 82 from native_header import native_header 83 declared, imports = native_header(raw.decode()) 84 if any(dep not in names.values() for dep in declared): 85 raise ValueError('unknown native dependency') 86 return raw, imports 87 imports, body = list(extra_imports), [] 88 for row in rows: 89 if row.startswith(b'!import'): 90 match = re.fullmatch(rb'!import "([^"\n]+)"\s+([^\s]+)\r?\n', row) 91 if not match: 92 raise ValueError('unsupported legacy import') 93 name, namespace = (value.decode() for value in match.groups()) 94 if name not in names: 95 raise ValueError('unknown dependency: ' + name) 96 imports.append((names[name], namespace)) 97 else: 98 body.append(row) 99 header = ''.join('!import ' + name + (' *' if ns == '!Local' else ' as ' + ns) + '\n' 100 for name, ns in imports) 101 header += '!export *\n' 102 return header.encode() + b''.join(body), imports 103 104 105 def terminal_export(raw, symbol): 106 """Rewrite reviewed bare selectors.""" 107 if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', symbol): 108 raise ValueError('invalid terminal export name') 109 rows = raw.splitlines(keepends=True) 110 last = next((i for i in range(len(rows)-1, -1, -1) if rows[i].strip()), None) 111 if last is None or rows[last].strip() != symbol.encode(): 112 raise ValueError('expected exact bare terminal selector') 113 if not any(re.match(rb'^' + symbol.encode() + rb'(?:[ \t]+[^=\n]+)?[ \t]*=', row) for row in rows[:last]): 114 raise ValueError('terminal selector has no authored definition') 115 del rows[last] 116 return b''.join(rows) 117 118 119 def reexport_candidate(raw, candidate, imports): 120 """Import-only compatibility examples, with explicit unchanged export intent.""" 121 if any(row.strip() and not row.startswith((b'--', b'!import', b'!source', b'!export')) for row in raw.splitlines()): 122 raise ValueError('reexport shim must contain only comments/import policy') 123 if any(ns != '!Local' for _, ns in imports): 124 raise ValueError('reexport shim requires local imports') 125 missing = [dep for dep, _ in imports if ('!export * from ' + dep).encode() not in candidate.splitlines()] 126 if b'!export *\n' not in candidate: 127 raise ValueError('reexport shim requires explicit local export policy') 128 return candidate.replace(b'!export *\n', b'!export *\n' + ''.join('!export * from ' + dep + '\n' for dep in missing).encode(), 1) 129 130 131 def digest(domain, payload): 132 return hashlib.sha256(domain.encode() + b'\0' + payload).hexdigest() 133 134 135 def checked_candidate_source(workspace, record, source_root=REPO): 136 source = (source_root / record['path']).read_bytes() 137 staged = (workspace / (record['alias'] + '.tri')).read_bytes() 138 if source != staged or hashlib.sha256(staged).hexdigest() != record['after_sha256']: 139 raise ValueError('staged candidate source changed: ' + record['alias']) 140 return staged 141 142 143 def dependency_sources(source_root, entries, reserved): 144 """Stage explicit source inputs alongside authenticated providers.""" 145 records, seen = [], set(reserved) 146 source_root = source_root.resolve() 147 for entry in entries: 148 name, separator, relative = entry.partition('=') 149 path = Path(relative) 150 if (not separator or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]*', name) or 151 name in seen or path.is_absolute() or '..' in path.parts or 152 not relative or not (source_root / path).resolve().is_relative_to(source_root)): 153 raise ValueError('invalid/duplicate source dependency: ' + entry) 154 payload = (source_root / path).read_bytes() 155 records.append(dict(alias=name, path=relative, after_sha256=hashlib.sha256(payload).hexdigest())) 156 seen.add(name) 157 return records 158 159 160 def check_source_dependencies(directory, source_root): 161 selection = json.loads((directory / 'selection.json').read_text()) 162 for record in selection.get('source_dependencies', []): 163 checked_candidate_source(directory / 'workspace', record, source_root) 164 165 166 def main(): 167 p = argparse.ArgumentParser(description=__doc__) 168 p.add_argument('tricu', type=Path) 169 p.add_argument('generation', type=Path, nargs='?', help='completed generation; omit only with explicit compiler bundle/providers') 170 p.add_argument('--compiler-bundle', type=Path) 171 p.add_argument('--compiler-providers', type=Path, help='explicit namespaced compiler provider bundle, not a fabricated generation') 172 p.add_argument('--compiler-io-version', type=cli_version, choices=(1,2)) 173 p.add_argument('--source-root', type=Path, default=REPO) 174 p.add_argument('directory', type=Path) 175 p.add_argument('--library', action='append', choices=list(LIBRARIES), required=True) 176 p.add_argument('--terminal-export', action='append', default=[], metavar='LIBRARY:SYMBOL', help='explicitly remove a reviewed bare terminal selector; keep/export its definition') 177 p.add_argument('--reexport-imports', action='append', default=[], help='explicitly retain an import-only shim as full local dependency reexports') 178 p.add_argument('--backend', choices=BACKENDS, default=DEFAULT_BACKEND, 179 help='application lowering backend (default: %(default)s)') 180 p.add_argument('--keep-going', action='store_true', help='record independent failures without blocking other candidates') 181 p.add_argument('--seconds', type=int, default=1200, help='per-command time guard') 182 p.add_argument('--batch-seconds', type=int, default=1200, help='whole batch guard, maximum 7200') 183 p.add_argument('--memory-mib', type=int, default=32768, help='sampled direct-child RSS / host heap ceiling') 184 p.add_argument('--capabilities', type=int, default=1, help='GHC capabilities; select eight for a full support rebuild') 185 p.add_argument('--resume', action='store_true', help='reuse only unchanged staged candidates/checkpoints') 186 p.add_argument('--dependencies', action='append', type=Path, default=[], help='verified completed candidate directories supplying full dependency manifests') 187 p.add_argument('--source-dependency', action='append', default=[], metavar='ALIAS=PATH', 188 help='explicit additional authored provider; no compiler/support alias overrides') 189 args = p.parse_args() 190 if not 1 <= args.seconds <= 7200 or not 1 <= args.batch_seconds <= 7200 or not 0 < args.memory_mib <= 65536 or args.capabilities < 1: 191 p.error('jobs must be bounded within 64 GiB and two hours') 192 if len(set(args.reexport_imports)) != len(args.reexport_imports) or not set(args.reexport_imports) <= set(args.library): 193 p.error('reexport policy must name selected candidates exactly once') 194 root = args.directory.resolve() 195 if args.resume: 196 if not root.is_dir(): 197 p.error('resume directory is missing') 198 else: 199 root.mkdir(exist_ok=False, parents=True) 200 from generation_io import versions, compiler_flags 201 generation = args.generation.resolve() if args.generation else None 202 source_root = args.source_root.resolve() 203 if generation: 204 if args.compiler_bundle or args.compiler_providers: 205 p.error('select generation or explicit compiler/provider bundles, not both') 206 assert json.loads((generation / 'status.json').read_text())['complete'] 207 io_version = versions(json.loads((generation / 'inputs.json').read_text()))[1] 208 else: 209 if not args.compiler_bundle or not args.compiler_providers: 210 p.error('explicit --compiler-bundle and --compiler-providers required without generation') 211 io_version = args.compiler_io_version or 1 212 if args.compiler_io_version is not None and args.compiler_io_version != io_version: 213 p.error('compiler IO override disagrees with generation metadata') 214 events = json.loads((root / 'events.json').read_text()) if args.resume else [] 215 deadline = time.monotonic() + args.batch_seconds 216 217 def run(arguments): 218 remaining = min(args.seconds, deadline - time.monotonic()) 219 if remaining <= 0: 220 raise RuntimeError('candidate-batch time guard') 221 command = [str(args.tricu.resolve()), *map(str, arguments), '+RTS', '-N' + str(args.capabilities), 222 '-M' + str(args.memory_mib) + 'm', '-RTS'] 223 started, peak, reason = time.monotonic(), 0, 'completed' 224 number = len(events) 225 with (root / f'{number}.stdout').open('w') as out, (root / f'{number}.stderr').open('w') as err: 226 proc = subprocess.Popen(command, stdout=out, stderr=err, start_new_session=True) 227 try: 228 monitor = ChildRSS(proc) 229 while proc.poll() is None: 230 sample = monitor.sample() 231 if sample is not None: 232 peak = max(peak, sample) 233 if peak > args.memory_mib * 1024: 234 reason = 'RSS guard'; break 235 if time.monotonic() - started > remaining: 236 reason = 'time guard'; break 237 time.sleep(0.05) 238 except (OSError, RuntimeError) as error: 239 reason = 'monitoring failure: ' + str(error) 240 finally: 241 if proc.poll() is None: 242 os.killpg(proc.pid, signal.SIGTERM) 243 try: 244 proc.wait(timeout=5) 245 except subprocess.TimeoutExpired: 246 os.killpg(proc.pid, signal.SIGKILL); proc.wait() 247 stdout = (root / f'{number}.stdout').read_text() 248 stderr = (root / f'{number}.stderr').read_text() 249 events.append(dict(command=command, seconds=time.monotonic()-started, 250 peak_kib=peak, reason=reason, code=proc.returncode, 251 stdout=stdout, stderr=stderr)) 252 (root / 'events.json').write_text(json.dumps(events, indent=2) + '\n') 253 if reason != 'completed' or proc.returncode: 254 raise RuntimeError(f'{reason}; code={proc.returncode}; {stderr}') 255 return stdout 256 257 tools, store, workspace = root / 'tools', root / 'application', root / 'workspace' 258 workspace.mkdir(exist_ok=args.resume); store.mkdir(exist_ok=args.resume) 259 # Use an isolated workspace to resolve staged sources and facades. 260 (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n') 261 bundle = generation / ('build-stable.arboricx' if args.backend == 'stable' else 'build.arboricx') if generation else args.compiler_bundle.resolve() 262 map_path = generation / 'source-map.v1' if generation else source_root / 'bootstrap/compiler-source-map.v1' 263 map_bytes = map_path.read_bytes() 264 selection = dict(backend=args.backend, map_sha256=hashlib.sha256(map_bytes).hexdigest(), compiler_sha256=hashlib.sha256(bundle.read_bytes()).hexdigest(), 265 compiler_io_version=io_version, source_root=str(source_root), 266 providers_sha256=hashlib.sha256(args.compiler_providers.read_bytes()).hexdigest() if args.compiler_providers else None) 267 mapped_names = {row.split('\t')[1] for row in map_bytes.decode().splitlines()[1:]} 268 source_dependencies = dependency_sources(source_root, args.source_dependency, mapped_names | {alias(name) for name in LIBRARIES}) 269 if source_dependencies: 270 selection['source_dependencies'] = source_dependencies 271 selection_file = root / 'selection.json' 272 if args.resume and (not selection_file.exists() or json.loads(selection_file.read_text()) != selection): 273 raise ValueError('resume compiler/provider/source selection changed') 274 selection_file.write_text(json.dumps(selection, indent=2)+'\n') 275 output = run(['arboricx', 'import', '--file', bundle, '--store', tools, '--indexed']) 276 compiler = re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 277 rows = [line.split('\t') for line in map_bytes.decode().splitlines()[1:]] 278 names = {legacy: mapped for _, mapped, legacy, _, _ in rows} 279 names.update({name: alias(name) for name in LIBRARIES}) 280 names.update({record['alias']: record['alias'] for record in source_dependencies}) 281 names.update({ 282 'STAGING_EXPERIMENT_MANIFEST': alias('source.staging-experiment'), 283 'STAGED_LIST_EXPERIMENT_MANIFEST': alias('source.staged-list-experiment'), 284 'STAGED_CONDITIONAL_COMPILER_MANIFEST': alias('source.staged-conditional-experiment'), 285 }) 286 selectors = {} 287 for entry in args.terminal_export: 288 name, symbol = entry.split(':', 1) 289 if name not in args.library or name in selectors: 290 raise ValueError('terminal export must name one selected candidate exactly once') 291 selectors[name] = symbol 292 records = [] 293 needed = set() 294 for record in source_dependencies: 295 payload = (source_root / record['path']).read_bytes() 296 candidate, _ = native_candidate(payload, names) 297 if candidate != payload: 298 raise ValueError('source dependency must already be a native module') 299 from native_header import native_header 300 needed.update(native_header(payload.decode())[0]) 301 staged = workspace / (record['alias'] + '.tri') 302 if args.resume and staged.read_bytes() != payload: 303 raise ValueError('resume source dependency changed') 304 staged.write_bytes(payload) 305 # Stage candidates before compiling so their declared edges are resolvable. 306 for name in args.library: 307 raw = (source_root / LIBRARIES[name]).read_bytes() 308 # Legacy patterns imports exposed prelude transitively. Native patterns 309 # owns only its declared exports; this demo uses equal? directly. 310 extra = [(names['prelude'], '!Local')] if name == 'demos.patternMatching' else [] 311 candidate, imports = native_candidate(terminal_export(raw, selectors[name]) if name in selectors else raw, names, extra) 312 if name in args.reexport_imports: 313 candidate = reexport_candidate(raw, candidate, imports) 314 staged = workspace / (alias(name) + '.tri') 315 if args.resume and staged.read_bytes() != candidate: 316 raise ValueError('resume candidate changed: ' + name) 317 staged.write_bytes(candidate) 318 from native_header import native_header 319 needed.update(native_header(candidate.decode())[0]) 320 records.append(dict(legacy=name, alias=alias(name), path=LIBRARIES[name], 321 before_sha256=hashlib.sha256(raw).hexdigest(), 322 after_sha256=hashlib.sha256(candidate).hexdigest(), 323 body_unchanged=name not in selectors, imports=imports, 324 terminal_export=selectors.get(name), definition_bodies_unchanged=True, 325 reexport_imports=name in args.reexport_imports, 326 explicit_extra_imports=extra, complete=False)) 327 if args.resume: 328 previous = json.loads((root / 'migration.json').read_text()) 329 if [(r['alias'], r['after_sha256']) for r in previous] != [(r['alias'], r['after_sha256']) for r in records]: 330 raise ValueError('resume membership or inputs changed') 331 records = previous 332 (root / 'migration.json').write_text(json.dumps(records, indent=2) + '\n') 333 cas = generation / 'cas' if generation else None 334 for _, mapped, _, _, _ in rows: 335 if mapped not in needed or mapped in {record['alias'] for record in records}: 336 continue 337 if cas is None: 338 from bootstrap_archive import select_exports 339 from bundle_dag import read 340 provider = root / ('provider-' + mapped + '.arboricx') 341 exports = read(args.compiler_providers)[3] 342 selected = [(key, key.split('.', 1)[1]) for key in exports if key.startswith(mapped + '.')] 343 if not selected: 344 raise ValueError('missing complete compiler provider: ' + mapped) 345 select_exports(args.compiler_providers, provider, selected) 346 reply = run(['arboricx', 'import', '--file', provider, '--store', store, '--indexed']) 347 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 348 (workspace / (mapped + '.tri')).write_text('!export * from ' + pin + '\n') 349 continue 350 pin = (cas / 'cache/source-build' / mapped).read_text().split('\t')[2] 351 payload = (cas / 'objects' / pin[:3] / pin).read_bytes() 352 assert digest('arboricx.module-manifest.v1', payload) == pin 353 for h in {pin, *[line.split('\t')[3] for line in payload.decode().splitlines()[1:]]}: 354 target = store / 'objects' / h[:3] / h 355 target.parent.mkdir(exist_ok=True, parents=True) 356 if not target.exists(): 357 os.link(cas / 'objects' / h[:3] / h, target) 358 (workspace / (mapped + '.tri')).write_text('!export * from ' + pin + '\n') 359 for dependency_directory in args.dependencies: 360 dependency_directory = dependency_directory.resolve() 361 dependency_store = dependency_directory / 'application' 362 for dependency in json.loads((dependency_directory / 'migration.json').read_text()): 363 mapped = dependency['alias'] 364 if mapped not in needed or mapped in [r['alias'] for r in records]: 365 continue 366 source = (source_root / dependency['path']).read_bytes() 367 if not dependency['complete'] or hashlib.sha256(source).hexdigest() != dependency['after_sha256']: 368 raise ValueError('stale/incomplete dependency candidate: ' + mapped) 369 pin = dependency['manifest'] 370 payload = (dependency_store / 'objects' / pin[:3] / pin).read_bytes() 371 if digest('arboricx.module-manifest.v1', payload) != pin: 372 raise ValueError('corrupt dependency manifest') 373 for h in {pin, *[line.split('\t')[3] for line in payload.decode().splitlines()[1:]]}: 374 target = store / 'objects' / h[:3] / h 375 target.parent.mkdir(exist_ok=True, parents=True) 376 if not target.exists(): 377 os.link(dependency_store / 'objects' / h[:3] / h, target) 378 (workspace / (mapped + '.tri')).write_text('!export * from ' + pin + '\n') 379 failures = [] 380 for record in records: 381 name = record['alias'] 382 checked_candidate_source(workspace, record, source_root) 383 if record['complete']: 384 continue 385 try: 386 run(['arboricx', 'compile', '--file', workspace / (name + '.tri'), 387 '--toolchain', compiler, *compiler_flags(io_version), '--tool-store', tools, '--store', store, 388 '--output', root / (name + '.arboricx'), '--compile-timeout', str(args.seconds)]) 389 except RuntimeError as error: 390 if not args.keep_going: 391 raise 392 failures.append(name) 393 print('Failed:', name, str(error), flush=True) 394 continue 395 checkpoint = (store / 'cache/source-build' / name).read_text().split('\t') 396 manifest = (store / 'objects' / checkpoint[2][:3] / checkpoint[2]).read_bytes() 397 receipt = (store / 'objects' / checkpoint[3][:3] / checkpoint[3]).read_bytes() 398 assert digest('arboricx.module-manifest.v1', manifest) == checkpoint[2] 399 assert digest('arboricx.build-receipt.v1', receipt) == checkpoint[3] 400 assert 'source\t' + digest('arboricx.source.v1', (workspace / (name + '.tri')).read_bytes()) in receipt.decode().splitlines() 401 checked_candidate_source(workspace, record, source_root) 402 record.update(complete=True, manifest=checkpoint[2], receipt=checkpoint[3]) 403 (root / 'migration.json').write_text(json.dumps(records, indent=2) + '\n') 404 print('Completed:', name, flush=True) 405 for record in records: 406 checked_candidate_source(workspace, record, source_root) 407 check_source_dependencies(root, source_root) 408 if map_path.read_bytes() != map_bytes: 409 raise ValueError('source map changed during preparation') 410 if hashlib.sha256(bundle.read_bytes()).hexdigest() != selection['compiler_sha256'] or (args.compiler_providers and 411 hashlib.sha256(args.compiler_providers.read_bytes()).hexdigest() != selection['providers_sha256']): 412 raise ValueError('compiler/provider inputs changed during preparation') 413 if failures: 414 raise SystemExit('Incomplete candidates: ' + ', '.join(failures)) 415 416 417 if __name__ == '__main__': 418 main()