native_library_migration.py (16971B)
1 #!/usr/bin/env python3 2 """Explicit portable compilation of unmapped library header candidates. 3 4 Preserves bodies/import order/namespaces and uses FULL immutable dependency 5 manifests, not selected-function facades or the recovery frontend. Does not edit 6 repository sources or publish fixtures; retained records support reviewed slices. 7 """ 8 import argparse 9 import hashlib 10 import json 11 import os 12 from pathlib import Path 13 import re 14 import signal 15 import subprocess 16 import time 17 from process_rss import ChildRSS 18 from generation_backend import BACKENDS, DEFAULT_BACKEND 19 20 REPO = Path(__file__).resolve().parent.parent 21 LIBRARIES = { 22 'socket': 'lib/socket.tri', 'binary': 'lib/binary.tri', 23 'parser': 'lib/parser.tri', 'patterns': 'lib/patterns.tri', 24 'intensional': 'lib/intensionalContracts.tri', 'http': 'lib/http.tri', 25 'arboricx.common': 'lib/arboricx/common.tri', 26 'arboricx.nodes': 'lib/arboricx/nodes.tri', 27 'arboricx.manifest': 'lib/arboricx/manifest.tri', 28 'arboricx': 'lib/arboricx/arboricx.tri', 29 'arboricx.dispatch': 'lib/arboricx/dispatch.tri', 30 'arboricx.server': 'lib/arboricx/server.tri', 31 } 32 for path in sorted((REPO / 'lib/source').glob('*.tri')): 33 LIBRARIES['source.' + path.stem] = str(path.relative_to(REPO)) 34 LIBRARIES['source.build-stable'] = 'lib/source/build-stable.tri' 35 LIBRARIES['source.export-cache'] = 'lib/source/export-cache.tri' 36 for name in ['staging-experiment', 'staged-list-experiment', 'staged-conditional-experiment']: 37 LIBRARIES['source.' + name] = 'lib/source/' + name + '.tri' 38 LIBRARIES['tests.httpEffects'] = 'test/httpEffects.tri' 39 LIBRARIES['tests.registryEffects'] = 'test/registryEffects.tri' 40 LIBRARIES['tests.localControlEffects'] = 'test/localControlEffects.tri' 41 LIBRARIES['tests.parserControl'] = 'test/parserControl.tri' 42 LIBRARIES['tests.stagingExperiment'] = 'test/stagingExperiment.tri' 43 LIBRARIES['tests.stagedConditionalRunner'] = 'test/stagedConditionalRunner.tri' 44 # Standard applications. 45 for path in sorted((REPO / 'demos').glob('*.tri')) + sorted((REPO / 'demos/interactionTrees').glob('*.tri')): 46 LIBRARIES['demos.' + '.'.join(path.relative_to(REPO / 'demos').with_suffix('').parts)] = str(path.relative_to(REPO)) 47 48 49 def alias(name): 50 return name.replace('.', '_').replace('-', '_') 51 52 53 def native_candidate(raw, names, extra_imports=()): 54 rows = raw.splitlines(keepends=True) 55 if any(row.startswith(b'!export ') for row in rows): 56 from native_header import native_header 57 declared, imports = native_header(raw.decode()) 58 if any(dep not in names.values() for dep in declared): 59 raise ValueError('unknown native dependency') 60 return raw, imports 61 imports, body = list(extra_imports), [] 62 for row in rows: 63 if row.startswith(b'!import'): 64 match = re.fullmatch(rb'!import "([^"\n]+)"\s+([^\s]+)\r?\n', row) 65 if not match: 66 raise ValueError('unsupported legacy import') 67 name, namespace = (value.decode() for value in match.groups()) 68 if name not in names: 69 raise ValueError('unknown dependency: ' + name) 70 imports.append((names[name], namespace)) 71 else: 72 body.append(row) 73 header = ''.join('!import ' + name + (' *' if ns == '!Local' else ' as ' + ns) + '\n' 74 for name, ns in imports) 75 header += '!export *\n' 76 return header.encode() + b''.join(body), imports 77 78 79 def terminal_export(raw, symbol): 80 """Reviewed bare selector only; never rewrite an arbitrary expression.""" 81 if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', symbol): 82 raise ValueError('invalid terminal export name') 83 rows = raw.splitlines(keepends=True) 84 last = next((i for i in range(len(rows)-1, -1, -1) if rows[i].strip()), None) 85 if last is None or rows[last].strip() != symbol.encode(): 86 raise ValueError('expected exact bare terminal selector') 87 if not any(re.match(rb'^' + symbol.encode() + rb'(?:[ \t]+[^=\n]+)?[ \t]*=', row) for row in rows[:last]): 88 raise ValueError('terminal selector has no authored definition') 89 del rows[last] 90 return b''.join(rows) 91 92 93 def reexport_candidate(raw, candidate, imports): 94 """Import-only compatibility examples, with explicit unchanged export intent.""" 95 if any(row.strip() and not row.startswith((b'--', b'!import', b'!source', b'!export')) for row in raw.splitlines()): 96 raise ValueError('reexport shim must contain only comments/import policy') 97 if any(ns != '!Local' for _, ns in imports): 98 raise ValueError('reexport shim requires local imports') 99 missing = [dep for dep, _ in imports if ('!export * from ' + dep).encode() not in candidate.splitlines()] 100 if b'!export *\n' not in candidate: 101 raise ValueError('reexport shim requires explicit local export policy') 102 return candidate.replace(b'!export *\n', b'!export *\n' + ''.join('!export * from ' + dep + '\n' for dep in missing).encode(), 1) 103 104 105 def digest(domain, payload): 106 return hashlib.sha256(domain.encode() + b'\0' + payload).hexdigest() 107 108 109 def checked_candidate_source(workspace, record, source_root=REPO): 110 source = (source_root / record['path']).read_bytes() 111 staged = (workspace / (record['alias'] + '.tri')).read_bytes() 112 if source != staged or hashlib.sha256(staged).hexdigest() != record['after_sha256']: 113 raise ValueError('staged candidate source changed: ' + record['alias']) 114 return staged 115 116 117 def main(): 118 p = argparse.ArgumentParser(description=__doc__) 119 p.add_argument('tricu', type=Path) 120 p.add_argument('generation', type=Path) 121 p.add_argument('directory', type=Path) 122 p.add_argument('--library', action='append', choices=list(LIBRARIES), required=True) 123 p.add_argument('--terminal-export', action='append', default=[], metavar='LIBRARY:SYMBOL', help='explicitly remove a reviewed bare terminal selector; keep/export its definition') 124 p.add_argument('--reexport-imports', action='append', default=[], help='explicitly retain an import-only shim as full local dependency reexports') 125 p.add_argument('--backend', choices=BACKENDS, default=DEFAULT_BACKEND, 126 help='application lowering backend (default: %(default)s)') 127 p.add_argument('--keep-going', action='store_true', help='record independent failures without blocking other candidates') 128 p.add_argument('--seconds', type=int, default=1200, help='per-command time guard') 129 p.add_argument('--batch-seconds', type=int, default=1200, help='whole batch guard, maximum 7200') 130 p.add_argument('--memory-mib', type=int, default=32768, help='sampled direct-child RSS / host heap ceiling') 131 p.add_argument('--capabilities', type=int, default=1, help='GHC capabilities; select eight for a full support rebuild') 132 p.add_argument('--resume', action='store_true', help='reuse only unchanged staged candidates/checkpoints') 133 p.add_argument('--dependencies', action='append', type=Path, default=[], help='verified completed candidate directories supplying full dependency manifests') 134 args = p.parse_args() 135 if not 1 <= args.seconds <= 7200 or not 1 <= args.batch_seconds <= 7200 or not 0 < args.memory_mib <= 65536 or args.capabilities < 1: 136 p.error('jobs must be bounded within 64 GiB and two hours') 137 if len(set(args.reexport_imports)) != len(args.reexport_imports) or not set(args.reexport_imports) <= set(args.library): 138 p.error('reexport policy must name selected candidates exactly once') 139 root = args.directory.resolve() 140 if args.resume: 141 if not root.is_dir(): 142 p.error('resume directory is missing') 143 else: 144 root.mkdir(exist_ok=False, parents=True) 145 generation = args.generation.resolve() 146 assert json.loads((generation / 'status.json').read_text())['complete'] 147 events = json.loads((root / 'events.json').read_text()) if args.resume else [] 148 deadline = time.monotonic() + args.batch_seconds 149 150 def run(arguments): 151 remaining = min(args.seconds, deadline - time.monotonic()) 152 if remaining <= 0: 153 raise RuntimeError('candidate-batch time guard') 154 command = [str(args.tricu.resolve()), *map(str, arguments), '+RTS', '-N' + str(args.capabilities), 155 '-M' + str(args.memory_mib) + 'm', '-RTS'] 156 started, peak, reason = time.monotonic(), 0, 'completed' 157 number = len(events) 158 with (root / f'{number}.stdout').open('w') as out, (root / f'{number}.stderr').open('w') as err: 159 proc = subprocess.Popen(command, stdout=out, stderr=err, start_new_session=True) 160 try: 161 monitor = ChildRSS(proc) 162 while proc.poll() is None: 163 sample = monitor.sample() 164 if sample is not None: 165 peak = max(peak, sample) 166 if peak > args.memory_mib * 1024: 167 reason = 'RSS guard'; break 168 if time.monotonic() - started > remaining: 169 reason = 'time guard'; break 170 time.sleep(0.05) 171 except (OSError, RuntimeError) as error: 172 reason = 'monitoring failure: ' + str(error) 173 finally: 174 if proc.poll() is None: 175 os.killpg(proc.pid, signal.SIGTERM) 176 try: 177 proc.wait(timeout=5) 178 except subprocess.TimeoutExpired: 179 os.killpg(proc.pid, signal.SIGKILL); proc.wait() 180 stdout = (root / f'{number}.stdout').read_text() 181 stderr = (root / f'{number}.stderr').read_text() 182 events.append(dict(command=command, seconds=time.monotonic()-started, 183 peak_kib=peak, reason=reason, code=proc.returncode, 184 stdout=stdout, stderr=stderr)) 185 (root / 'events.json').write_text(json.dumps(events, indent=2) + '\n') 186 if reason != 'completed' or proc.returncode: 187 raise RuntimeError(f'{reason}; code={proc.returncode}; {stderr}') 188 return stdout 189 190 tools, store, workspace = root / 'tools', root / 'application', root / 'workspace' 191 workspace.mkdir(exist_ok=args.resume); store.mkdir(exist_ok=args.resume) 192 bundle = generation / ('build-stable.arboricx' if args.backend == 'stable' else 'build.arboricx') 193 output = run(['arboricx', 'import', '--file', bundle, '--store', tools, '--indexed']) 194 compiler = re.search(r'Created module alias .* -> ([0-9a-f]{64})', output)[1] 195 rows = [line.split('\t') for line in (generation / 'source-map.v1').read_text().splitlines()[1:]] 196 names = {legacy: mapped for _, mapped, legacy, _, _ in rows} 197 names.update({name: alias(name) for name in LIBRARIES}) 198 names.update({ 199 'STAGING_EXPERIMENT_MANIFEST': alias('source.staging-experiment'), 200 'STAGED_LIST_EXPERIMENT_MANIFEST': alias('source.staged-list-experiment'), 201 'STAGED_CONDITIONAL_COMPILER_MANIFEST': alias('source.staged-conditional-experiment'), 202 }) 203 selectors = {} 204 for entry in args.terminal_export: 205 name, symbol = entry.split(':', 1) 206 if name not in args.library or name in selectors: 207 raise ValueError('terminal export must name one selected candidate exactly once') 208 selectors[name] = symbol 209 records = [] 210 needed = set() 211 # Stage candidates before compiling so their declared edges are resolvable. 212 for name in args.library: 213 raw = (REPO / LIBRARIES[name]).read_bytes() 214 # Legacy patterns imports exposed prelude transitively. Native patterns 215 # owns only its declared exports; this demo uses equal? directly. 216 extra = [(names['prelude'], '!Local')] if name == 'demos.patternMatching' else [] 217 candidate, imports = native_candidate(terminal_export(raw, selectors[name]) if name in selectors else raw, names, extra) 218 if name in args.reexport_imports: 219 candidate = reexport_candidate(raw, candidate, imports) 220 staged = workspace / (alias(name) + '.tri') 221 if args.resume and staged.read_bytes() != candidate: 222 raise ValueError('resume candidate changed: ' + name) 223 staged.write_bytes(candidate) 224 from native_header import native_header 225 needed.update(native_header(candidate.decode())[0]) 226 records.append(dict(legacy=name, alias=alias(name), path=LIBRARIES[name], 227 before_sha256=hashlib.sha256(raw).hexdigest(), 228 after_sha256=hashlib.sha256(candidate).hexdigest(), 229 body_unchanged=name not in selectors, imports=imports, 230 terminal_export=selectors.get(name), definition_bodies_unchanged=True, 231 reexport_imports=name in args.reexport_imports, 232 explicit_extra_imports=extra, complete=False)) 233 if args.resume: 234 previous = json.loads((root / 'migration.json').read_text()) 235 if [(r['alias'], r['after_sha256']) for r in previous] != [(r['alias'], r['after_sha256']) for r in records]: 236 raise ValueError('resume membership or inputs changed') 237 records = previous 238 (root / 'migration.json').write_text(json.dumps(records, indent=2) + '\n') 239 cas = generation / 'cas' 240 for _, mapped, _, _, _ in rows: 241 if mapped not in needed or mapped in {record['alias'] for record in records}: 242 continue 243 pin = (cas / 'cache/source-build' / mapped).read_text().split('\t')[2] 244 payload = (cas / 'objects' / pin[:3] / pin).read_bytes() 245 assert digest('arboricx.module-manifest.v1', payload) == pin 246 for h in {pin, *[line.split('\t')[3] for line in payload.decode().splitlines()[1:]]}: 247 target = store / 'objects' / h[:3] / h 248 target.parent.mkdir(exist_ok=True, parents=True) 249 if not target.exists(): 250 os.link(cas / 'objects' / h[:3] / h, target) 251 (workspace / (mapped + '.tri')).write_text('!export * from ' + pin + '\n') 252 for dependency_directory in args.dependencies: 253 dependency_directory = dependency_directory.resolve() 254 dependency_store = dependency_directory / 'application' 255 for dependency in json.loads((dependency_directory / 'migration.json').read_text()): 256 mapped = dependency['alias'] 257 if mapped not in needed or mapped in [r['alias'] for r in records]: 258 continue 259 source = (REPO / dependency['path']).read_bytes() 260 if not dependency['complete'] or hashlib.sha256(source).hexdigest() != dependency['after_sha256']: 261 raise ValueError('stale/incomplete dependency candidate: ' + mapped) 262 pin = dependency['manifest'] 263 payload = (dependency_store / 'objects' / pin[:3] / pin).read_bytes() 264 if digest('arboricx.module-manifest.v1', payload) != pin: 265 raise ValueError('corrupt dependency manifest') 266 for h in {pin, *[line.split('\t')[3] for line in payload.decode().splitlines()[1:]]}: 267 target = store / 'objects' / h[:3] / h 268 target.parent.mkdir(exist_ok=True, parents=True) 269 if not target.exists(): 270 os.link(dependency_store / 'objects' / h[:3] / h, target) 271 (workspace / (mapped + '.tri')).write_text('!export * from ' + pin + '\n') 272 failures = [] 273 for record in records: 274 name = record['alias'] 275 checked_candidate_source(workspace, record) 276 if record['complete']: 277 continue 278 try: 279 run(['arboricx', 'compile', '--file', workspace / (name + '.tri'), 280 '--toolchain', compiler, '--tool-store', tools, '--store', store, 281 '--output', root / (name + '.arboricx'), '--compile-timeout', str(args.seconds)]) 282 except RuntimeError as error: 283 if not args.keep_going: 284 raise 285 failures.append(name) 286 print('Failed:', name, str(error), flush=True) 287 continue 288 checkpoint = (store / 'cache/source-build' / name).read_text().split('\t') 289 manifest = (store / 'objects' / checkpoint[2][:3] / checkpoint[2]).read_bytes() 290 receipt = (store / 'objects' / checkpoint[3][:3] / checkpoint[3]).read_bytes() 291 assert digest('arboricx.module-manifest.v1', manifest) == checkpoint[2] 292 assert digest('arboricx.build-receipt.v1', receipt) == checkpoint[3] 293 assert 'source\t' + digest('arboricx.source.v1', (workspace / (name + '.tri')).read_bytes()) in receipt.decode().splitlines() 294 checked_candidate_source(workspace, record) 295 record.update(complete=True, manifest=checkpoint[2], receipt=checkpoint[3]) 296 (root / 'migration.json').write_text(json.dumps(records, indent=2) + '\n') 297 print('Completed:', name, flush=True) 298 if failures: 299 raise SystemExit('Incomplete candidates: ' + ', '.join(failures)) 300 301 302 if __name__ == '__main__': 303 main()