prepare_generation_worker.py (10068B)
1 #!/usr/bin/env python3 2 """Prepare an explicit artifact-only transition worker for an explicitly selected generation backend. 3 The recovery frontend is never invoked. Compilation APIs are retained self-built 4 exports; current map/graph/transport policy is compiled by the selected portable 5 compiler. Generation backend selection is separate from worker compilation. 6 """ 7 import argparse 8 import hashlib 9 import json 10 from pathlib import Path 11 import re 12 import subprocess 13 import time 14 from process_rss import run_guarded 15 from generation_backend import BACKENDS, DEFAULT_BACKEND 16 17 ROOT = Path(__file__).resolve().parent.parent 18 parser = argparse.ArgumentParser(description=__doc__) 19 parser.add_argument('tricu', type=Path) 20 parser.add_argument('previous', type=Path) 21 parser.add_argument('output', type=Path) 22 parser.add_argument('--compiler-bundle', type=Path, required=True) 23 parser.add_argument('--backend', choices=BACKENDS, default=DEFAULT_BACKEND) 24 parser.add_argument('--plan-only', action='store_true', help='report selected support and flat size without writes or execution') 25 parser.add_argument('--support-flat-bytes', type=int, default=64 * 1024 * 1024, help='explicit flat transport budget for retained support') 26 parser.add_argument('--legacy-compiler-header', action='store_true', 27 help='explicit one-time bridge header for a retained pre-import compiler') 28 parser.add_argument('--memory-mib', type=int, default=32768) 29 parser.add_argument('--seconds', type=float, default=1800, help='whole preparation budget, maximum 7200') 30 args = parser.parse_args() 31 if not 0 < args.memory_mib <= 65536 or not 0 < args.seconds <= 7200 or args.support_flat_bytes <= 0: 32 parser.error('preparation limits must be within 64 GiB and two hours') 33 start = time.monotonic() 34 peak_kib = 0 35 tricu = str(args.tricu.resolve()) 36 old = args.previous.resolve() / 'cas' 37 root = args.output.resolve() 38 store, tools = root / 'store', root / 'tools' 39 40 def remaining(): 41 budget = args.seconds - (time.monotonic() - start) 42 if budget <= 0: 43 raise TimeoutError('worker preparation wall-time guard') 44 return budget 45 46 def run(command): 47 global peak_kib 48 try: 49 reply, peak = run_guarded([tricu, *map(str, command), '+RTS', '-N1', 50 '-M' + str(args.memory_mib) + 'm', '-RTS'], 51 memory_mib=args.memory_mib, seconds=remaining()) 52 except subprocess.CalledProcessError as error: 53 print(error.output, flush=True) 54 raise 55 peak_kib = max(peak_kib, peak) 56 return reply 57 58 def imported(bundle, destination, indexed=False): 59 reply = run(['arboricx', 'import', '--file', bundle, '--store', destination, *(['--indexed'] if indexed else ['--flat', '--max-flat-bytes', str(args.support_flat_bytes)])]) 60 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 61 62 parts = [] 63 for name in ['object', 'module', 'lock', 'build', 'map']: 64 text = (ROOT / 'lib/source' / (name + '.tri')).read_text() 65 # This explicit bridge flattens only unqualified dependencies into its 66 # selected support facade. Never discard an authored namespace silently. 67 if name == 'object': 68 # Preserve the qualified arithmetic binding under a distinct facade name. 69 if '!import native_math as N' not in text or set(re.findall(r'\bN\.([A-Za-z_?]+)', text)) != {'lowerHex'}: 70 raise ValueError('unexpected qualified object dependencies') 71 text = text.replace('!import native_math as N', '').replace('N.lowerHex', 'workerNativeLowerHex') 72 if any(line.startswith('!import ') and line.split()[-1:] != ['*'] 73 for line in text.splitlines()): 74 raise ValueError('transition flattening requires unqualified native uses: ' + name) 75 text = '\n'.join(line for line in text.splitlines() 76 if not line.startswith(('!import', '!source', '!use', '!export'))) 77 if name == 'build': 78 # Native packing is retained; this unused bridge bundle entrypoint is 79 # omitted, not the canonical units or either generation backend. 80 text = re.sub(r'sourceBuildGraphBundle workspace.*?(?=-- Same graph/IO)', '', text, flags=re.S) 81 parts.append(text) 82 body = '\n\n'.join(parts) 83 # Conservative local reachability for the selected entrypoint. Token mentions 84 # can retain extra definitions (including comments), never rewrite expressions. 85 # The portable frontend still parses/resolves every retained definition. This 86 # avoids transporting the unused other backend's nearly-GiB flat compiler root. 87 entrypoint = BACKENDS[args.backend][0] 88 matches = list(re.finditer(r'^([A-Za-z_][A-Za-z0-9_?\-]*)[^\n=]*=', body, re.M)) 89 blocks = {} 90 order = [] 91 for index, match in enumerate(matches): 92 name = match[1] 93 if name not in blocks: 94 order.append(name); blocks[name] = '' 95 blocks[name] += body[match.start():matches[index + 1].start() if index + 1 < len(matches) else len(body)] 96 needed, pending = set(), [entrypoint] 97 while pending: 98 name = pending.pop() 99 if name in needed: continue 100 if name not in blocks: raise ValueError('missing worker definition: ' + name) 101 needed.add(name) 102 pending.extend(set(re.findall(r'[A-Za-z_][A-Za-z0-9_?\-]*', blocks[name])) & blocks.keys() - needed) 103 body = '\n'.join(blocks[name] for name in order if name in needed) 104 local = set(re.findall(r'^([A-Za-z_][A-Za-z0-9_?\-]*)[^\n=]*=', body, re.M)) 105 # These constructors are defined below even when the previous IO module already 106 # exports them. Do not also import them and trigger canonical shadow rejection. 107 constructor_names = ('readTreeTerm', 'hashTreeTerm', 'writeTreeTermIfAbsent', 108 'readSourceFile', 'encodeSourceText', 'readIndexedTree', 109 'hashIndexedTree', 'writeIndexedTreeIfAbsent') 110 local.update(constructor_names) 111 words = set(re.findall(r'[A-Za-z_][A-Za-z0-9_?\-]*', body)) 112 selected = {} 113 for module in ['prelude', 'io', 'source_unit', 'source_resolve', 'source_lexer', 'parser', 'native_math']: 114 pin = (old / 'cache/source-build' / module).read_text().split('\t')[2] 115 assert re.fullmatch('[a-f0-9]{64}', pin), 'invalid retained manifest pin' 116 raw = (old / 'objects' / pin[:3] / pin).read_bytes() 117 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + raw).hexdigest() == pin 118 for row in raw.decode().splitlines()[1:]: 119 _, name, kind, tree, abi = row.split('\t') 120 assert kind == 'arboricx.tree-term.v1' and abi == 'arboricx.abi.tree.v1' 121 if module == 'native_math': 122 if name != 'lowerHex': continue 123 name = 'workerNativeLowerHex' 124 if name in words and name not in local: 125 assert re.fullmatch('[a-f0-9]{64}', tree), 'invalid retained tree pin' 126 selected[name] = tree 127 # New IO constructors are portable data, not Haskell-compiled glue. 128 constructor_lines = [line for line in (ROOT / 'lib/io.tri').read_text().splitlines() 129 if any(line.startswith(name + ' =') for name in constructor_names)] 130 if sorted(line.split()[0] for line in constructor_lines) != sorted(constructor_names): 131 raise ValueError('missing or duplicate required IO constructor: ' + ', '.join(constructor_names)) 132 constructors = '\n'.join(constructor_lines) 133 expanded = sum((old / 'objects' / pin[:3] / pin).stat().st_size for pin in selected.values()) 134 if args.plan_only: 135 print(json.dumps(dict(selected=selected, expanded_support_bytes=expanded, 136 support_flat_bytes=args.support_flat_bytes, backend=args.backend, 137 entrypoint=entrypoint, local_definitions=len(needed)), indent=2)) 138 raise SystemExit(0) 139 if expanded > args.support_flat_bytes: 140 raise ValueError('selected support exceeds explicit flat transport budget; inspect --plan-only') 141 root.mkdir(parents=True, exist_ok=False) 142 command = ['arboricx', 'export', '--store', old, '--output', root / 'support.arboricx'] 143 for name, tree in selected.items(): 144 command += ['--target', tree, '--name', name] 145 print(run(command), end='') 146 support = imported(root / 'support.arboricx', store) 147 manifest_bytes = (store / 'objects' / support[:3] / support).read_bytes() 148 assert hashlib.sha256(b'arboricx.module-manifest.v1\0' + manifest_bytes).hexdigest() == support 149 rows = [row.split('\t') for row in manifest_bytes.decode().splitlines()[1:]] 150 assert len(rows) == len(selected) and {row[1]: row[3] for row in rows} == selected 151 for row in rows: 152 assert (row[0], row[2], row[4]) == ('export', 'arboricx.tree-term.v1', 'arboricx.abi.tree.v1') 153 for tree in set(selected.values()): 154 hasher = hashlib.sha256(b'arboricx.tree-term.v1\0') 155 with (store / 'objects' / tree[:3] / tree).open('rb') as stream: 156 for chunk in iter(lambda: stream.read(1024 * 1024), b''): hasher.update(chunk) 157 assert hasher.hexdigest() == tree, 'transported support digest mismatch' 158 (root / 'support.tri').write_text(f'!export * from {support}\n') 159 header = ('!source support\n!use support *\n' if args.legacy_compiler_header else '!import support *\n') 160 source = header + '!export ' + entrypoint + '\n' + constructors + '\n' + body 161 (root / 'worker.tri').write_text(source) 162 compiler = imported(args.compiler_bundle.resolve(), tools, True) 163 print(run(['arboricx', 'compile', '--file', root / 'worker.tri', '--store', store, 164 '--toolchain', compiler, '--tool-store', tools, '--output', root / 'worker.arboricx', 165 '--compile-timeout', str(max(1, min(900, int(remaining()))))]), end='') 166 pin = imported(root / 'worker.arboricx', store, indexed=True) 167 (root / 'worker.pin').write_text(pin) 168 (root / 'provenance.json').write_text(json.dumps(dict( 169 support=support, compiler=compiler, worker=pin, selected=selected, 170 backend=args.backend, entrypoint=entrypoint, local_definitions=sorted(needed), 171 elapsed_seconds=time.monotonic()-start, peak_kib=peak_kib, 172 memory_mib=args.memory_mib, seconds=args.seconds, 173 source_sha256=hashlib.sha256(source.encode()).hexdigest(), 174 artifact_sha256=hashlib.sha256((root / 'worker.arboricx').read_bytes()).hexdigest()), indent=2)+'\n') 175 print('Worker:', root / 'worker.arboricx')