repair_support_fixture.py (19100B)
1 #!/usr/bin/env python3 2 """Publish authenticated support candidates, allowing changed bodies and exports. 3 4 Original snapshots stay immutable. Rebuilt modules get their actual source digest, 5 full manifest and original compilation receipt; no interface-compatibility promise. 6 Unrebuilt origin modules still require the fast gate's complete-provider identity reuse proof. 7 Multiple preparation directories may be combined with explicit --replace aliases. 8 """ 9 import argparse 10 import hashlib 11 import json 12 from pathlib import Path 13 import re 14 import shutil 15 import subprocess 16 import time 17 18 from bundle_dag import read, merge_bundles 19 from native_library_migration import digest 20 from process_rss import run_guarded 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def select_candidates(directories, replacements): 26 selected, duplicates = {}, set() 27 if len(set(replacements)) != len(replacements): 28 raise ValueError('duplicate --replace alias') 29 for directory in directories: 30 for record in json.loads((directory / 'migration.json').read_text()): 31 alias = record['alias'] 32 if not record['complete']: 33 raise ValueError('incomplete candidate: ' + alias) 34 if alias in selected: 35 duplicates.add(alias) 36 if alias not in replacements: 37 raise ValueError('duplicate candidate requires --replace: ' + alias) 38 selected[alias] = (directory, record) 39 if set(replacements) != duplicates: 40 raise ValueError('--replace must name exactly the replaced aliases') 41 if not selected: 42 raise ValueError('no candidates selected') 43 return selected 44 45 46 def relocate_patterns(original, candidates): 47 """Explicit compiler-to-support move; never widen ordinary rewrite policy.""" 48 if 'patterns' not in candidates: 49 raise ValueError('relocation requires rebuilt patterns') 50 record, payload = candidates['patterns'] 51 if (record['legacy'], record['path']) != ('patterns', 'lib/patterns.tri'): 52 raise ValueError('invalid patterns relocation identity') 53 rows = [row.split('\t') for row in original if row.startswith('module\t')] 54 if any(row[2] == 'patterns' or row[3] == 'lib/patterns.tri' for row in rows): 55 raise ValueError('patterns already belongs to support') 56 return [*original, '\t'.join(['module', 'patterns', 'patterns', 57 'lib/patterns.tri', digest('arboricx.source.v1', payload)])] 58 59 60 def add_binary64_support(membership, candidates, compiler_rows): 61 """Add the complete, rebuilt binary64 cohort with disjoint compiler ownership.""" 62 from native_library_migration import BINARY64_SUPPORT, alias 63 rows = [row.split('\t') for row in membership if row.startswith('module\t')] 64 occupied = {row[2] for row in rows + compiler_rows if row[:1] == ['module']} 65 paths = {row[3] for row in rows + compiler_rows if row[:1] == ['module']} 66 modules, declarations = [], [] 67 for legacy, path in BINARY64_SUPPORT.items(): 68 name = alias(legacy) 69 if name in occupied or path in paths: 70 raise ValueError('binary64 ownership collision: ' + name) 71 if name not in candidates: 72 raise ValueError('missing rebuilt binary64 provider: ' + name) 73 record, payload = candidates[name] 74 if (record['legacy'], record['path']) != (legacy, path): 75 raise ValueError('binary64 provider identity mismatch: ' + name) 76 modules.append('\t'.join(['module', legacy, name, path, digest('arboricx.source.v1', payload)])) 77 declarations.append('\t'.join(['added', legacy, name, path])) 78 return membership + modules, declarations 79 80 81 def updated_module_rows(original, candidates): 82 """Keep cohort membership/identity, not rebuilt source fingerprints.""" 83 rows = [row.split('\t') for row in original if row.startswith('module\t')] 84 aliases = {row[2] for row in rows} 85 if len(rows) != len(aliases) or not set(candidates) <= aliases: 86 raise ValueError('invalid support cohort membership') 87 result = [] 88 for tag, legacy, alias, path, source in rows: 89 if alias in candidates: 90 record, payload = candidates[alias] 91 if (record['legacy'], record['path']) != (legacy, path): 92 raise ValueError('candidate module identity changed: ' + alias) 93 source = digest('arboricx.source.v1', payload) 94 result.append('\t'.join([tag, legacy, alias, path, source])) 95 return result 96 97 98 def checked_receipt(payload, source, manifest, compiler): 99 rows = [row.split('\t') for row in payload.decode().splitlines()] 100 if rows[:1] != [['arboricx.build-receipt.v1']]: 101 raise ValueError('invalid candidate receipt header') 102 for key, value in [('source', digest('arboricx.source.v1', source)), 103 ('output', manifest), ('compiler', compiler), 104 ('options', digest('arboricx.build-options.v1', b'source-module.stable.v1'))]: 105 if [row for row in rows if row[0] == key] != [[key, value]]: 106 raise ValueError('candidate receipt mismatch: ' + key) 107 return rows 108 109 110 def check_support_dependencies(rows, providers): 111 """Conservative support-provider identity check, including unused imports. 112 113 Compiler providers are separately pinned by preparation; this checks support 114 providers against the cohort being published, not an older dependency directory. 115 The last local field is the dependency build receipt, not a source digest. 116 Its source may be a staged manifest facade rather than the authored library. 117 """ 118 seen = set() 119 for row in rows: 120 if not row or (row[0] == 'pin' and len(row) != 5): 121 raise ValueError('invalid dependency row') 122 if row[0] == 'local': 123 if len(row) != 4 or row[1] in seen: 124 raise ValueError('invalid/duplicate local dependency') 125 seen.add(row[1]) 126 if row[0] in ('local', 'pin') and row[1] in providers: 127 if row[2] != providers[row[1]]: 128 raise ValueError('stale support dependency: ' + row[1]) 129 130 131 def main(): 132 parser = argparse.ArgumentParser(description=__doc__) 133 parser.add_argument('tricu', type=Path) 134 parser.add_argument('directory', type=Path, nargs='+') 135 parser.add_argument('--replace', action='append', default=[], metavar='ALIAS') 136 parser.add_argument('--relocate-patterns', action='store_true', 137 help='explicitly move rebuilt patterns from compiler to support membership') 138 parser.add_argument('--add-io-support', action='store_true', help='explicit complete ten-module IO support expansion; genuine rebuilds required') 139 parser.add_argument('--compiler-io-cohort', action='store_true', help='explicit v2 compiler ownership of core/text/storage; rebuild general io in support') 140 parser.add_argument('--add-binary64-support', action='store_true', help='publish the closed thirteen-module binary64 cohort with rebuild receipts') 141 parser.add_argument('--compiler-fixture', type=Path, help='explicit freshly packed compiler candidate; default is the published fixture') 142 parser.add_argument('--source-root', type=Path, default=ROOT, help='authored source snapshot; must exactly match each prepared source/digest') 143 parser.add_argument('--output-directory', type=Path, required=True, 144 help='new candidate directory; never overwrite published fixtures') 145 parser.add_argument('--verification-memory-mib', type=int, default=8192, 146 help='full native-manifest audit RSS guard; heap is half this') 147 parser.add_argument('--capabilities', type=int, default=1) 148 args = parser.parse_args() 149 if not 1024 <= args.verification_memory_mib <= 65536 or args.capabilities < 1: 150 parser.error('invalid audit resource limits') 151 source_root = args.source_root.resolve() 152 inputs = ROOT / 'test/fixtures' 153 output = args.output_directory.resolve() 154 output.mkdir(parents=True, exist_ok=False) 155 selected = select_candidates([p.resolve() for p in args.directory], args.replace) 156 from native_library_migration import check_source_dependencies 157 for directory in {directory for directory, _ in selected.values()}: 158 check_source_dependencies(directory, source_root) 159 deadline = time.monotonic() + 1200 160 verifications = [] 161 162 def verify_bundle(bundle, indexed=False): 163 remaining = deadline - time.monotonic() 164 if remaining <= 0: 165 raise RuntimeError('support publication time guard') 166 # Full native-manifest reproduction expands flat trees, unlike indexed 167 # packing. It retains its own explicit memory budget and child deadline. 168 started = time.monotonic() 169 try: 170 text, peak = run_guarded([ 171 str(args.tricu.resolve()), 'arboricx', 'import', '--file', str(bundle), 172 '--store', str(output / 'verification-store'), *(['--indexed'] if indexed else ['--flat']), 173 '+RTS', '-N1' if indexed else '-N' + str(args.capabilities), 174 '-M512m' if indexed else '-M' + str(args.verification_memory_mib // 2) + 'm', '-RTS'], 175 memory_mib=1024 if indexed else args.verification_memory_mib, seconds=min(300, remaining)) 176 except subprocess.CalledProcessError as error: 177 print(error.output, flush=True) 178 raise 179 verifications.append(dict(bundle=bundle.name, indexed=indexed, 180 seconds=time.monotonic()-started, peak_kib=peak, output=text)) 181 (output / 'verification.json').write_text(json.dumps(verifications, indent=2) + '\n') 182 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', text)[1] 183 184 for name in ('support-origin.v1', 'support-origin.arboricx', 'support-origin-compiler.arboricx'): 185 shutil.copyfile(inputs / name, output / name) 186 old = (output / 'support-origin.v1').read_text().splitlines() 187 original = output / 'support-origin.arboricx' 188 compiler = args.compiler_fixture.resolve() if args.compiler_fixture else inputs / 'compiler-library.arboricx' 189 read(compiler) # Validate candidate bundle framing before binding its exact bytes. 190 compiler_bytes = compiler.read_bytes() 191 if ('bundle\t' + digest('tricu.support-library-fixture.v1', original.read_bytes()) not in old or 192 'compiler-fixture\t' + digest('tricu.compiler-library-fixture.v1', 193 (output / 'support-origin-compiler.arboricx').read_bytes()) not in old): 194 raise ValueError('original support provenance mismatch') 195 provenance = json.loads(next(row.split('\t', 1)[1] for row in old if row.startswith('provenance\t'))) 196 providers = {record['alias']: record['manifest'] for record in provenance} 197 providers.update({alias: record['manifest'] for alias, (_, record) in selected.items()}) 198 candidates, identities, sources = {}, {}, {} 199 for alias, (directory, record) in selected.items(): 200 source = (source_root / record['path']).read_bytes() 201 if (source != (directory / 'workspace' / (alias + '.tri')).read_bytes() or 202 hashlib.sha256(source).hexdigest() != record['after_sha256']): 203 raise ValueError('authored source differs from compiled candidate: ' + alias) 204 candidates[alias] = (record, source) 205 sources[record['path']] = source 206 imported = json.loads((directory / 'events.json').read_text())[0]['stdout'] 207 identities[alias] = re.search(r'sourceBuildGraph -> ([0-9a-f]{64})', imported)[1] 208 membership = relocate_patterns(old, candidates) if args.relocate_patterns else old 209 additions = [] 210 compiler_owned = {'io': 'lib/io.tri', 'io_storage': 'lib/io/storage.tri', 211 'io_text': 'lib/io/text.tri'} if args.compiler_io_cohort else {} 212 if args.compiler_io_cohort: 213 if not args.add_io_support: 214 raise ValueError('compiler IO cohort requires explicit IO support expansion') 215 rows = [line.split('\t') for line in compiler.with_suffix('.v1').read_text().splitlines()] 216 if ['bundle', digest('tricu.compiler-library-fixture.v1', compiler_bytes)] not in rows: 217 raise ValueError('compiler metadata identity mismatch') 218 required = dict(compiler_owned, compiler_io='lib/source/io.tri') 219 for alias, path in required.items(): 220 owned = [row for row in rows if row[:1] == ['module'] and row[2] == alias] 221 if len(owned) != 1 or owned[0][3:] != [path, digest('arboricx.source.v1', (source_root/path).read_bytes())]: 222 raise ValueError('compiler source ownership mismatch: ' + alias) 223 if set(compiler_owned) & candidates.keys(): 224 raise ValueError('compiler-owned modules submitted as support candidates') 225 if args.add_io_support: 226 if not args.relocate_patterns: 227 raise ValueError('IO support expansion requires the existing patterns relocation') 228 from native_library_migration import IO_SUPPORT 229 expanded = {alias: path for alias, path in IO_SUPPORT.items() if alias not in compiler_owned} 230 for alias, path in expanded.items(): 231 if alias not in candidates or (candidates[alias][0]['legacy'], candidates[alias][0]['path']) != (alias, path): 232 raise ValueError('missing/mismatched rebuilt IO provider: ' + alias) 233 if any(row.startswith('module\t') and row.split('\t')[2] == alias for row in membership): 234 raise ValueError('IO addition already in original cohort') 235 membership.append('\t'.join(['module', alias, alias, path, digest('arboricx.source.v1', candidates[alias][1])])) 236 additions.append('\t'.join(['added', alias, alias, path])) 237 if args.add_binary64_support: 238 if not args.compiler_io_cohort: 239 raise ValueError('binary64 expansion requires canonical compiler IO ownership') 240 membership, binary64_additions = add_binary64_support(membership, candidates, rows) 241 additions += binary64_additions 242 modules = updated_module_rows(membership, candidates) 243 if args.relocate_patterns: 244 from fixture_sources import source_rows 245 if any(row[2] == 'patterns' for row in source_rows(inputs) if row[0] == 'compiler-library'): 246 raise ValueError('patterns still belongs to the published compiler fixture') 247 selections = [] 248 evidence = output / 'support-receipts' 249 evidence.mkdir() 250 for alias, (directory, record) in selected.items(): 251 def obj(domain, pin): 252 payload = (directory / 'application/objects' / pin[:3] / pin).read_bytes() 253 if digest(domain, payload) != pin: 254 raise ValueError('corrupt candidate object: ' + pin) 255 return payload 256 receipt = obj('arboricx.build-receipt.v1', record['receipt']) 257 rows = checked_receipt(receipt, candidates[alias][1], record['manifest'], identities[alias]) 258 check_support_dependencies(rows, providers) 259 manifest = obj('arboricx.module-manifest.v1', record['manifest']) 260 bundle = directory / (alias + '.arboricx') 261 names = read(bundle)[3] 262 if set(names) != {row.split('\t')[1] for row in manifest.decode().splitlines()[1:]}: 263 raise ValueError('candidate bundle/manifest exports differ: ' + alias) 264 # No comparison with the original export set: these kernels were really 265 # compiled. Still bind every published tree to the authenticated receipt. 266 kinds = {row.split('\t')[2] for row in manifest.decode().splitlines()[1:]} 267 if kinds not in ({'arboricx.tree-term.v1'}, {'arboricx.indexed-tree.v1'}): 268 raise ValueError('unsupported or mixed native manifest kinds: ' + alias) 269 if verify_bundle(bundle, indexed=kinds == {'arboricx.indexed-tree.v1'}) != record['manifest']: 270 raise ValueError('candidate kernels differ from receipt manifest: ' + alias) 271 selections.append((bundle, [(name, alias + '.' + name) for name in names])) 272 (evidence / (alias + '.receipt')).write_bytes(receipt) 273 (evidence / (alias + '.manifest')).write_bytes(manifest) 274 merge_bundles(output / 'support-rebuilt.arboricx', selections, prune_unreachable=True) 275 retained = [(name, name) for name in read(original)[3] if name.split('.', 1)[0] not in selected] 276 merge_bundles(output / 'support-library.arboricx', [ 277 (original, retained), (output / 'support-rebuilt.arboricx', 278 [(name, name) for name in read(output / 'support-rebuilt.arboricx')[3]])], prune_unreachable=True) 279 for name in ('support-rebuilt.arboricx', 'support-library.arboricx'): 280 verify_bundle(output / name, indexed=True) 281 # Do not finish a candidate if source/compiler inputs changed while packing. 282 if any((source_root / path).read_bytes() != payload for path, payload in sources.items()): 283 raise ValueError('source changed during publication') 284 for directory in {directory for directory, _ in selected.values()}: 285 check_source_dependencies(directory, source_root) 286 if compiler.read_bytes() != compiler_bytes: 287 raise ValueError('compiler fixture changed during publication') 288 metadata = ['tricu.support-library-fixture.v7' if args.add_binary64_support else 289 'tricu.support-library-fixture.v6' if args.compiler_io_cohort else 290 'tricu.support-library-fixture.v4' if args.add_io_support else 291 'tricu.support-library-fixture.v3' if args.relocate_patterns else 292 'tricu.support-library-fixture.v2', *modules] 293 if args.relocate_patterns: 294 metadata.append('relocated\tcompiler-library\tpatterns\tpatterns\tlib/patterns.tri') 295 metadata += additions 296 metadata += ['\t'.join(['compiler-provider', alias, path]) for alias, path in compiler_owned.items()] 297 for label, domain, payload in [ 298 ('compiler-fixture', 'tricu.compiler-library-fixture.v1', compiler_bytes), 299 ('origin-metadata', 'tricu.support-origin-metadata.v1', (output / 'support-origin.v1').read_bytes()), 300 ('origin-compiler', 'tricu.compiler-library-fixture.v1', (output / 'support-origin-compiler.arboricx').read_bytes()), 301 ('origin-support', 'tricu.support-library-fixture.v1', original.read_bytes()), 302 ('rebuilt-support', 'tricu.support-library-fixture.v1', (output / 'support-rebuilt.arboricx').read_bytes())]: 303 metadata.append(label + '\t' + digest(domain, payload)) 304 metadata += ['\t'.join(['rebuilt', alias, record['receipt'], record['manifest'], identities[alias]]) 305 for alias, (_, record) in selected.items()] 306 metadata += ['reuse\t' + alias for alias in sorted({row.split('\t')[2] for row in modules} - set(selected))] 307 metadata += ['bundle\t' + digest('tricu.support-library-fixture.v1', (output / 'support-library.arboricx').read_bytes())] 308 (output / 'support-library.v1').write_text('\n'.join(metadata) + '\n') 309 print(f'Packed {len(selected)} authenticated modules; all full manifests reproduced. ' 310 'Candidate only; fast-gate reuse proof and source freshness still required.') 311 312 313 if __name__ == '__main__': 314 main()