repair_support_fixture.py (13835B)
1 #!/usr/bin/env python3 2 """Publish authenticated support candidates, allowing changed bodies and exports. 3 4 Original snapshots stay immutable. Rebuilt modules get their actual source digest, 5 full manifest and original compilation receipt; no interface-compatibility promise. 6 Unrebuilt origin modules still require the fast gate's complete-provider identity reuse proof. 7 Multiple preparation directories may be combined with explicit --replace aliases. 8 """ 9 import argparse 10 import hashlib 11 import json 12 from pathlib import Path 13 import re 14 import shutil 15 import subprocess 16 import time 17 18 from bundle_dag import read, merge_bundles 19 from native_library_migration import digest 20 from process_rss import run_guarded 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def select_candidates(directories, replacements): 26 selected, duplicates = {}, set() 27 if len(set(replacements)) != len(replacements): 28 raise ValueError('duplicate --replace alias') 29 for directory in directories: 30 for record in json.loads((directory / 'migration.json').read_text()): 31 alias = record['alias'] 32 if not record['complete']: 33 raise ValueError('incomplete candidate: ' + alias) 34 if alias in selected: 35 duplicates.add(alias) 36 if alias not in replacements: 37 raise ValueError('duplicate candidate requires --replace: ' + alias) 38 selected[alias] = (directory, record) 39 if set(replacements) != duplicates: 40 raise ValueError('--replace must name exactly the replaced aliases') 41 if not selected: 42 raise ValueError('no candidates selected') 43 return selected 44 45 46 def relocate_patterns(original, candidates): 47 """Explicit compiler-to-support move; never widen ordinary rewrite policy.""" 48 if 'patterns' not in candidates: 49 raise ValueError('relocation requires rebuilt patterns') 50 record, payload = candidates['patterns'] 51 if (record['legacy'], record['path']) != ('patterns', 'lib/patterns.tri'): 52 raise ValueError('invalid patterns relocation identity') 53 rows = [row.split('\t') for row in original if row.startswith('module\t')] 54 if any(row[2] == 'patterns' or row[3] == 'lib/patterns.tri' for row in rows): 55 raise ValueError('patterns already belongs to support') 56 return [*original, '\t'.join(['module', 'patterns', 'patterns', 57 'lib/patterns.tri', digest('arboricx.source.v1', payload)])] 58 59 60 def updated_module_rows(original, candidates): 61 """Keep cohort membership/identity, not rebuilt source fingerprints.""" 62 rows = [row.split('\t') for row in original if row.startswith('module\t')] 63 aliases = {row[2] for row in rows} 64 if len(rows) != len(aliases) or not set(candidates) <= aliases: 65 raise ValueError('invalid support cohort membership') 66 result = [] 67 for tag, legacy, alias, path, source in rows: 68 if alias in candidates: 69 record, payload = candidates[alias] 70 if (record['legacy'], record['path']) != (legacy, path): 71 raise ValueError('candidate module identity changed: ' + alias) 72 source = digest('arboricx.source.v1', payload) 73 result.append('\t'.join([tag, legacy, alias, path, source])) 74 return result 75 76 77 def checked_receipt(payload, source, manifest, compiler): 78 rows = [row.split('\t') for row in payload.decode().splitlines()] 79 if rows[:1] != [['arboricx.build-receipt.v1']]: 80 raise ValueError('invalid candidate receipt header') 81 for key, value in [('source', digest('arboricx.source.v1', source)), 82 ('output', manifest), ('compiler', compiler), 83 ('options', digest('arboricx.build-options.v1', b'source-module.stable.v1'))]: 84 if [row for row in rows if row[0] == key] != [[key, value]]: 85 raise ValueError('candidate receipt mismatch: ' + key) 86 return rows 87 88 89 def check_support_dependencies(rows, providers): 90 """Conservative support-provider identity check, including unused imports. 91 92 Compiler providers are separately pinned by preparation; this checks support 93 providers against the cohort being published, not an older dependency directory. 94 The last local field is the dependency build receipt, not a source digest. 95 Its source may be a staged manifest facade rather than the authored library. 96 """ 97 seen = set() 98 for row in rows: 99 if not row or (row[0] == 'pin' and len(row) != 5): 100 raise ValueError('invalid dependency row') 101 if row[0] == 'local': 102 if len(row) != 4 or row[1] in seen: 103 raise ValueError('invalid/duplicate local dependency') 104 seen.add(row[1]) 105 if row[0] in ('local', 'pin') and row[1] in providers: 106 if row[2] != providers[row[1]]: 107 raise ValueError('stale support dependency: ' + row[1]) 108 109 110 def main(): 111 parser = argparse.ArgumentParser(description=__doc__) 112 parser.add_argument('tricu', type=Path) 113 parser.add_argument('directory', type=Path, nargs='+') 114 parser.add_argument('--replace', action='append', default=[], metavar='ALIAS') 115 parser.add_argument('--relocate-patterns', action='store_true', 116 help='explicitly move rebuilt patterns from compiler to support membership') 117 parser.add_argument('--output-directory', type=Path, required=True, 118 help='new candidate directory; never overwrite published fixtures') 119 parser.add_argument('--verification-memory-mib', type=int, default=8192, 120 help='full native-manifest audit RSS guard; heap is half this') 121 parser.add_argument('--capabilities', type=int, default=1) 122 args = parser.parse_args() 123 if not 1024 <= args.verification_memory_mib <= 65536 or args.capabilities < 1: 124 parser.error('invalid audit resource limits') 125 inputs = ROOT / 'test/fixtures' 126 output = args.output_directory.resolve() 127 output.mkdir(parents=True, exist_ok=False) 128 selected = select_candidates([p.resolve() for p in args.directory], args.replace) 129 deadline = time.monotonic() + 1200 130 verifications = [] 131 132 def verify_bundle(bundle, indexed=False): 133 remaining = deadline - time.monotonic() 134 if remaining <= 0: 135 raise RuntimeError('support publication time guard') 136 # Full native-manifest reproduction expands flat trees, unlike indexed 137 # packing. It retains its own explicit memory budget and child deadline. 138 started = time.monotonic() 139 try: 140 text, peak = run_guarded([ 141 str(args.tricu.resolve()), 'arboricx', 'import', '--file', str(bundle), 142 '--store', str(output / 'verification-store'), *(['--indexed'] if indexed else ['--flat']), 143 '+RTS', '-N1' if indexed else '-N' + str(args.capabilities), 144 '-M512m' if indexed else '-M' + str(args.verification_memory_mib // 2) + 'm', '-RTS'], 145 memory_mib=1024 if indexed else args.verification_memory_mib, seconds=min(300, remaining)) 146 except subprocess.CalledProcessError as error: 147 print(error.output, flush=True) 148 raise 149 verifications.append(dict(bundle=bundle.name, indexed=indexed, 150 seconds=time.monotonic()-started, peak_kib=peak, output=text)) 151 (output / 'verification.json').write_text(json.dumps(verifications, indent=2) + '\n') 152 return re.search(r'Created module alias .* -> ([0-9a-f]{64})', text)[1] 153 154 for name in ('support-origin.v1', 'support-origin.arboricx', 'support-origin-compiler.arboricx'): 155 shutil.copyfile(inputs / name, output / name) 156 old = (output / 'support-origin.v1').read_text().splitlines() 157 original = output / 'support-origin.arboricx' 158 compiler = inputs / 'compiler-library.arboricx' 159 compiler_bytes = compiler.read_bytes() 160 if ('bundle\t' + digest('tricu.support-library-fixture.v1', original.read_bytes()) not in old or 161 'compiler-fixture\t' + digest('tricu.compiler-library-fixture.v1', 162 (output / 'support-origin-compiler.arboricx').read_bytes()) not in old): 163 raise ValueError('original support provenance mismatch') 164 provenance = json.loads(next(row.split('\t', 1)[1] for row in old if row.startswith('provenance\t'))) 165 providers = {record['alias']: record['manifest'] for record in provenance} 166 providers.update({alias: record['manifest'] for alias, (_, record) in selected.items()}) 167 candidates, identities, sources = {}, {}, {} 168 for alias, (directory, record) in selected.items(): 169 source = (ROOT / record['path']).read_bytes() 170 if (source != (directory / 'workspace' / (alias + '.tri')).read_bytes() or 171 hashlib.sha256(source).hexdigest() != record['after_sha256']): 172 raise ValueError('authored source differs from compiled candidate: ' + alias) 173 candidates[alias] = (record, source) 174 sources[record['path']] = source 175 imported = json.loads((directory / 'events.json').read_text())[0]['stdout'] 176 identities[alias] = re.search(r'sourceBuildGraph -> ([0-9a-f]{64})', imported)[1] 177 membership = relocate_patterns(old, candidates) if args.relocate_patterns else old 178 modules = updated_module_rows(membership, candidates) 179 if args.relocate_patterns: 180 from fixture_sources import source_rows 181 if any(row[2] == 'patterns' for row in source_rows(inputs) if row[0] == 'compiler-library'): 182 raise ValueError('patterns still belongs to the published compiler fixture') 183 selections = [] 184 evidence = output / 'support-receipts' 185 evidence.mkdir() 186 for alias, (directory, record) in selected.items(): 187 def obj(domain, pin): 188 payload = (directory / 'application/objects' / pin[:3] / pin).read_bytes() 189 if digest(domain, payload) != pin: 190 raise ValueError('corrupt candidate object: ' + pin) 191 return payload 192 receipt = obj('arboricx.build-receipt.v1', record['receipt']) 193 rows = checked_receipt(receipt, candidates[alias][1], record['manifest'], identities[alias]) 194 check_support_dependencies(rows, providers) 195 manifest = obj('arboricx.module-manifest.v1', record['manifest']) 196 bundle = directory / (alias + '.arboricx') 197 names = read(bundle)[3] 198 if set(names) != {row.split('\t')[1] for row in manifest.decode().splitlines()[1:]}: 199 raise ValueError('candidate bundle/manifest exports differ: ' + alias) 200 # No comparison with the original export set: these kernels were really 201 # compiled. Still bind every published tree to the authenticated receipt. 202 kinds = {row.split('\t')[2] for row in manifest.decode().splitlines()[1:]} 203 if kinds not in ({'arboricx.tree-term.v1'}, {'arboricx.indexed-tree.v1'}): 204 raise ValueError('unsupported or mixed native manifest kinds: ' + alias) 205 if verify_bundle(bundle, indexed=kinds == {'arboricx.indexed-tree.v1'}) != record['manifest']: 206 raise ValueError('candidate kernels differ from receipt manifest: ' + alias) 207 selections.append((bundle, [(name, alias + '.' + name) for name in names])) 208 (evidence / (alias + '.receipt')).write_bytes(receipt) 209 (evidence / (alias + '.manifest')).write_bytes(manifest) 210 merge_bundles(output / 'support-rebuilt.arboricx', selections, prune_unreachable=True) 211 retained = [(name, name) for name in read(original)[3] if name.split('.', 1)[0] not in selected] 212 merge_bundles(output / 'support-library.arboricx', [ 213 (original, retained), (output / 'support-rebuilt.arboricx', 214 [(name, name) for name in read(output / 'support-rebuilt.arboricx')[3]])], prune_unreachable=True) 215 for name in ('support-rebuilt.arboricx', 'support-library.arboricx'): 216 verify_bundle(output / name, indexed=True) 217 # Do not finish a candidate if source/compiler inputs changed while packing. 218 if any((ROOT / path).read_bytes() != payload for path, payload in sources.items()): 219 raise ValueError('source changed during publication') 220 if compiler.read_bytes() != compiler_bytes: 221 raise ValueError('compiler fixture changed during publication') 222 metadata = ['tricu.support-library-fixture.v3' if args.relocate_patterns else 223 'tricu.support-library-fixture.v2', *modules] 224 if args.relocate_patterns: 225 metadata.append('relocated\tcompiler-library\tpatterns\tpatterns\tlib/patterns.tri') 226 for label, domain, payload in [ 227 ('compiler-fixture', 'tricu.compiler-library-fixture.v1', compiler_bytes), 228 ('origin-metadata', 'tricu.support-origin-metadata.v1', (output / 'support-origin.v1').read_bytes()), 229 ('origin-compiler', 'tricu.compiler-library-fixture.v1', (output / 'support-origin-compiler.arboricx').read_bytes()), 230 ('origin-support', 'tricu.support-library-fixture.v1', original.read_bytes()), 231 ('rebuilt-support', 'tricu.support-library-fixture.v1', (output / 'support-rebuilt.arboricx').read_bytes())]: 232 metadata.append(label + '\t' + digest(domain, payload)) 233 metadata += ['\t'.join(['rebuilt', alias, record['receipt'], record['manifest'], identities[alias]]) 234 for alias, (_, record) in selected.items()] 235 metadata += ['reuse\t' + alias for alias in sorted({row.split('\t')[2] for row in modules} - set(selected))] 236 metadata += ['bundle\t' + digest('tricu.support-library-fixture.v1', (output / 'support-library.arboricx').read_bytes())] 237 (output / 'support-library.v1').write_text('\n'.join(metadata) + '\n') 238 print(f'Packed {len(selected)} authenticated modules; all full manifests reproduced. ' 239 'Candidate only; fast-gate reuse proof and source freshness still required.') 240 241 242 if __name__ == '__main__': 243 main()