source_layout.py (10355B)
1 #!/usr/bin/env python3 2 """Rebuild owner-relative layout and its consumers; check independent trees. 3 4 Opt-in, through Nix, using the authenticated packaged stable compiler. Unchanged 5 transitive fixture sources are verified before reuse. Never publishes fixtures. 6 """ 7 import argparse 8 import hashlib 9 import json 10 from pathlib import Path 11 import subprocess 12 import time 13 14 from bundle_dag import merge_bundles, read 15 from fixture_sources import snapshot_bytes, verify 16 from pattern_lowering import audit 17 from shipped_compiler import verify as verify_compiler 18 from source_order import Runner, field 19 from http_headers import LEAF, TRUE, expect 20 from local_control import decode_text 21 from source_layout_cases import semantic_cases, parser_cases 22 from tooling_batch_models import chars 23 24 ROOT = Path(__file__).resolve().parent.parent 25 26 27 def sha(data): 28 return hashlib.sha256(data).hexdigest() 29 30 31 def main(): 32 p = argparse.ArgumentParser(description=__doc__) 33 p.add_argument('tricu', type=Path) 34 p.add_argument('--directory', type=Path, required=True) 35 p.add_argument('--legacy-program', type=Path, help='optional retained pre-fix parseGateProgram bundle for historical rejection control') 36 p.add_argument('--resume', action='store_true', help='diagnostic reuse only, not generation acceptance') 37 args = p.parse_args() 38 root = args.directory.resolve() 39 root.mkdir(parents=True, exist_ok=args.resume) 40 workspace = root / 'workspace' 41 workspace.mkdir(exist_ok=args.resume) 42 (workspace / 'tricu.workspace').write_text('tricu.workspace.v1\n') 43 runner = Runner(args.tricu, root) 44 metadata = verify_compiler() 45 fixture_rows = {row[2]: row for row in verify(ROOT / 'test/fixtures')} 46 entries = {} 47 for line in (ROOT / 'bootstrap/compiler-source-map.v1').read_text().splitlines()[1:]: 48 _, alias, _, path, deps = line.split('\t') 49 entries[alias] = (path, [] if deps == '-' else deps.split(',')) 50 needed = [] 51 52 def visit(alias): 53 if alias not in needed: 54 for dep in entries[alias][1]: 55 visit(dep) 56 needed.append(alias) 57 visit('source_unit') 58 affected = {'source_parser', 'source_program'} 59 for alias in needed: 60 if any(dep in affected for dep in entries[alias][1]): 61 affected.add(alias) 62 active = affected | {dep for alias in affected for dep in entries[alias][1]} 63 report = dict(complete=False, resumed=args.resume, compiler_sha256=metadata['sha256'], 64 runtime_sha256=sha(runner.exe.read_bytes()), sources={}, reused={}, cases=[], 65 harness_sha256=sha(Path(__file__).read_bytes()), 66 cases_sha256=sha((ROOT / 'test/source_layout_cases.py').read_bytes())) 67 try: 68 for alias in needed: 69 data = (ROOT / entries[alias][0]).read_bytes() 70 report['sources'][alias] = sha(data) 71 if alias not in affected: 72 assert snapshot_bytes(ROOT / 'test/fixtures', fixture_rows[alias][4]) == data, ('stale reuse', alias) 73 tool = runner.imported(ROOT / metadata['bundle'], 'producer') 74 manifest = (runner.store / 'objects' / tool[:3] / tool).read_text() 75 producer = next(line.split('\t')[3] for line in manifest.splitlines()[1:] 76 if line.split('\t')[1] == 'sourceBuildGraph') 77 for alias in needed: 78 if alias not in active: 79 continue 80 path = workspace / (alias + '.tri') 81 if alias in affected: 82 path.write_bytes((ROOT / entries[alias][0]).read_bytes()) 83 else: 84 cohort, _, _, _, source_pin = fixture_rows[alias] 85 fixture = ROOT / 'test/fixtures' / (cohort + '.arboricx') 86 bundle = root / (alias + '-retained.arboricx') 87 merge_bundles(bundle, [(fixture, [(name, name[len(alias)+1:]) for name in read(fixture)[3] 88 if name.startswith(alias + '.')])], prune_unreachable=True) 89 pin = runner.imported(bundle, alias + '-retained') 90 path.write_text('!export * from ' + pin + '\n') 91 report['reused'][alias] = dict(manifest=pin, source_pin=source_pin) 92 for alias in ['source_unit', 'source_program', 'source_parser']: 93 runner.run('compile-' + alias, ['compile', '--file', workspace / (alias + '.tri'), 94 '--store', runner.store, '--toolchain', tool, '--tool-store', runner.store, 95 '--output', root / (alias + '.arboricx'), '--compile-timeout', '600'], seconds=630) 96 report['modules'] = audit(workspace, runner.store, producer, report['reused']) 97 pins = {alias: runner.imported(root / (alias + '.arboricx'), alias) 98 for alias in ['source_unit', 'source_program', 'source_parser']} 99 for alias, pin in pins.items(): 100 assert pin == report['modules'][alias]['manifest'] 101 env = root / 'environment.tree' 102 env.write_bytes(LEAF) 103 for label, source, value in semantic_cases(): 104 path = root / (label + '.source.tree') 105 path.write_bytes(chars(source)) 106 answer = runner.invoke(label, pins['source_unit'], 'compileStableModuleDefinitions', 107 ['--arg-tree-file', env, '--arg-tree-file', path]) 108 expect(field(answer, 0), TRUE, label + ' compilation') 109 bindings, found = field(answer, 1, 0), None 110 while bindings != LEAF: 111 binding = field(bindings, 0) 112 if decode_text(field(binding, 0)) == 'main': 113 found = field(binding, 1) 114 bindings = field(bindings, 1) 115 expect(found, value, label + ' independent value') 116 expect(field(answer, 1, 1), LEAF, label + ' remainder') 117 report['cases'].append(dict(label=label, sha256=sha(answer))) 118 for label, source, operation, expected in parser_cases(): 119 path = root / (label + '.source.tree') 120 path.write_bytes(chars(source)) 121 alias = 'source_program' if operation == 'parseGateProgram' else 'source_parser' 122 answer = runner.invoke(label, pins[alias], operation, ['--arg-tree-file', path]) 123 expect(answer, expected, label + ' complete located reply') 124 report['cases'].append(dict(label=label, sha256=sha(answer))) 125 # Parse every real compiler/library/demo consumer with both parsers. 126 # Replace only native header lines with equal-length line comments: 127 # body positions remain exact, and no imports need execution here. 128 reference = root / 'reference-program.arboricx' 129 cohort = fixture_rows['source_program'][0] 130 fixture = ROOT / 'test/fixtures' / (cohort + '.arboricx') 131 merge_bundles(reference, [(fixture, [('source_program.parseGateProgram', 'parseGateProgram')])], 132 prune_unreachable=True) 133 reference_pin = runner.imported(reference, 'reference-program') 134 report['consumers'] = [] 135 for source_path in sorted({p for directory in ['lib', 'demos', 'tricu-apps'] 136 for p in (ROOT / directory).rglob('*.tri')}): 137 text = source_path.read_text() 138 body = ''.join(('--' + ' ' * (len(line.rstrip('\n')) - 2) + ('\n' if line.endswith('\n') else '')) 139 if line.startswith('!') else line for line in text.splitlines(keepends=True)) 140 path = root / 'consumer.source.tree' 141 path.write_bytes(chars(body)) 142 label = 'consumer-' + str(source_path.relative_to(ROOT)).replace('/', '-') 143 before = runner.invoke(label + '-reference', reference_pin, 'parseGateProgram', ['--arg-tree-file', path]) 144 after = runner.invoke(label, pins['source_program'], 'parseGateProgram', ['--arg-tree-file', path]) 145 expect(field(before, 0), TRUE, label + ' reference parse') 146 expect(after, before, label + ' existing source AST and spans') 147 report['consumers'].append(dict(path=str(source_path.relative_to(ROOT)), source_sha256=sha(text.encode()), 148 reply_sha256=sha(after))) 149 label, source, operation, expected = next(row for row in parser_cases() if row[0] == 'ast-layout-2') 150 path = root / 'wrong-layout-control.source.tree' 151 path.write_bytes(chars(source)) 152 wrong = runner.invoke('wrong-layout-control', pins['source_parser'], 'parseGateSource', ['--arg-tree-file', path]) 153 assert wrong != expected, 'wrong parser passed the program-layout oracle' 154 report['wrong_parser_rejected'] = True 155 if args.legacy_program: 156 legacy = root / 'legacy-program.arboricx' 157 legacy.write_bytes(args.legacy_program.read_bytes()) 158 legacy_pin = runner.imported(legacy, 'legacy-program') 159 before = runner.invoke('old-layout-control', legacy_pin, operation, ['--arg-tree-file', path]) 160 assert before != expected, 'retired first-body-token layout passed the new oracle' 161 report['legacy_program_sha256'] = sha(legacy.read_bytes()) 162 report['old_layout_rejected'] = True 163 # Existing local matching/recursion and diagnostic controls run against 164 # these genuinely rebuilt consumers, not a frozen parser fixture. 165 control = root / ('local-controls-' + str(time.time_ns())) 166 subprocess.run(['python3', str(ROOT / 'test/local_control.py'), str(runner.exe), 167 '--unit', str(root / 'source_unit.arboricx'), 168 '--program', str(root / 'source_program.arboricx'), 169 '--directory', str(control)], check=True, timeout=300) 170 report['local_controls'] = str(control) 171 for alias in needed: 172 assert sha((ROOT / entries[alias][0]).read_bytes()) == report['sources'][alias], 'source changed during probe' 173 report['complete'] = True 174 except Exception as error: 175 report['failure'] = str(error) 176 raise 177 finally: 178 report['seconds'] = time.monotonic() - runner.started 179 (root / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 180 print(f'PASS: {len(report["cases"])} layout cases, local controls and source/receipt audits') 181 182 183 if __name__ == '__main__': 184 main()