provenance.py (14149B)
1 #!/usr/bin/env python3 2 """Native receipt retention controls; optional real-library offline rebuilds. 3 4 Default tests use independently constructed CAS/receipt bytes, not compilation. 5 --accepted consumes retained library_inspector.py artifacts, captures four packs 6 from a disposable producer copy, deletes that copy, and rebuilds only from packs. 7 """ 8 import argparse 9 import hashlib 10 import json 11 import os 12 from pathlib import Path 13 import re 14 import shutil 15 import struct 16 import subprocess 17 import tempfile 18 import time 19 from stable_bundle import assembled, sections 20 21 ROOT = Path(__file__).resolve().parent.parent 22 RECEIPT = 'arboricx.build-receipt.v1' 23 SOURCE = 'arboricx.source.v1' 24 INPUTS = 'arboricx.build-inputs.v1' 25 OPTIONS = 'arboricx.build-options.v1' 26 MANIFEST = 'arboricx.module-manifest.v1' 27 INDEXED = 'arboricx.indexed-tree.v1' 28 ABI = 'arboricx.abi.tree.v1' 29 30 31 def digest(kind, data): 32 return hashlib.sha256(kind.encode() + b'\0' + data).hexdigest() 33 34 35 def put(store, kind, data): 36 pin = digest(kind, data) 37 path = store / 'objects' / pin[:3] / pin 38 path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(data) 39 return pin 40 41 42 def object_path(store, pin): 43 return store / 'objects' / pin[:3] / pin 44 45 46 def snapshot(root): 47 return {str(p.relative_to(root)): (p.read_bytes(), p.stat().st_mtime_ns) 48 for p in root.rglob('*') if p.is_file()} 49 50 51 def controls(exe): 52 with tempfile.TemporaryDirectory(prefix='provenance-controls-') as temp: 53 root = Path(temp); store = root / 'cas'; workspace = root / 'workspace'; workspace.mkdir() 54 def run(*args, good=True): 55 result = subprocess.run([str(exe), 'provenance', *map(str, args), '+RTS', '-N1', '-M512m', '-RTS'], 56 cwd=root, capture_output=True, text=True, timeout=30) 57 assert (result.returncode == 0) == good, (args, result.stdout, result.stderr) 58 return result.stdout + result.stderr 59 # A leaf is NOT a compiler. Successful verification with this fixture 60 # must not interpret its sourceBuildGraph export. 61 m, _ = sections(ROOT / 'test/fixtures/id.arboricx'); at = 12 62 for _ in range(9): 63 size = struct.unpack_from('>I', m, at)[0]; at += 4 + size 64 assert struct.unpack_from('>I', m, at)[0] == 0 65 def text(t): 66 b = t.encode(); return struct.pack('>I', len(b)) + b 67 manifest = (m[:at+5] + struct.pack('>II', 1, 0) + text('default') + struct.pack('>I', 1) 68 + text('root') + struct.pack('>I', 0) + text('term') + text(ABI) + struct.pack('>II', 0, 0)) 69 leaf = put(store, INDEXED, assembled(manifest, struct.pack('>QI', 1, 1) + b'\0')) 70 def module(name): 71 return put(store, MANIFEST, (MANIFEST + '\nexport\t' + name + '\t' + INDEXED + '\t' + leaf + '\t' + ABI + '\n').encode()) 72 compiler, output = module('sourceBuildGraph'), module('main') 73 source = b'!export main\nmain = t\n' 74 (workspace / 'main.tri').write_bytes(source) 75 options = digest(OPTIONS, b'source-module.stable.v1') 76 rows = 'source\t' + digest(SOURCE, source) + '\ncompiler\t' + leaf + '\noptions\t' + options + '\n' 77 receipt = put(store, RECEIPT, (RECEIPT + '\n' + rows + 'output\t' + output + '\n').encode()) 78 key = digest('arboricx.source-build-key.v1', ('arboricx.source-build-key.v1\n' + rows).encode()) 79 cache = store / 'cache/source-build/main'; cache.parent.mkdir(parents=True) 80 cache.write_text('\t'.join(['arboricx.source-build-cache.v1', key, output, receipt])) 81 def capture(target, good=True, selected_receipt=None): 82 return run('capture', '--file', workspace / 'main.tri', '--store', store, 83 '--tool-store', store, '--toolchain', compiler, '--output', target, 84 *(['--receipt', selected_receipt] if selected_receipt else []), good=good) 85 (workspace / 'main.tri').write_bytes(source + b'-- drift\n') 86 capture(root / 'stale', False); assert not (root / 'stale').exists() 87 (workspace / 'main.tri').write_bytes(source) 88 pack = root / 'pack' 89 pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', capture(pack))[1] 90 before = snapshot(pack) 91 assert 'Producer claims only; no build executed' in run('verify', '--pack', pack, '--pin', pin) 92 assert snapshot(pack) == before 93 capture(pack, False); assert snapshot(pack) == before 94 run('verify', '--pack', pack, '--pin', '0'*64, good=False) 95 run('rebuild', '--pack', pack, '--pin', pin, '--trust-compiler', '0'*64, '--output', root / 'unauthorized', good=False) 96 assert not (root / 'unauthorized').exists() 97 packed = pack / 'store' 98 for kind, obj in [(SOURCE, digest(SOURCE, source)), (OPTIONS, options), (RECEIPT, receipt), (INDEXED, leaf)]: 99 path = object_path(packed, obj); original = path.read_bytes() 100 path.unlink(); run('verify', '--pack', pack, '--pin', pin, good=False) 101 path.write_bytes(original + b'corrupt'); run('verify', '--pack', pack, '--pin', pin, good=False) 102 path.write_bytes(original) 103 # Recompute the descriptor pin to exercise semantic validation, not merely SHA checks. 104 descriptor = pack / 'build-inputs'; original = descriptor.read_bytes() 105 for changed in [original.replace(b'entry\tmain', b'entry\t../outside'), 106 original.replace(b'unit\tmain', b'unit\tother'), original + b'unknown\tfield\n']: 107 descriptor.write_bytes(changed) 108 run('verify', '--pack', pack, '--pin', digest(INPUTS, changed), good=False) 109 descriptor.write_bytes(original) 110 objects = packed / 'objects'; moved = packed / 'real-objects' 111 objects.rename(moved); objects.symlink_to(moved, target_is_directory=True) 112 run('verify', '--pack', pack, '--pin', pin, good=False) 113 objects.unlink(); moved.rename(objects) 114 # An immutable receipt can be selected even after the cache locator is gone. 115 cache.unlink() 116 explicit = capture(root / 'explicit-receipt', selected_receipt=receipt) 117 assert pin in explicit 118 # Distribute only the pack; the compiler/library producer is no longer present. 119 shutil.rmtree(store); shutil.rmtree(workspace) 120 assert 'Verified retained bytes' in run('verify', '--pack', pack, '--pin', pin) 121 print('PASS native receipt retention, no-execution verification, stale source, corruption, missing inputs, trust gate, path and symlink controls') 122 123 124 def acceptance(exe, accepted, directory): 125 directory.mkdir(parents=True, exist_ok=False) 126 metadata = json.loads((accepted / 'report.json').read_text()) 127 compiler = metadata['compiler_manifest'] 128 producer = directory / 'disposable-producer'; producer.mkdir() 129 for name in ['producer', 'tools', 'application', 'consumer']: 130 shutil.copytree(accepted / name, producer / name) 131 isolated = directory / 'isolated'; isolated.mkdir() 132 report = dict(complete=False, commands=[], packs={}, compiler=compiler) 133 started = time.monotonic() 134 def record(): (directory / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 135 def run(*args, good=True, seconds=120, namespace='provenance'): 136 remaining = 2400 - (time.monotonic() - started) 137 if remaining <= 0: raise TimeoutError('whole provenance acceptance deadline') 138 before = time.monotonic() 139 result = subprocess.run([str(exe), namespace, *map(str, args), '+RTS', '-N1', '-M8192m', '-RTS'], 140 cwd=isolated, capture_output=True, text=True, timeout=min(seconds, remaining), 141 env=dict(os.environ, tricu_datadir=str(directory / 'NO-PACKAGED-COMPILER'))) 142 report['commands'].append(dict(namespace=namespace, args=list(map(str,args)), code=result.returncode, stdout=result.stdout, 143 stderr=result.stderr, seconds=time.monotonic()-before)); record() 144 assert (result.returncode == 0) == good, report['commands'][-1] 145 return result.stdout 146 for entry in ['base_release', 'io_release', 'inspection_release', 'main']: 147 workspace = producer / ('consumer' if entry == 'main' else 'producer/workspace') 148 store = producer / ('application' if entry == 'main' else 'producer/store') 149 pack = directory / ('pack-' + entry) 150 args = ['--project', workspace / 'tricu.project'] if entry == 'main' else [] 151 reply = run('capture', '--file', workspace / (entry + '.tri'), '--store', store, 152 '--tool-store', producer / 'tools', '--toolchain', compiler, '--output', pack, *args) 153 pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', reply)[1] 154 report['packs'][entry] = pin 155 shutil.rmtree(producer) 156 assert not producer.exists() 157 for entry, pin in report['packs'].items(): 158 print('Rebuilding only from retained inputs:', entry, flush=True) 159 pack = directory / ('pack-' + entry) 160 saved = snapshot(pack) 161 run('verify', '--pack', pack, '--pin', pin) 162 result = run('rebuild', '--pack', pack, '--pin', pin, '--trust-compiler', compiler, 163 '--output', directory / ('rebuild-' + entry), '--compile-timeout', '900', seconds=930) 164 assert 'Reproduced native module and receipt exactly' in result 165 assert snapshot(pack) == saved 166 # Two genuine histories, one executable: an ignored source comment changes 167 # source/receipt/pack identities without changing the native module. 168 history = directory / 'history-producer' 169 shutil.copytree(directory / 'rebuild-main', history) 170 (history / 'rebuild.txt').unlink() 171 main = history / 'workspace/main.tri' 172 main.write_bytes(main.read_bytes() + b'\n-- Independently retained source history.\n') 173 run('compile', '--file', main, '--store', history / 'application', '--tool-store', history / 'tools', 174 '--toolchain', compiler, '--output', history / 'inspector.arboricx', '--compile-timeout', '900', 175 seconds=930, namespace='arboricx') 176 alternate = directory / 'alternate-history' 177 reply = run('capture', '--file', main, '--store', history / 'application', '--tool-store', history / 'tools', 178 '--toolchain', compiler, '--project', history / 'workspace/tricu.project', '--output', alternate) 179 alternate_pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', reply)[1] 180 def identities(pack): 181 receipt = (pack / 'build-inputs').read_text().splitlines()[2].split('\t')[1] 182 module = object_path(pack / 'store', receipt).read_text().splitlines()[4].split('\t')[1] 183 return receipt, module 184 old_receipt, old_module = identities(directory / 'pack-main') 185 new_receipt, new_module = identities(alternate) 186 assert old_module == new_module and old_receipt != new_receipt and alternate_pin != report['packs']['main'] 187 shutil.rmtree(history) 188 reply = run('rebuild', '--pack', alternate, '--pin', alternate_pin, '--trust-compiler', compiler, 189 '--output', directory / 'rebuild-alternate-history', '--compile-timeout', '900', seconds=930) 190 assert 'Reproduced native module and receipt exactly' in reply 191 report['distinct_histories'] = dict(module=old_module, old_receipt=old_receipt, 192 new_receipt=new_receipt, alternate_pack=alternate_pin) 193 # A valid, fully authenticated producer claim can still be false. Change 194 # the source and its native receipt to claim the old inspector output, with 195 # no dependencies. Include a matching malicious producer cache. Copying that 196 # cache into the rebuild store could falsely "reproduce" the claimed output. 197 forged = directory / 'false-claim' 198 shutil.copytree(directory / 'pack-main', forged) 199 store = forged / 'store' 200 rows = (forged / 'build-inputs').read_text().splitlines() 201 old_receipt = rows[2].split('\t')[1] 202 receipt_rows = object_path(store, old_receipt).read_text().splitlines() 203 source = b'!export main\nmain = 1\n' 204 src = put(store, SOURCE, source) 205 new_receipt_bytes = ('\n'.join([RECEIPT, 'source\t'+src, receipt_rows[2], receipt_rows[3], receipt_rows[4]]) + '\n').encode() 206 new_receipt = put(store, RECEIPT, new_receipt_bytes) 207 new_descriptor = ('\n'.join([INPUTS, 'entry\tmain', 'receipt\t'+new_receipt, rows[3], 'unit\tmain\t'+new_receipt] 208 + [row for row in rows if row.startswith('project\t')]) + '\n').encode() 209 (forged / 'build-inputs').write_bytes(new_descriptor) 210 pin = digest(INPUTS, new_descriptor) 211 key_body = '\n'.join(['arboricx.source-build-key.v1', 'source\t'+src, receipt_rows[2], receipt_rows[3]]) + '\n' 212 key = digest('arboricx.source-build-key.v1', key_body.encode()) 213 poison = store / 'cache/source-build/main'; poison.parent.mkdir(parents=True, exist_ok=True) 214 poison.write_text('\t'.join(['arboricx.source-build-cache.v1', key, receipt_rows[4].split('\t')[1], new_receipt])) 215 run('verify', '--pack', forged, '--pin', pin) 216 mismatch = run('rebuild', '--pack', forged, '--pin', pin, '--trust-compiler', compiler, 217 '--output', directory / 'mismatching-rebuild', '--compile-timeout', '900', seconds=930, good=False) 218 assert 'Rebuild mismatch' in mismatch and 'tree different' in mismatch 219 report['false_claim_detected'] = True 220 report['complete'] = True; report['seconds'] = time.monotonic()-started; record() 221 print('PASS four offline reproductions; exact native receipts and modules; distinct histories share one executable; authenticated false claim/cache poison rejected by fresh rebuild') 222 223 224 def main(): 225 p = argparse.ArgumentParser(description=__doc__) 226 p.add_argument('tricu', type=Path) 227 p.add_argument('--accepted', type=Path) 228 p.add_argument('--directory', type=Path) 229 args = p.parse_args() 230 if bool(args.accepted) != bool(args.directory): p.error('--accepted and --directory must be used together') 231 exe = args.tricu.resolve() 232 controls(exe) 233 if args.accepted: acceptance(exe, args.accepted.resolve(), args.directory.resolve()) 234 235 236 if __name__ == '__main__': 237 main()