provenance.py (17318B)
1 #!/usr/bin/env python3 2 """Native receipt retention controls; optional real-library offline rebuilds. 3 4 Default tests use independently constructed CAS/receipt bytes, not compilation. 5 --accepted consumes retained library_inspector.py artifacts, captures four packs 6 from a disposable producer copy, deletes that copy, and rebuilds only from packs. 7 """ 8 import argparse 9 import hashlib 10 import json 11 import os 12 from pathlib import Path 13 import re 14 import shutil 15 import struct 16 import subprocess 17 import tempfile 18 import time 19 from stable_bundle import assembled, sections 20 21 ROOT = Path(__file__).resolve().parent.parent 22 RECEIPT = 'arboricx.build-receipt.v1' 23 SOURCE = 'arboricx.source.v1' 24 INPUTS = 'arboricx.build-inputs.v1' 25 OPTIONS = 'arboricx.build-options.v1' 26 MANIFEST = 'arboricx.module-manifest.v1' 27 INDEXED = 'arboricx.indexed-tree.v1' 28 ABI = 'arboricx.abi.tree.v1' 29 30 31 def digest(kind, data): 32 return hashlib.sha256(kind.encode() + b'\0' + data).hexdigest() 33 34 35 def put(store, kind, data): 36 pin = digest(kind, data) 37 path = store / 'objects' / pin[:3] / pin 38 path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(data) 39 return pin 40 41 42 def object_path(store, pin): 43 return store / 'objects' / pin[:3] / pin 44 45 46 def snapshot(root): 47 return {str(p.relative_to(root)): (p.read_bytes(), p.stat().st_mtime_ns) 48 for p in root.rglob('*') if p.is_file()} 49 50 51 def controls(exe): 52 with tempfile.TemporaryDirectory(prefix='provenance-controls-') as temp: 53 root = Path(temp); store = root / 'cas'; workspace = root / 'workspace'; workspace.mkdir() 54 def run(*args, good=True): 55 result = subprocess.run([str(exe), 'provenance', *map(str, args), '+RTS', '-N1', '-M512m', '-RTS'], 56 cwd=root, capture_output=True, text=True, timeout=30) 57 assert (result.returncode == 0) == good, (args, result.stdout, result.stderr) 58 return result.stdout + result.stderr 59 # A leaf is NOT a compiler. Successful verification with this fixture 60 # must not interpret its sourceBuildGraph export. 61 m, _ = sections(ROOT / 'test/fixtures/id.arboricx'); at = 12 62 for _ in range(9): 63 size = struct.unpack_from('>I', m, at)[0]; at += 4 + size 64 assert struct.unpack_from('>I', m, at)[0] == 0 65 def text(t): 66 b = t.encode(); return struct.pack('>I', len(b)) + b 67 manifest = (m[:at+5] + struct.pack('>II', 1, 0) + text('default') + struct.pack('>I', 1) 68 + text('root') + struct.pack('>I', 0) + text('term') + text(ABI) + struct.pack('>II', 0, 0)) 69 leaf = put(store, INDEXED, assembled(manifest, struct.pack('>QI', 1, 1) + b'\0')) 70 def module(name): 71 return put(store, MANIFEST, (MANIFEST + '\nexport\t' + name + '\t' + INDEXED + '\t' + leaf + '\t' + ABI + '\n').encode()) 72 compiler, output = module('sourceBuildGraph'), module('main') 73 source = b'!export main\nmain = t\n' 74 (workspace / 'main.tri').write_bytes(source) 75 options = digest(OPTIONS, b'source-module.stable.v1') 76 rows = 'source\t' + digest(SOURCE, source) + '\ncompiler\t' + leaf + '\noptions\t' + options + '\n' 77 receipt = put(store, RECEIPT, (RECEIPT + '\n' + rows + 'output\t' + output + '\n').encode()) 78 key = digest('arboricx.source-build-key.v1', ('arboricx.source-build-key.v1\n' + rows).encode()) 79 cache = store / 'cache/source-build/main'; cache.parent.mkdir(parents=True) 80 cache.write_text('\t'.join(['arboricx.source-build-cache.v1', key, output, receipt])) 81 def capture(target, good=True, selected_receipt=None, io_version=1): 82 return run('capture', '--file', workspace / 'main.tri', '--store', store, 83 '--tool-store', store, '--toolchain', compiler, '--toolchain-io-version', io_version, '--output', target, 84 *(['--receipt', selected_receipt] if selected_receipt else []), good=good) 85 (workspace / 'main.tri').write_bytes(source + b'-- drift\n') 86 capture(root / 'stale', False); assert not (root / 'stale').exists() 87 (workspace / 'main.tri').write_bytes(source) 88 pack = root / 'pack' 89 pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', capture(pack))[1] 90 before = snapshot(pack) 91 assert 'Producer claims only; no build executed' in run('verify', '--pack', pack, '--pin', pin) 92 assert snapshot(pack) == before 93 capture(pack, False); assert snapshot(pack) == before 94 run('verify', '--pack', pack, '--pin', '0'*64, good=False) 95 run('rebuild', '--pack', pack, '--pin', pin, '--trust-compiler', '0'*64, '--output', root / 'unauthorized', good=False) 96 assert not (root / 'unauthorized').exists() 97 # A separately domain-separated v2 descriptor remains inert even though 98 # this deliberately bogus compiler cannot execute either protocol. 99 v2 = root/'v2-pack' 100 pin2 = re.search(r'Retained build inputs: ([0-9a-f]{64})', capture(v2, io_version=2))[1] 101 descriptor2 = v2/'build-inputs'; original2 = descriptor2.read_bytes() 102 assert original2.startswith(b'arboricx.build-inputs.v2\n') 103 assert b'compiler-io\t2\n' in original2 104 assert pin2 == digest('arboricx.build-inputs.v2', original2) and pin2 != pin 105 before2 = snapshot(v2) 106 assert 'Compiler IO version 2' in run('verify','--pack',v2,'--pin',pin2) 107 assert snapshot(v2) == before2 108 run('verify','--pack',v2,'--pin',digest(INPUTS,original2),good=False) 109 for changed in [original2.replace(b'compiler-io\t2\n',b''), 110 original2.replace(b'compiler-io\t2',b'compiler-io\t1'), 111 original2.replace(b'compiler-io\t2',b'compiler-io\t02'), 112 original2.replace(b'compiler-io\t2',b'compiler-io\t3'), 113 original2.replace(b'compiler-io\t2',b'compiler-io\ttrue'), 114 original2 + b'compiler-io\t2\n']: 115 descriptor2.write_bytes(changed) 116 run('verify','--pack',v2,'--pin',digest('arboricx.build-inputs.v2',changed),good=False) 117 descriptor2.write_bytes(original2) 118 for target, selected in [('default-authority',[]),('wrong-authority',['--trust-io-version','1'])]: 119 run('rebuild','--pack',v2,'--pin',pin2,'--trust-compiler',compiler, 120 '--output',root/target,*selected,good=False) 121 assert not (root/target).exists() 122 run('rebuild','--pack',pack,'--pin',pin,'--trust-compiler',compiler, 123 '--trust-io-version','2','--output',root/'v1-wrong-authority',good=False) 124 assert not (root/'v1-wrong-authority').exists() 125 for flag, command in [('toolchain-io-version','capture'),('trust-io-version','rebuild')]: 126 for value in ['0','3','01','02','+2','true','-1']: 127 target=root/('invalid-'+command+'-'+value) 128 arguments = (['--file',workspace/'main.tri','--store',store,'--tool-store',store, 129 '--toolchain',compiler] if command=='capture' else 130 ['--pack',v2,'--pin',pin2,'--trust-compiler',compiler]) 131 reply=run(command,*arguments,'--output',target,'--'+flag,value,good=False) 132 assert 'IO version must be 1 or 2' in reply,reply 133 assert not target.exists() 134 packed = pack / 'store' 135 for kind, obj in [(SOURCE, digest(SOURCE, source)), (OPTIONS, options), (RECEIPT, receipt), (INDEXED, leaf)]: 136 path = object_path(packed, obj); original = path.read_bytes() 137 path.unlink(); run('verify', '--pack', pack, '--pin', pin, good=False) 138 path.write_bytes(original + b'corrupt'); run('verify', '--pack', pack, '--pin', pin, good=False) 139 path.write_bytes(original) 140 # Recompute the descriptor pin to exercise semantic validation, not merely SHA checks. 141 descriptor = pack / 'build-inputs'; original = descriptor.read_bytes() 142 for changed in [original.replace(b'entry\tmain', b'entry\t../outside'), 143 original.replace(b'unit\tmain', b'unit\tother'), original + b'unknown\tfield\n']: 144 descriptor.write_bytes(changed) 145 run('verify', '--pack', pack, '--pin', digest(INPUTS, changed), good=False) 146 descriptor.write_bytes(original) 147 objects = packed / 'objects'; moved = packed / 'real-objects' 148 objects.rename(moved); objects.symlink_to(moved, target_is_directory=True) 149 run('verify', '--pack', pack, '--pin', pin, good=False) 150 objects.unlink(); moved.rename(objects) 151 # An immutable receipt can be selected even after the cache locator is gone. 152 cache.unlink() 153 explicit = capture(root / 'explicit-receipt', selected_receipt=receipt) 154 assert pin in explicit 155 # Distribute only the pack; the compiler/library producer is no longer present. 156 shutil.rmtree(store); shutil.rmtree(workspace) 157 assert 'Verified retained bytes' in run('verify', '--pack', pack, '--pin', pin) 158 print('PASS native receipt retention, no-execution verification, stale source, corruption, missing inputs, trust gate, path and symlink controls') 159 160 161 def acceptance(exe, accepted, directory): 162 directory.mkdir(parents=True, exist_ok=False) 163 metadata = json.loads((accepted / 'report.json').read_text()) 164 compiler = metadata['compiler_manifest'] 165 compiler_io = metadata.get('compiler_io_version', 1) 166 assert type(compiler_io) is int and compiler_io in (1, 2) 167 producer = directory / 'disposable-producer'; producer.mkdir() 168 for name in ['producer', 'tools', 'application', 'consumer']: 169 shutil.copytree(accepted / name, producer / name) 170 isolated = directory / 'isolated'; isolated.mkdir() 171 report = dict(complete=False, commands=[], packs={}, compiler=compiler, compiler_io_version=compiler_io) 172 started = time.monotonic() 173 def record(): (directory / 'report.json').write_text(json.dumps(report, indent=2) + '\n') 174 def run(*args, good=True, seconds=120, namespace='provenance'): 175 if args[0] in ('compile', 'capture'): 176 args = (*args, '--toolchain-io-version', str(compiler_io)) 177 elif args[0] == 'rebuild': 178 args = (*args, '--trust-io-version', str(compiler_io)) 179 remaining = 2400 - (time.monotonic() - started) 180 if remaining <= 0: raise TimeoutError('whole provenance acceptance deadline') 181 before = time.monotonic() 182 result = subprocess.run([str(exe), namespace, *map(str, args), '+RTS', '-N1', '-M8192m', '-RTS'], 183 cwd=isolated, capture_output=True, text=True, timeout=min(seconds, remaining), 184 env=dict(os.environ, tricu_datadir=str(directory / 'NO-PACKAGED-COMPILER'))) 185 report['commands'].append(dict(namespace=namespace, args=list(map(str,args)), code=result.returncode, stdout=result.stdout, 186 stderr=result.stderr, seconds=time.monotonic()-before)); record() 187 assert (result.returncode == 0) == good, report['commands'][-1] 188 return result.stdout 189 for entry in ['base_release', 'io_release', 'inspection_release', 'main']: 190 workspace = producer / ('consumer' if entry == 'main' else 'producer/workspace') 191 store = producer / ('application' if entry == 'main' else 'producer/store') 192 pack = directory / ('pack-' + entry) 193 args = ['--project', workspace / 'tricu.project'] if entry == 'main' else [] 194 reply = run('capture', '--file', workspace / (entry + '.tri'), '--store', store, 195 '--tool-store', producer / 'tools', '--toolchain', compiler, '--output', pack, *args) 196 pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', reply)[1] 197 report['packs'][entry] = pin 198 shutil.rmtree(producer) 199 assert not producer.exists() 200 for entry, pin in report['packs'].items(): 201 print('Rebuilding only from retained inputs:', entry, flush=True) 202 pack = directory / ('pack-' + entry) 203 saved = snapshot(pack) 204 run('verify', '--pack', pack, '--pin', pin) 205 result = run('rebuild', '--pack', pack, '--pin', pin, '--trust-compiler', compiler, 206 '--output', directory / ('rebuild-' + entry), '--compile-timeout', '900', seconds=930) 207 assert 'Reproduced native module and receipt exactly' in result 208 assert snapshot(pack) == saved 209 # Two genuine histories, one executable: an ignored source comment changes 210 # source/receipt/pack identities without changing the native module. 211 history = directory / 'history-producer' 212 shutil.copytree(directory / 'rebuild-main', history) 213 (history / 'rebuild.txt').unlink() 214 main = history / 'workspace/main.tri' 215 main.write_bytes(main.read_bytes() + b'\n-- Independently retained source history.\n') 216 run('compile', '--file', main, '--store', history / 'application', '--tool-store', history / 'tools', 217 '--toolchain', compiler, '--output', history / 'inspector.arboricx', '--compile-timeout', '900', 218 seconds=930, namespace='arboricx') 219 alternate = directory / 'alternate-history' 220 reply = run('capture', '--file', main, '--store', history / 'application', '--tool-store', history / 'tools', 221 '--toolchain', compiler, '--project', history / 'workspace/tricu.project', '--output', alternate) 222 alternate_pin = re.search(r'Retained build inputs: ([0-9a-f]{64})', reply)[1] 223 def identities(pack): 224 receipt = (pack / 'build-inputs').read_text().splitlines()[2].split('\t')[1] 225 module = object_path(pack / 'store', receipt).read_text().splitlines()[4].split('\t')[1] 226 return receipt, module 227 old_receipt, old_module = identities(directory / 'pack-main') 228 new_receipt, new_module = identities(alternate) 229 assert old_module == new_module and old_receipt != new_receipt and alternate_pin != report['packs']['main'] 230 shutil.rmtree(history) 231 reply = run('rebuild', '--pack', alternate, '--pin', alternate_pin, '--trust-compiler', compiler, 232 '--output', directory / 'rebuild-alternate-history', '--compile-timeout', '900', seconds=930) 233 assert 'Reproduced native module and receipt exactly' in reply 234 report['distinct_histories'] = dict(module=old_module, old_receipt=old_receipt, 235 new_receipt=new_receipt, alternate_pack=alternate_pin) 236 # A valid, fully authenticated producer claim can still be false. Change 237 # the source and its native receipt to claim the old inspector output, with 238 # no dependencies. Include a matching malicious producer cache. Copying that 239 # cache into the rebuild store could falsely "reproduce" the claimed output. 240 forged = directory / 'false-claim' 241 shutil.copytree(directory / 'pack-main', forged) 242 store = forged / 'store' 243 rows = (forged / 'build-inputs').read_text().splitlines() 244 old_receipt = rows[2].split('\t')[1] 245 receipt_rows = object_path(store, old_receipt).read_text().splitlines() 246 source = b'!export main\nmain = 1\n' 247 src = put(store, SOURCE, source) 248 new_receipt_bytes = ('\n'.join([RECEIPT, 'source\t'+src, receipt_rows[2], receipt_rows[3], receipt_rows[4]]) + '\n').encode() 249 new_receipt = put(store, RECEIPT, new_receipt_bytes) 250 new_descriptor = ('\n'.join([rows[0], 'entry\tmain', 'receipt\t'+new_receipt, rows[3]] 251 + [row for row in rows if row.startswith('compiler-io\t')] 252 + ['unit\tmain\t'+new_receipt] 253 + [row for row in rows if row.startswith('project\t')]) + '\n').encode() 254 (forged / 'build-inputs').write_bytes(new_descriptor) 255 pin = digest(rows[0], new_descriptor) 256 key_body = '\n'.join(['arboricx.source-build-key.v1', 'source\t'+src, receipt_rows[2], receipt_rows[3]]) + '\n' 257 key = digest('arboricx.source-build-key.v1', key_body.encode()) 258 poison = store / 'cache/source-build/main'; poison.parent.mkdir(parents=True, exist_ok=True) 259 poison.write_text('\t'.join(['arboricx.source-build-cache.v1', key, receipt_rows[4].split('\t')[1], new_receipt])) 260 run('verify', '--pack', forged, '--pin', pin) 261 mismatch = run('rebuild', '--pack', forged, '--pin', pin, '--trust-compiler', compiler, 262 '--output', directory / 'mismatching-rebuild', '--compile-timeout', '900', seconds=930, good=False) 263 assert 'Rebuild mismatch' in mismatch and 'tree different' in mismatch 264 report['false_claim_detected'] = True 265 report['complete'] = True; report['seconds'] = time.monotonic()-started; record() 266 print('PASS four offline reproductions; exact native receipts and modules; distinct histories share one executable; authenticated false claim/cache poison rejected by fresh rebuild') 267 268 269 def main(): 270 p = argparse.ArgumentParser(description=__doc__) 271 p.add_argument('tricu', type=Path) 272 p.add_argument('--accepted', type=Path) 273 p.add_argument('--directory', type=Path) 274 args = p.parse_args() 275 if bool(args.accepted) != bool(args.directory): p.error('--accepted and --directory must be used together') 276 exe = args.tricu.resolve() 277 controls(exe) 278 if args.accepted: acceptance(exe, args.accepted.resolve(), args.directory.resolve()) 279 280 281 if __name__ == '__main__': 282 main()