stable_bundle.py (15538B)
1 #!/usr/bin/env python3 2 """Stable bundle assembly: publication, independent byte goldens, real bundles, 3 and preloaded execution intervals. Haskell only; opt-in, no default-gate work. 4 5 python3 test/stable_bundle.py result/bin/tricu --compiler-bundle STABLE.arboricx \ 6 --directory /tmp/stable-bundle --input-bundle APPLICATION.arboricx 7 8 Sampling uses fresh reads each iteration, excludes input IO/forcing, compilation, 9 loading and equality; includes full output traversal, checkpoint overhead and GC. 10 """ 11 import argparse 12 import hashlib 13 import json 14 from pathlib import Path 15 import re 16 import statistics 17 import struct 18 import subprocess 19 import tempfile 20 import time 21 22 ROOT = Path(__file__).resolve().parent.parent 23 24 25 def assembled(manifest, nodes): 26 def section(identifier, offset, size): 27 return struct.pack('>IHHIQQI', identifier, 1, 1, 0, offset, size, 0) 28 return (b'ARBORICX' + struct.pack('>HHIQQ', 1, 0, 2, 0, 32) 29 + section(1, 96, len(manifest)) + section(2, 96 + len(manifest), len(nodes)) 30 + manifest + nodes) 31 32 33 def byte_tree(payload): 34 # Independent canonical tree-term encoding: byte = LSB-first bit list; 35 # byte string = Fork cells ending in Leaf. No compiler/evaluator involved. 36 numbers = [] 37 for value in range(256): 38 encoded = bytearray() 39 while value: 40 encoded.extend(b'\x02\x01\x00' if value & 1 else b'\x02\x00') 41 value >>= 1 42 numbers.append(bytes(encoded) + b'\x00') 43 return b''.join(b'\x02' + numbers[value] for value in payload) + b'\x00' 44 45 46 def sections(path): 47 data = path.read_bytes() 48 assert data[:8] == b'ARBORICX' 49 assert struct.unpack('>HHIQQ', data[8:32]) == (1, 0, 2, 0, 32), path 50 parts = {} 51 for at in (32, 64): 52 identifier, major, minor, flags, offset, size, reserved = struct.unpack('>IHHIQQI', data[at:at+32]) 53 assert (major, minor, flags, reserved) == (1, 1, 0, 0) 54 parts[identifier] = data[offset:offset+size] 55 assert len(parts[identifier]) == size 56 assert assembled(parts[1], parts[2]) == data, path 57 return parts[1], parts[2] 58 59 60 def main(): 61 parser = argparse.ArgumentParser(description=__doc__) 62 parser.add_argument('tricu', type=Path) 63 parser.add_argument('--compiler-bundle', required=True, type=Path) 64 parser.add_argument('--directory', type=Path) 65 parser.add_argument('--input-bundle', type=Path, action='append', default=[]) 66 parser.add_argument('--lengths', default='4096,16384') 67 parser.add_argument('--repeat', type=int, default=9) 68 parser.add_argument('--cold-repeat', type=int, default=5, 69 help='Independent processes directly invoking the functions, without the IO adapter') 70 parser.add_argument('--reverse-order', action='store_true') 71 args = parser.parse_args() 72 parser.error("Retired implicit recovery preparation: this historical workflow has no verified native-support input route. Use current portable candidate tooling; do not reuse its old support/cache artifacts.") 73 assert 2 <= args.repeat <= 20 74 assert 2 <= args.cold_repeat <= 10 75 lengths = [int(n) for n in args.lengths.split(',')] 76 assert all(0 <= n <= 65536 for n in lengths) 77 tricu = str(args.tricu.resolve()) 78 root = args.directory.resolve() if args.directory else Path(tempfile.mkdtemp(prefix='stable-bundle-')) 79 root.mkdir(parents=True, exist_ok=True) 80 store, tools = root / 'store', root / 'tools' 81 preparation = [] 82 83 def run(command, timeout=180): 84 reply = subprocess.run([tricu, *map(str, command)], cwd=ROOT, capture_output=True, 85 text=True, timeout=timeout) 86 assert reply.returncode == 0, (command, reply.stdout, reply.stderr) 87 return reply 88 89 def imported(bundle, destination=store, indexed=False): 90 reply = run(['arboricx', 'import', '--file', bundle, '--store', destination, 91 *(['--indexed'] if indexed else ['--flat'])]).stdout 92 pin = re.search(r'Created module alias .* -> ([0-9a-f]{64})', reply)[1] 93 rows = (destination / 'objects' / pin[:3] / pin).read_text().splitlines()[1:] 94 return pin, {r.split('\t')[1]: r.split('\t')[3] for r in rows} 95 96 compiler, _ = imported(args.compiler_bundle.resolve(), tools, indexed=True) 97 # Bootstrap facade stays in the checkout for workspace module resolution. 98 # Cache validity includes all library source and the selected executable. 99 digest = hashlib.sha256(tricu.encode()) 100 for path in sorted((ROOT / 'lib').rglob('*.tri')): 101 digest.update(path.read_bytes()) 102 support_text = ('!import "prelude" B\n!import "io" I\n!import "source.bundle" W\n' + 103 ''.join(f'{n} = B.{n}\n' for n in ['append', 'add', 'length', 'pair', 'true', 'and?', 104 'triage', 'matchBool', 'equal?', 'ok', 'y', 'succ', 'lazyList', 'lazyResult']) + 105 ''.join(f'{n} = I.{n}\n' for n in ['io', 'bind', 'yield', 'pure', 'readFile', 'writeBytes', 'onOk_', 'abortIO']) + 106 ''.join(f'{n} = W.{n}\n' for n in ['sourceBundleHeader', 'sourceBundleSection']) + 107 'reference = W.sourceBundleAssemble\n' 108 'passthrough = B.append\n' 109 'forceTree = B.y (self value : B.triage B.true (child : self child) ' 110 '(left right : B.and? (self left) (self right)) value)\n') 111 digest.update(support_text.encode()) 112 stamp = root / 'support.sha256' 113 support_bundle = root / 'support.arboricx' 114 if not support_bundle.exists() or not stamp.exists() or stamp.read_text() != digest.hexdigest(): 115 with tempfile.TemporaryDirectory(prefix='.stable-bundle-support-', dir=ROOT / 'test') as temporary: 116 source = Path(temporary) / 'support.tri' 117 source.write_text(support_text) 118 names = [line.split(' = ')[0] for line in support_text.splitlines() if ' = ' in line] 119 started = time.monotonic() 120 run(['arboricx', 'compile', '--file', source, '--store', store, 121 *[a for name in names for a in ('--name', name)], '--output', support_bundle], timeout=600) 122 preparation.append(dict(name='support', seconds=time.monotonic()-started)) 123 stamp.write_text(digest.hexdigest()) 124 support, support_exports = imported(support_bundle) 125 (root / 'bundle_support.tri').write_text(f'!export * from {support}\n') 126 127 def portable(name, text): 128 source, bundle = root / (name + '.tri'), root / (name + '.arboricx') 129 identity = hashlib.sha256((compiler + support + text).encode()).hexdigest() 130 stamp = root / (name + '.sha256') 131 if not bundle.exists() or not stamp.exists() or stamp.read_text() != identity: 132 source.write_text(text) 133 started = time.monotonic() 134 run(['arboricx', 'compile', '--file', source, '--store', store, 135 '--toolchain', compiler, '--tool-store', tools, '--output', bundle, 136 '--compile-timeout', '240'], timeout=270) 137 preparation.append(dict(name=name, seconds=time.monotonic()-started, bytes=bundle.stat().st_size)) 138 stamp.write_text(identity) 139 return bundle, imported(bundle)[1] 140 141 candidate, candidate_exports = portable('assembly', (ROOT / 'lib/source/bundle-stable.tri').read_text()) 142 adapter, _ = portable('adapter', (ROOT / 'test/stableBundleRuntime.tri').read_text()) 143 adapter_pin, _ = imported(adapter, indexed=True) 144 variants = {'reference': support_exports['reference'], 'stable-direct': candidate_exports['assemble']} 145 # Serialize both a known-first-argument partial application and a known 146 # input constructor whose tail is still unknown. Only ordinary syntax. 147 partial, partial_exports = portable('specialized', 148 '!import assembly as A\n!import bundle_support *\n' 149 '!export known\n!export prefix\n' 150 'known = A.assemble "abc"\nprefix tail nodes = A.assemble (pair 97 tail) nodes\n') 151 152 def paths(label, manifest, nodes, expected=None): 153 files = [root / (label + suffix) for suffix in ('.manifest', '.nodes', '.expected')] 154 for path, content in zip(files, (manifest, nodes, assembled(manifest, nodes) if expected is None else expected)): 155 path.write_bytes(content) 156 return files 157 158 def invoke(function, files, repeat=1): 159 return run(['arboricx', 'run', '--store', store, '--module', adapter_pin, '--name', 'main', 160 '--arg-tree-pin', function, *[a for path in files for a in ('--arg', path)], 161 '--io', '--allow-read', root, '--repeat', repeat, '--profile', '--result', '--format', 'fsl', 162 '+RTS', '-T', '-N1', '-RTS'], timeout=300) 163 164 cases = [('empty', b'', b''), ('small', b'abc', b'de'), 165 ('carry', b'x'*258, b'y'*257), ('manifest-only', bytes(range(256)), b''), 166 ('nodes-only', b'', bytes(range(256)))] 167 for label, manifest, nodes in cases: 168 files = paths(label, manifest, nodes) 169 for mode, function in variants.items(): 170 assert invoke(function, files).stdout.strip() == 'ok: Stem Leaf', (label, mode) 171 print(f'{label}: independent byte golden passed for both variants', flush=True) 172 files = paths('prefix', b'bc', b'de', assembled(b'abc', b'de')) 173 assert invoke(partial_exports['prefix'], files).stdout.strip() == 'ok: Stem Leaf' 174 # Adapt known's remaining one-argument interface without re-running compilation. 175 known_adapter, known_exports = portable('known_adapter', 176 '!import specialized as S\n!export applyKnown\napplyKnown unused nodes = S.known nodes\n') 177 assert invoke(known_exports['applyKnown'], files).stdout.strip() == 'ok: Stem Leaf' 178 print('partial application and known-head specialization passed', flush=True) 179 180 samples = [('synthetic-' + str(n), bytes((i % 256 for i in range(n//4))), 181 bytes(((i*17+3) % 256 for i in range(n-n//4))), None) for n in lengths] 182 for i, path in enumerate(args.input_bundle): 183 manifest, nodes = sections(path.resolve()) 184 samples.append((f'artifact-{i}-{path.stem}', manifest, nodes, path.resolve())) 185 report = [] 186 measured = {**variants, 'traversal-control': support_exports['passthrough']} 187 candidate_pin, _ = imported(candidate) 188 direct_exports = {'reference': (support, 'reference'), 'stable-direct': (candidate_pin, 'assemble'), 189 'traversal-control': (support, 'passthrough')} 190 cold_report = [] 191 for index, (label, manifest, nodes, original) in enumerate(samples): 192 files = paths(label, manifest, nodes) 193 order = list(measured) 194 if (index + args.reverse_order) % 2: 195 order.reverse() 196 for mode in order: 197 measured_files = (paths(label + '-control', manifest, nodes, manifest + nodes) 198 if mode == 'traversal-control' else files) 199 reply = invoke(measured[mode], measured_files, args.repeat) 200 assert reply.stdout.splitlines().count('ok: Stem Leaf') == args.repeat, reply.stdout 201 checkpoints = {p['profile']: p for line in reply.stderr.splitlines() if line.startswith('{') for p in [json.loads(line)]} 202 measurements = [] 203 for i in range(1, args.repeat+1): 204 start, end = checkpoints[f'yield-{i}-1'], checkpoints[f'yield-{i}-2'] 205 measurements.append(dict(ms=(end['wall_ns']-start['wall_ns'])/1e6, 206 allocated=end['allocated_bytes']-start['allocated_bytes'], 207 gc_ms=(end['gc_elapsed_ns']-start['gc_elapsed_ns'])/1e6)) 208 warm = measurements[1:] 209 assert all(s['allocated'] >= measurements[0]['allocated'] * 0.5 for s in warm), 'possible shared-result timing' 210 report.append(dict(case=label, mode=mode, manifest_bytes=len(manifest), nodes_bytes=len(nodes), samples=measurements)) 211 print(f'{label}/{mode}: {len(manifest)}+{len(nodes)} bytes; warm median ' 212 f'{statistics.median(s["ms"] for s in warm):.3f} ms, ' 213 f'{statistics.median(s["allocated"] for s in warm)/1e6:.3f} MB allocated', flush=True) 214 # Each direct invocation gets a fresh process and independently decoded 215 # concrete arguments. Host result forcing avoids the TC forceTree cost. 216 # Rendering/equality follow invoke-end and are outside the measured region. 217 arguments = [root / (label + suffix) for suffix in ('.manifest.tree', '.nodes.tree')] 218 for path, payload in zip(arguments, (manifest, nodes)): 219 path.write_bytes(byte_tree(payload)) 220 golden_trees = {False: byte_tree(assembled(manifest, nodes)).translate(bytes.maketrans(b'\x00\x01\x02', b'012')).decode(), 221 True: byte_tree(manifest + nodes).translate(bytes.maketrans(b'\x00\x01\x02', b'012')).decode()} 222 direct_samples = {mode: [] for mode in measured} 223 for iteration in range(args.cold_repeat): 224 for mode in (order if iteration % 2 == 0 else list(reversed(order))): 225 module, export = direct_exports[mode] 226 reply = run(['arboricx', 'run', '--store', store, '--module', module, '--name', export, 227 *[a for path in arguments for a in ('--arg-tree-file', path)], 228 '--format', 'ternary', '--profile', '+RTS', '-T', '-N1', '-RTS']) 229 assert reply.stdout.strip() == golden_trees[mode == 'traversal-control'], (label, mode, 'direct output') 230 profiles = {p['profile']: p for line in reply.stderr.splitlines() if line.startswith('{') for p in [json.loads(line)]} 231 start, end = profiles['invoke-begin-1'], profiles['invoke-end-1'] 232 direct_samples[mode].append(dict(ms=(end['wall_ns']-start['wall_ns'])/1e6, 233 allocated=end['allocated_bytes']-start['allocated_bytes'], 234 gc_ms=(end['gc_elapsed_ns']-start['gc_elapsed_ns'])/1e6)) 235 for mode, measurements in direct_samples.items(): 236 cold_report.append(dict(case=label, mode=mode, manifest_bytes=len(manifest), nodes_bytes=len(nodes), samples=measurements)) 237 print(f'{label}/{mode}: direct cold median {statistics.median(s["ms"] for s in measurements):.3f} ms, ' 238 f'{statistics.median(s["allocated"] for s in measurements)/1e6:.3f} MB allocated', flush=True) 239 (root / ('cold-results-reversed.json' if args.reverse_order else 'cold-results.json')).write_text(json.dumps(cold_report, indent=2)+'\n') 240 if original: 241 output = root / (label + '-rebuilt.arboricx') 242 run(['arboricx', 'run', '--store', store, '--module', adapter_pin, '--name', 'emit', 243 '--arg-tree-pin', variants['stable-direct'], '--arg', files[0], '--arg', files[1], '--arg', output, 244 '--io', '--allow-read', root, '--allow-write', root]) 245 assert output.read_bytes() == original.read_bytes() 246 imported(output, root / (label + '-verified')) 247 print(f'{label}: written artifact byte-identical; verified/imported into a separate store', flush=True) 248 (root / ('results-reversed.json' if args.reverse_order else 'results.json')).write_text(json.dumps(report, indent=2)+'\n') 249 wrong = paths('wrong', b'abc', b'de', b'wrong') 250 assert invoke(variants['stable-direct'], wrong).stdout.strip() == 'ok: Leaf' 251 print('wrong-output control passed', flush=True) 252 if preparation: 253 (root / 'preparation.json').write_text(json.dumps(preparation, indent=2)+'\n') 254 print(f'Artifacts and samples: {root}', flush=True) 255 256 257 if __name__ == '__main__': 258 main()