portable_cli.py (21246B)
1 #!/usr/bin/env python3 2 """Opt-in native-module CLI checks; bootstrap preparation is explicit and separate.""" 3 import argparse 4 import hashlib 5 import json 6 import pathlib 7 import re 8 import subprocess 9 import tempfile 10 import time 11 from generation_backend import BACKENDS, DEFAULT_BACKEND 12 13 parser = argparse.ArgumentParser() 14 parser.add_argument("tricu") 15 parser.add_argument("--backend", choices=BACKENDS, default=DEFAULT_BACKEND, 16 help="lowering backend (default: %(default)s)") 17 parser.add_argument("--alternate-compiler-bundle", type=pathlib.Path, 18 help="Check switching backend/cache identity using the other backend's artifact") 19 parser.add_argument("--compiler-bundle", type=pathlib.Path, 20 help="Reuse an explicitly prepared sourceBuildGraph bundle") 21 transport = parser.add_mutually_exclusive_group() 22 transport.add_argument("--indexed", action="store_true", default=True, help="Use compact indexed storage (default)") 23 transport.add_argument("--flat", action="store_false", dest="indexed", help="Explicit flat transport probe") 24 parser.add_argument("--max-flat-bytes", type=int, default=64 * 1024 * 1024) 25 args = parser.parse_args() 26 indexed_flags = ["--indexed"] if args.indexed else ["--flat", "--max-flat-bytes", str(args.max_flat_bytes)] 27 tricu = str(pathlib.Path(args.tricu).resolve()) 28 repo = pathlib.Path(__file__).resolve().parent.parent 29 30 def run(command, *, cwd, ok=True, contains=None, seconds=180): 31 start = time.monotonic() 32 reply = subprocess.run([*command, '+RTS', '-N1', '-M8192m', '-RTS'], cwd=cwd, text=True, capture_output=True, timeout=seconds) 33 assert (reply.returncode == 0) == ok, (command, reply.returncode, reply.stdout, reply.stderr) 34 if contains is not None: 35 assert contains in reply.stdout + reply.stderr, (command, reply.stdout, reply.stderr) 36 return reply.stdout.strip(), time.monotonic() - start 37 38 with tempfile.TemporaryDirectory(prefix="tricu-portable-cli-") as temporary: 39 root = pathlib.Path(temporary) 40 tools, store, workspace = root / "tools", root / "application-store", root / "application" 41 workspace.mkdir() 42 bundle = args.compiler_bundle 43 if bundle is None: 44 if args.backend != "stable": 45 parser.error("--compiler-bundle is required for a non-stable backend") 46 bundle = repo / "bootstrap/build-stable.arboricx" 47 output, _ = run([tricu, "arboricx", "import", "--file", str(bundle.resolve()), 48 "--store", str(tools), *indexed_flags], cwd=root) 49 manifest = re.search(r"Created module alias .* -> ([0-9a-f]{64})", output).group(1) 50 compiler_tree = re.search(r"sourceBuildGraph -> ([0-9a-f]{64})", output).group(1) 51 compiler_payload = (tools / "objects" / compiler_tree[:3] / compiler_tree).read_bytes() 52 compiler_domain = "arboricx.indexed-tree.v1" if args.indexed else "arboricx.tree-term.v1" 53 assert hashlib.sha256(compiler_domain.encode() + b"\0" + compiler_payload).hexdigest() == compiler_tree 54 assert not args.indexed or len(compiler_payload) < 1024 * 1024 55 print(f"compiler CAS bytes={len(compiler_payload):,} kind={compiler_domain}", flush=True) 56 flags = ["--toolchain", manifest, "--tool-store", str(tools), "--store", str(store)] 57 patterns = workspace / "patterns.tri" 58 app = workspace / "app.tri" 59 patterns.write_text("!export first\n!export shape\n" 60 "first whole{t left right} = left\nfirst _ = 99\n" 61 "shape t = 0\nshape (t t) = 1\nshape _ = 2\n") 62 app.write_text("!import patterns as P\n!export main\n!export probe\n" 63 "main = P.first (t 23 24)\nprobe = P.shape (t (t t))\n") 64 assert not store.exists() 65 # Every application invocation is outside the checkout, without a workspace 66 # alias file, and uses a store containing no compiler or standard library. 67 text, cold = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 68 assert text == "23", text 69 text, warm = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 70 assert text == "23", text 71 print(f"portable eval cold={cold:.2f}s warm={warm:.2f}s", flush=True) 72 profile = subprocess.run([tricu, "eval", str(app), *flags, "--format", "number", 73 "--compile-profile", "+RTS", "-T", "-N1", "-M8192m", "-RTS"], 74 cwd=root, text=True, capture_output=True, timeout=180) 75 assert profile.returncode == 0 and profile.stdout.strip() == "23", profile.stderr 76 stages = [json.loads(row) for row in profile.stderr.splitlines()] 77 assert [stage["compiler_stage"] for stage in stages] == [ 78 "begin", "manifest-verified", "executable-read", "executable-digest-verified", 79 "executable-decoded", "build-returned", "publication-verified"] 80 assert all(a["wall_ns"] <= b["wall_ns"] for a, b in zip(stages, stages[1:])) 81 begin, decoded, end = (stages[i]["wall_ns"] for i in (0, 4, -1)) 82 print(f"profile tool-load={(decoded-begin)/1e9:.3f}s build={(end-decoded)/1e9:.3f}s", flush=True) 83 84 def get(domain, digest): 85 payload = (store / "objects" / digest[:3] / digest).read_bytes() 86 assert hashlib.sha256(domain.encode() + b"\0" + payload).hexdigest() == digest 87 return payload 88 89 def checkpoint(name): 90 fields = (store / "cache" / "source-build" / name).read_text().split("\t") 91 assert fields[0] == "arboricx.source-build-cache.v1" 92 receipt = get("arboricx.build-receipt.v1", fields[3]).decode().splitlines() 93 assert f"compiler\t{compiler_tree}" in receipt 94 assert f"output\t{fields[2]}" in receipt 95 return fields[2], fields[3], receipt 96 97 initial, initial_receipt, receipt = checkpoint("app") 98 expected_options = "source-module.stable.v1" if args.backend == "stable" else "source-module.reference.v1" 99 assert "options\t" + hashlib.sha256(b"arboricx.build-options.v1\0" + expected_options.encode()).hexdigest() in receipt 100 assert any(row.startswith("local\tpatterns\t") for row in receipt) 101 checkpoint("patterns") 102 assert not (store / "objects" / compiler_tree[:3] / compiler_tree).exists() 103 104 for selected, expected in [("main", "23"), ("probe", "2")]: 105 artifact = root / (selected + ".arboricx") 106 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 107 "--name", selected, "--output", str(artifact)], cwd=root) 108 imported_one, _ = run([tricu, "arboricx", "import", "--file", str(artifact), 109 "--store", str(root / ("runtime-" + selected))], cwd=root) 110 selected_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", imported_one).group(1) 111 text, _ = run([tricu, "arboricx", "run", "--module", selected_pin, "--name", selected, 112 "--store", str(root / ("runtime-" + selected)), "--format", "number"], cwd=root) 113 assert text == expected, (selected, text) 114 all_exports = root / "all.arboricx" 115 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 116 "--output", str(all_exports)], cwd=root) 117 imported, _ = run([tricu, "arboricx", "import", "--file", str(all_exports), 118 "--store", str(root / "runtime"), *indexed_flags], cwd=root) 119 assert "main ->" in imported and "probe ->" in imported 120 runtime_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", imported).group(1) 121 for selected, expected in [("main", "23"), ("probe", "2")]: 122 text, _ = run([tricu, "arboricx", "run", "--store", str(root / "runtime"), 123 "--module", runtime_pin, "--name", selected, "--format", "number"], cwd=root) 124 assert text == expected, (selected, text) 125 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 126 "--name", "missing", "--output", str(root / "missing.arboricx")], 127 cwd=root, ok=False, contains="Module export not found") 128 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 129 "--name", "main", "--name", "main", "--output", str(root / "duplicate.arboricx")], 130 cwd=root, ok=False, contains="Duplicate --name") 131 assert not (root / "missing.arboricx").exists() 132 assert not (root / "duplicate.arboricx").exists() 133 # Ordinary source-free execution shares the verified loader. 134 text, _ = run([tricu, "arboricx", "run", "--store", str(store), 135 "--module", initial, "--name", "main", "--format", "number"], cwd=root) 136 assert text == "23" 137 138 if args.alternate_compiler_bundle: 139 alternate, _ = run([tricu, "arboricx", "import", "--file", str(args.alternate_compiler_bundle.resolve()), 140 "--store", str(tools), *indexed_flags], cwd=root) 141 alternate_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", alternate).group(1) 142 alternate_tree = re.search(r"sourceBuildGraph -> ([0-9a-f]{64})", alternate).group(1) 143 alternate_flags = ["--toolchain", alternate_pin, "--tool-store", str(tools), "--store", str(store)] 144 text, _ = run([tricu, "eval", str(app), *alternate_flags, "--format", "number"], cwd=root) 145 assert text == "23" 146 switched = (store / "cache" / "source-build" / "app").read_text().split("\t") 147 switched_receipt = get("arboricx.build-receipt.v1", switched[3]).decode() 148 other_options = "source-module.reference.v1" if args.backend == "stable" else "source-module.stable.v1" 149 assert "compiler\t" + alternate_tree in switched_receipt 150 assert "options\t" + hashlib.sha256(b"arboricx.build-options.v1\0" + other_options.encode()).hexdigest() in switched_receipt 151 assert switched[3] != initial_receipt 152 run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 153 assert checkpoint("app")[:2] == (initial, initial_receipt) 154 print("backend switch: cache/receipt identities separated and restored", flush=True) 155 156 if args.backend == "stable": 157 recursive = workspace / "recursive.tri" 158 recursive.write_text('!export copy\ncopy [] = []\ncopy [head | tail] = t head (copy tail)\ncopy _ = []\n') 159 caller = workspace / "caller.tri" 160 caller.write_text('!import recursive as R\n!export main\nmain = R.copy "hello"\n') 161 text, _ = run([tricu, "eval", str(caller), *flags, "--format", "string"], cwd=root) 162 assert text == '"hello"', text 163 if args.alternate_compiler_bundle: 164 run([tricu, "eval", str(caller), *alternate_flags], cwd=root, ok=False, contains="acyclic") 165 recursive.write_text('!export copy\ncopy = copy\n') 166 run([tricu, "eval", str(caller), *flags], cwd=root, ok=False, 167 contains="recursive function requires parameters") 168 print("native recursive dependency, optimized rejection, recursive-value diagnostic pass", flush=True) 169 170 patterns.write_text(patterns.read_text().replace("= left", "= right")) 171 text, changed = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 172 assert text == "24", text 173 updated, updated_receipt, _ = checkpoint("app") 174 assert (initial, initial_receipt) != (updated, updated_receipt) 175 print(f"changed dependency rebuild={changed:.2f}s", flush=True) 176 # Header consolidation is acquisition + binding, not a store alias fallback. 177 defaults = workspace / 'defaults.tri' 178 defaults.write_text('!import patterns\n!import patterns as P\n!export main\nmain = patterns.first (t 22 23)\n') 179 text, _ = run([tricu, 'eval', str(defaults), *flags, '--format', 'number'], cwd=root) 180 assert text == '23', text 181 _, _, default_rows = checkpoint('defaults') 182 assert len([row for row in default_rows if row.startswith('local\tpatterns\t')]) == 1 183 barrel = workspace / 'barrel.tri' 184 barrel.write_text('!export * from patterns\n') 185 through = workspace / 'through.tri' 186 through.write_text('!import barrel *\n!export main\nmain = first (t 22 23)\n') 187 text, _ = run([tricu, 'eval', str(through), *flags, '--format', 'number'], cwd=root) 188 assert text == '23', text 189 pattern_manifest, _, _ = checkpoint('patterns') 190 assert checkpoint('barrel')[0] == pattern_manifest # exact executable trees 191 pinned = workspace / 'pinned.tri' 192 pinned.write_text('!import '+pattern_manifest+' as P\n!export main\nmain = P.first (t 22 23)\n') 193 text, _ = run([tricu, 'eval', str(pinned), *flags, '--format', 'number'], cwd=root) 194 assert text == '23', text 195 assert not any(row.startswith('local\t') for row in checkpoint('pinned')[2]) 196 build_only = workspace / 'build_only.tri' 197 build_only.write_text('!source patterns\n!export main\nmain = 7\n') 198 text, _ = run([tricu, 'eval', str(build_only), *flags, '--format', 'number'], cwd=root) 199 assert text == '7', text 200 201 # Build-scoped manifest/tree reuse must not turn an unused import into an 202 # unchecked pin. Repeated qualifiers share one authenticated provider, but 203 # every export remains subject to integrity checking, cold and warm. 204 unused = workspace / 'unused.tri' 205 unused.write_text('!import '+pattern_manifest+' as P\n!import '+pattern_manifest+ 206 ' as Q\n!export main\nmain = 7\n') 207 text, _ = run([tricu, 'eval', str(unused), *flags, '--format', 'number'], cwd=root) 208 assert text == '7', text 209 unused_checkpoint = (store / 'cache/source-build/unused').read_bytes() 210 provider_bytes = get('arboricx.module-manifest.v1', pattern_manifest) 211 shape_hash = next(row.split('\t')[3] for row in provider_bytes.decode().splitlines()[1:] 212 if row.split('\t')[1] == 'shape') 213 shape_path = store / 'objects' / shape_hash[:3] / shape_hash 214 shape_bytes = shape_path.read_bytes() 215 for cold in (False, True): 216 if cold: 217 unused.write_text(unused.read_text() + '-- force a source cache miss\n') 218 shape_path.write_bytes(shape_bytes + b'corrupt') 219 run([tricu, 'eval', str(unused), *flags], cwd=root, ok=False, contains='CAS digest mismatch') 220 assert (store / 'cache/source-build/unused').read_bytes() == unused_checkpoint 221 shape_path.write_bytes(shape_bytes) 222 provider_path = store / 'objects' / pattern_manifest[:3] / pattern_manifest 223 provider_path.write_bytes(provider_bytes + b'corrupt') 224 run([tricu, 'eval', str(unused), *flags], cwd=root, ok=False, contains='CAS digest mismatch') 225 provider_path.write_bytes(provider_bytes) 226 text, _ = run([tricu, 'eval', str(unused), *flags, '--format', 'number'], cwd=root) 227 assert text == '7', text 228 print('unused repeated imports: cold/warm tree and manifest corruption rejected', flush=True) 229 230 patterns.write_text("!export first\nfirst x x = missing\nfirst _ _ = t\n") 231 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="unique capture") 232 # A failure must not replace the last successful checkpoint. 233 assert checkpoint("app")[:2] == (updated, updated_receipt) 234 235 def bad_source(name, text, message): 236 path = workspace / (name + ".tri") 237 path.write_text(text) 238 run([tricu, "eval", str(path), *flags], cwd=root, ok=False, contains=message) 239 240 bad_source('old_native', '!use patterns *\n!export main\nmain = t\n', 'unknown native header') 241 bad_source('bare_hash', '!import '+pattern_manifest+'\n!export main\nmain = t\n', 'hash imports require') 242 for index, header in enumerate(['!source patterns\n!import patterns\n', 243 '!import patterns\n!source patterns\n', 244 '!source patterns\n!export * from patterns\n']): 245 bad_source('redundant'+str(index), header+'!export main\nmain = t\n', 'redundant build-only') 246 # Restore a valid dependency before collision probes. 247 patterns.write_text('!export first\nfirst x = x\n') 248 bad_source('collision', '!import patterns *\n!export first\nfirst = t\n', 'first') 249 bad_source('duplicate_import', '!import patterns *\n!import patterns *\n!export main\nmain = t\n', 'duplicate imported name') 250 bad_source("legacy", '!import "prelude" !Local\nmain = t\n', "Portable compilation failed") 251 # A moving alias in the application CAS must not rescue a missing local. 252 run([tricu, 'arboricx', 'import', '--file', str(all_exports), 253 '--store', str(store), '--module', 'absent'], cwd=root) 254 bad_source("undefined", "!import absent *\n!export main\nmain = t\n", "does not exist") 255 bad_source("cycle", "!import cycle *\n!export main\nmain = t\n", "dependency cycle") 256 bad_source("expression", "!export main\nmain = t\nmain\n", "Portable compilation failed") 257 bad_source("noMain", "!export value\nvalue = t\n", "Module export not found: main") 258 run([tricu, "eval", str(app), str(app), *flags], cwd=root, ok=False, 259 contains="one native module") 260 run([tricu, "eval", str(app), "--tool-store", str(tools)], cwd=root, ok=False, 261 contains="--tool-store requires --toolchain") 262 run([tricu, "eval", str(app), "--toolchain", "portable-compiler", "--tool-store", str(tools), 263 "--store", str(store)], cwd=root, ok=False, contains="Invalid module manifest hash") 264 run([tricu, "eval", str(app), "--toolchain", manifest, "--tool-store", str(root / "absent"), 265 "--store", str(store)], cwd=root, ok=False, contains="Module manifest not found") 266 267 # Compiler IO permissions are not inherited by the application action. 268 message = root / "message.txt" 269 message.write_text("hello native") 270 action = workspace / "action.tri" 271 action.write_text('!export main\nmain = (t "tricuIO" (t 1 (t 20 ' 272 + json.dumps(str(message)) + ')))\n') 273 denied, _ = run([tricu, "eval", str(action), *flags, "--io", "--format", "decode"], cwd=root) 274 assert "permission denied" in denied and "hello native" not in denied 275 run([tricu, "eval", str(action), *flags, "--io", "--allow-read", str(message), 276 "--format", "decode"], cwd=root, contains="hello native") 277 run([tricu, "eval", str(action), *flags, "--compile-timeout", "0"], cwd=root, 278 ok=False, contains="--compile-timeout must be") 279 280 # A tiny, abstract-input-normalizable tool loops when the name is a 281 # concrete string. Verify cancellation rather than just option validation. 282 slow_source = root / "slow.tri" 283 slow_source.write_text('!export sourceBuildGraph\ny = ((mut wait fun : wait mut (x : fun (wait mut x)))\n' 284 ' (x : x x) (a0 a1 a2 : t (t a0) (t t a2) a1))\n' 285 'triage leaf stem fork = t (t leaf stem) fork\n' 286 'worker self input = triage t (child : t) (left right : self input) input\n' 287 'sourceBuildGraph directory root identity name = y worker name\n') 288 slow_bundle = root / "slow.arboricx" 289 run([tricu, "arboricx", "compile", "--file", str(slow_source), "--name", "sourceBuildGraph", 290 "--output", str(slow_bundle), "--store", str(root / "slow-bootstrap")], cwd=root) 291 slow_import, _ = run([tricu, "arboricx", "import", "--file", str(slow_bundle), 292 "--store", str(tools), *indexed_flags], cwd=root) 293 slow_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", slow_import).group(1) 294 run([tricu, "eval", str(action), "--toolchain", slow_pin, "--tool-store", str(tools), 295 "--store", str(store), "--compile-timeout", "1"], cwd=root, ok=False, 296 contains="deadline exceeded", seconds=15) 297 298 # Valid digest but unsupported ABI, then corrupt manifest/tree bytes: all 299 # must fail before tool invocation, rather than recompiling source. 300 def tool_object(domain, payload): 301 digest = hashlib.sha256(domain.encode() + b"\0" + payload).hexdigest() 302 path = tools / "objects" / digest[:3] / digest 303 path.parent.mkdir(parents=True, exist_ok=True) 304 path.write_bytes(payload) 305 return digest, path 306 307 forged, _ = tool_object("arboricx.module-manifest.v1", 308 ("arboricx.module-manifest.v1\nexport\tsourceBuildGraph\tarboricx.tree-term.v1\t" 309 + compiler_tree + "\tbad.abi\n").encode()) 310 run([tricu, "eval", str(app), "--toolchain", forged, "--tool-store", str(tools), 311 "--store", str(store)], cwd=root, ok=False, contains="Unsupported module export ABI") 312 wrong_kind, _ = tool_object("arboricx.module-manifest.v1", 313 ("arboricx.module-manifest.v1\nexport\tsourceBuildGraph\tother.object.v1\t" 314 + compiler_tree + "\tarboricx.abi.tree.v1\n").encode()) 315 run([tricu, "eval", str(app), "--toolchain", wrong_kind, "--tool-store", str(tools), 316 "--store", str(store)], cwd=root, ok=False, contains="Unsupported module export object kind") 317 manifest_path = tools / "objects" / manifest[:3] / manifest 318 original = manifest_path.read_bytes() 319 manifest_path.write_bytes(original + b"corrupt") 320 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="manifest digest mismatch") 321 manifest_path.write_bytes(original) 322 tree_path = tools / "objects" / compiler_tree[:3] / compiler_tree 323 tree_path.write_bytes(tree_path.read_bytes() + b"corrupt") 324 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="tree digest mismatch") 325 print("Native module CLI: patterns, execution, rebuilds, diagnostics and integrity passed", flush=True)