portable_cli.py (22261B)
1 #!/usr/bin/env python3 2 """Opt-in native-module CLI checks; bootstrap preparation is explicit and separate.""" 3 import argparse 4 import hashlib 5 import json 6 import os 7 import pathlib 8 import re 9 import subprocess 10 import tempfile 11 import time 12 from generation_backend import BACKENDS, DEFAULT_BACKEND 13 from generation_io import cli_version, compiler_flags 14 15 parser = argparse.ArgumentParser() 16 parser.add_argument("tricu") 17 parser.add_argument("--backend", choices=BACKENDS, default=DEFAULT_BACKEND, 18 help="lowering backend (default: %(default)s)") 19 parser.add_argument("--alternate-compiler-bundle", type=pathlib.Path, 20 help="Check switching backend/cache identity using the other backend's artifact") 21 parser.add_argument("--compiler-io-version", type=cli_version, choices=(2,), default=2) 22 parser.add_argument("--compiler-bundle", type=pathlib.Path, 23 help="Reuse an explicitly prepared sourceBuildGraph bundle") 24 transport = parser.add_mutually_exclusive_group() 25 transport.add_argument("--indexed", action="store_true", default=True, help="Use compact indexed storage (default)") 26 transport.add_argument("--flat", action="store_false", dest="indexed", help="Explicit flat transport probe") 27 parser.add_argument("--max-flat-bytes", type=int, default=64 * 1024 * 1024) 28 args = parser.parse_args() 29 indexed_flags = ["--indexed"] if args.indexed else ["--flat", "--max-flat-bytes", str(args.max_flat_bytes)] 30 tricu = str(pathlib.Path(args.tricu).resolve()) 31 repo = pathlib.Path(__file__).resolve().parent.parent 32 33 def run(command, *, cwd, ok=True, contains=None, seconds=180): 34 start = time.monotonic() 35 reply = subprocess.run([*command, '+RTS', '-N1', '-M8192m', '-RTS'], cwd=cwd, text=True, capture_output=True, timeout=seconds) 36 assert (reply.returncode == 0) == ok, (command, reply.returncode, reply.stdout, reply.stderr) 37 if contains is not None: 38 assert contains in reply.stdout + reply.stderr, (command, reply.stdout, reply.stderr) 39 return reply.stdout.strip(), time.monotonic() - start 40 41 with tempfile.TemporaryDirectory(prefix="tricu-portable-cli-") as temporary: 42 root = pathlib.Path(temporary) 43 tools, store, workspace = root / "tools", root / "application-store", root / "application" 44 workspace.mkdir() 45 bundle = args.compiler_bundle 46 if bundle is None: 47 if args.backend != "stable": 48 parser.error("--compiler-bundle is required for a non-stable backend") 49 bundle = repo / "bootstrap/build-stable.arboricx" 50 output, _ = run([tricu, "arboricx", "import", "--file", str(bundle.resolve()), 51 "--store", str(tools), *indexed_flags], cwd=root) 52 manifest = re.search(r"Created module alias .* -> ([0-9a-f]{64})", output).group(1) 53 compiler_tree = re.search(r"sourceBuildGraph -> ([0-9a-f]{64})", output).group(1) 54 compiler_payload = (tools / "objects" / compiler_tree[:3] / compiler_tree).read_bytes() 55 compiler_domain = "arboricx.indexed-tree.v1" if args.indexed else "arboricx.tree-term.v1" 56 assert hashlib.sha256(compiler_domain.encode() + b"\0" + compiler_payload).hexdigest() == compiler_tree 57 assert not args.indexed or len(compiler_payload) < 1024 * 1024 58 print(f"compiler CAS bytes={len(compiler_payload):,} kind={compiler_domain}", flush=True) 59 flags = ["--toolchain", manifest, *compiler_flags(args.compiler_io_version), "--tool-store", str(tools), "--store", str(store)] 60 patterns = workspace / "patterns.tri" 61 app = workspace / "app.tri" 62 patterns.write_text("!export first\n!export shape\n" 63 "first whole{t left right} = left\nfirst _ = 99\n" 64 "shape t = 0\nshape (t t) = 1\nshape _ = 2\n") 65 app.write_text("!import patterns as P\n!export main\n!export probe\n" 66 "main = P.first (t 23 24)\nprobe = P.shape (t (t t))\n") 67 assert not store.exists() 68 # Every application invocation is outside the checkout, without a workspace 69 # alias file, and uses a store containing no compiler or standard library. 70 text, cold = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 71 assert text == "23", text 72 text, warm = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 73 assert text == "23", text 74 print(f"portable eval cold={cold:.2f}s warm={warm:.2f}s", flush=True) 75 profile = subprocess.run([tricu, "eval", str(app), *flags, "--format", "number", 76 "--compile-profile", "+RTS", "-T", "-N1", "-M8192m", "-RTS"], 77 cwd=root, text=True, capture_output=True, timeout=180) 78 assert profile.returncode == 0 and profile.stdout.strip() == "23", profile.stderr 79 stages = [json.loads(row) for row in profile.stderr.splitlines()] 80 assert [stage["compiler_stage"] for stage in stages] == [ 81 "begin", "manifest-verified", "executable-read", "executable-digest-verified", 82 "executable-decoded", "build-returned", "publication-verified"] 83 assert all(a["wall_ns"] <= b["wall_ns"] for a, b in zip(stages, stages[1:])) 84 begin, decoded, end = (stages[i]["wall_ns"] for i in (0, 4, -1)) 85 print(f"profile tool-load={(decoded-begin)/1e9:.3f}s build={(end-decoded)/1e9:.3f}s", flush=True) 86 87 def get(domain, digest): 88 payload = (store / "objects" / digest[:3] / digest).read_bytes() 89 assert hashlib.sha256(domain.encode() + b"\0" + payload).hexdigest() == digest 90 return payload 91 92 def checkpoint(name): 93 fields = (store / "cache" / "source-build" / name).read_text().split("\t") 94 assert fields[0] == "arboricx.source-build-cache.v1" 95 receipt = get("arboricx.build-receipt.v1", fields[3]).decode().splitlines() 96 assert f"compiler\t{compiler_tree}" in receipt 97 assert f"output\t{fields[2]}" in receipt 98 return fields[2], fields[3], receipt 99 100 initial, initial_receipt, receipt = checkpoint("app") 101 expected_options = "source-module.stable.v1" if args.backend == "stable" else "source-module.reference.v1" 102 assert "options\t" + hashlib.sha256(b"arboricx.build-options.v1\0" + expected_options.encode()).hexdigest() in receipt 103 assert any(row.startswith("local\tpatterns\t") for row in receipt) 104 checkpoint("patterns") 105 assert not (store / "objects" / compiler_tree[:3] / compiler_tree).exists() 106 107 for selected, expected in [("main", "23"), ("probe", "2")]: 108 artifact = root / (selected + ".arboricx") 109 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 110 "--name", selected, "--output", str(artifact)], cwd=root) 111 imported_one, _ = run([tricu, "arboricx", "import", "--file", str(artifact), 112 "--store", str(root / ("runtime-" + selected))], cwd=root) 113 selected_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", imported_one).group(1) 114 text, _ = run([tricu, "arboricx", "run", "--module", selected_pin, "--name", selected, 115 "--store", str(root / ("runtime-" + selected)), "--format", "number"], cwd=root) 116 assert text == expected, (selected, text) 117 all_exports = root / "all.arboricx" 118 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 119 "--output", str(all_exports)], cwd=root) 120 imported, _ = run([tricu, "arboricx", "import", "--file", str(all_exports), 121 "--store", str(root / "runtime"), *indexed_flags], cwd=root) 122 assert "main ->" in imported and "probe ->" in imported 123 runtime_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", imported).group(1) 124 for selected, expected in [("main", "23"), ("probe", "2")]: 125 text, _ = run([tricu, "arboricx", "run", "--store", str(root / "runtime"), 126 "--module", runtime_pin, "--name", selected, "--format", "number"], cwd=root) 127 assert text == expected, (selected, text) 128 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 129 "--name", "missing", "--output", str(root / "missing.arboricx")], 130 cwd=root, ok=False, contains="Module export not found") 131 run([tricu, "arboricx", "compile", "--file", str(app), *flags, 132 "--name", "main", "--name", "main", "--output", str(root / "duplicate.arboricx")], 133 cwd=root, ok=False, contains="Duplicate --name") 134 assert not (root / "missing.arboricx").exists() 135 assert not (root / "duplicate.arboricx").exists() 136 # Ordinary source-free execution shares the verified loader. 137 text, _ = run([tricu, "arboricx", "run", "--store", str(store), 138 "--module", initial, "--name", "main", "--format", "number"], cwd=root) 139 assert text == "23" 140 141 if args.alternate_compiler_bundle: 142 alternate, _ = run([tricu, "arboricx", "import", "--file", str(args.alternate_compiler_bundle.resolve()), 143 "--store", str(tools), *indexed_flags], cwd=root) 144 alternate_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", alternate).group(1) 145 alternate_tree = re.search(r"sourceBuildGraph -> ([0-9a-f]{64})", alternate).group(1) 146 alternate_flags = ["--toolchain", alternate_pin, *compiler_flags(args.compiler_io_version), "--tool-store", str(tools), "--store", str(store)] 147 text, _ = run([tricu, "eval", str(app), *alternate_flags, "--format", "number"], cwd=root) 148 assert text == "23" 149 switched = (store / "cache" / "source-build" / "app").read_text().split("\t") 150 switched_receipt = get("arboricx.build-receipt.v1", switched[3]).decode() 151 other_options = "source-module.reference.v1" if args.backend == "stable" else "source-module.stable.v1" 152 assert "compiler\t" + alternate_tree in switched_receipt 153 assert "options\t" + hashlib.sha256(b"arboricx.build-options.v1\0" + other_options.encode()).hexdigest() in switched_receipt 154 assert switched[3] != initial_receipt 155 run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 156 assert checkpoint("app")[:2] == (initial, initial_receipt) 157 print("backend switch: cache/receipt identities separated and restored", flush=True) 158 159 if args.backend == "stable": 160 recursive = workspace / "recursive.tri" 161 recursive.write_text('!export copy\ncopy [] = []\ncopy [head | tail] = t head (copy tail)\ncopy _ = []\n') 162 caller = workspace / "caller.tri" 163 caller.write_text('!import recursive as R\n!export main\nmain = R.copy "hello"\n') 164 text, _ = run([tricu, "eval", str(caller), *flags, "--format", "string"], cwd=root) 165 assert text == '"hello"', text 166 if args.alternate_compiler_bundle: 167 run([tricu, "eval", str(caller), *alternate_flags], cwd=root, ok=False, contains="acyclic") 168 recursive.write_text('!export copy\ncopy = copy\n') 169 run([tricu, "eval", str(caller), *flags], cwd=root, ok=False, 170 contains="recursive function requires parameters") 171 print("native recursive dependency, optimized rejection, recursive-value diagnostic pass", flush=True) 172 173 patterns.write_text(patterns.read_text().replace("= left", "= right")) 174 text, changed = run([tricu, "eval", str(app), *flags, "--format", "number"], cwd=root) 175 assert text == "24", text 176 updated, updated_receipt, _ = checkpoint("app") 177 assert (initial, initial_receipt) != (updated, updated_receipt) 178 print(f"changed dependency rebuild={changed:.2f}s", flush=True) 179 # Header consolidation is acquisition + binding, not a store alias fallback. 180 defaults = workspace / 'defaults.tri' 181 defaults.write_text('!import patterns\n!import patterns as P\n!export main\nmain = patterns.first (t 22 23)\n') 182 text, _ = run([tricu, 'eval', str(defaults), *flags, '--format', 'number'], cwd=root) 183 assert text == '23', text 184 _, _, default_rows = checkpoint('defaults') 185 assert len([row for row in default_rows if row.startswith('local\tpatterns\t')]) == 1 186 barrel = workspace / 'barrel.tri' 187 barrel.write_text('!export * from patterns\n') 188 through = workspace / 'through.tri' 189 through.write_text('!import barrel *\n!export main\nmain = first (t 22 23)\n') 190 text, _ = run([tricu, 'eval', str(through), *flags, '--format', 'number'], cwd=root) 191 assert text == '23', text 192 pattern_manifest, _, _ = checkpoint('patterns') 193 assert checkpoint('barrel')[0] == pattern_manifest # exact executable trees 194 pinned = workspace / 'pinned.tri' 195 pinned.write_text('!import '+pattern_manifest+' as P\n!export main\nmain = P.first (t 22 23)\n') 196 text, _ = run([tricu, 'eval', str(pinned), *flags, '--format', 'number'], cwd=root) 197 assert text == '23', text 198 assert not any(row.startswith('local\t') for row in checkpoint('pinned')[2]) 199 build_only = workspace / 'build_only.tri' 200 build_only.write_text('!source patterns\n!export main\nmain = 7\n') 201 text, _ = run([tricu, 'eval', str(build_only), *flags, '--format', 'number'], cwd=root) 202 assert text == '7', text 203 204 # Build-scoped manifest/tree reuse must not turn an unused import into an 205 # unchecked pin. Repeated qualifiers share one authenticated provider, but 206 # every export remains subject to integrity checking, cold and warm. 207 unused = workspace / 'unused.tri' 208 unused.write_text('!import '+pattern_manifest+' as P\n!import '+pattern_manifest+ 209 ' as Q\n!export main\nmain = 7\n') 210 text, _ = run([tricu, 'eval', str(unused), *flags, '--format', 'number'], cwd=root) 211 assert text == '7', text 212 unused_checkpoint = (store / 'cache/source-build/unused').read_bytes() 213 provider_bytes = get('arboricx.module-manifest.v1', pattern_manifest) 214 shape_hash = next(row.split('\t')[3] for row in provider_bytes.decode().splitlines()[1:] 215 if row.split('\t')[1] == 'shape') 216 shape_path = store / 'objects' / shape_hash[:3] / shape_hash 217 shape_bytes = shape_path.read_bytes() 218 for cold in (False, True): 219 if cold: 220 unused.write_text(unused.read_text() + '-- force a source cache miss\n') 221 shape_path.write_bytes(shape_bytes + b'corrupt') 222 run([tricu, 'eval', str(unused), *flags], cwd=root, ok=False, contains='CAS digest mismatch') 223 assert (store / 'cache/source-build/unused').read_bytes() == unused_checkpoint 224 shape_path.write_bytes(shape_bytes) 225 provider_path = store / 'objects' / pattern_manifest[:3] / pattern_manifest 226 provider_path.write_bytes(provider_bytes + b'corrupt') 227 run([tricu, 'eval', str(unused), *flags], cwd=root, ok=False, contains='CAS digest mismatch') 228 provider_path.write_bytes(provider_bytes) 229 text, _ = run([tricu, 'eval', str(unused), *flags, '--format', 'number'], cwd=root) 230 assert text == '7', text 231 print('unused repeated imports: cold/warm tree and manifest corruption rejected', flush=True) 232 233 patterns.write_text("!export first\nfirst x x = missing\nfirst _ _ = t\n") 234 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="unique capture") 235 # A failure must not replace the last successful checkpoint. 236 assert checkpoint("app")[:2] == (updated, updated_receipt) 237 238 def bad_source(name, text, message): 239 path = workspace / (name + ".tri") 240 path.write_text(text) 241 run([tricu, "eval", str(path), *flags], cwd=root, ok=False, contains=message) 242 243 bad_source('old_native', '!use patterns *\n!export main\nmain = t\n', 'unknown native header') 244 bad_source('bare_hash', '!import '+pattern_manifest+'\n!export main\nmain = t\n', 'hash imports require') 245 for index, header in enumerate(['!source patterns\n!import patterns\n', 246 '!import patterns\n!source patterns\n', 247 '!source patterns\n!export * from patterns\n']): 248 bad_source('redundant'+str(index), header+'!export main\nmain = t\n', 'redundant build-only') 249 # Restore a valid dependency before collision probes. 250 patterns.write_text('!export first\nfirst x = x\n') 251 bad_source('collision', '!import patterns *\n!export first\nfirst = t\n', 'first') 252 bad_source('duplicate_import', '!import patterns *\n!import patterns *\n!export main\nmain = t\n', 'duplicate imported name') 253 bad_source("legacy", '!import "prelude" !Local\nmain = t\n', "Portable compilation failed") 254 # A moving alias in the application CAS must not rescue a missing local. 255 run([tricu, 'arboricx', 'import', '--file', str(all_exports), 256 '--store', str(store), '--module', 'absent'], cwd=root) 257 bad_source("undefined", "!import absent *\n!export main\nmain = t\n", "does not exist") 258 bad_source("cycle", "!import cycle *\n!export main\nmain = t\n", "dependency cycle") 259 bad_source("expression", "!export main\nmain = t\nmain\n", "Portable compilation failed") 260 bad_source("noMain", "!export value\nvalue = t\n", "Module export not found: main") 261 run([tricu, "eval", str(app), str(app), *flags], cwd=root, ok=False, 262 contains="one native module") 263 run([tricu, "eval", str(app), "--tool-store", str(tools)], cwd=root, ok=False, 264 contains="--tool-store requires --toolchain") 265 run([tricu, "eval", str(app), "--toolchain", "portable-compiler", "--tool-store", str(tools), 266 "--store", str(store)], cwd=root, ok=False, contains="Invalid module manifest hash") 267 run([tricu, "eval", str(app), "--toolchain", manifest, "--tool-store", str(root / "absent"), 268 "--store", str(store)], cwd=root, ok=False, contains="Module manifest not found") 269 270 # The compiler's ability to build a source does not confer OS read access 271 # to the resulting v2 application. No grant flags or implicit state slot. 272 if os.geteuid() == 0: 273 parser.error("OS-denial application control requires non-root") 274 message = root / "message.txt" 275 message.write_text("hello native") 276 action = workspace / "action.tri" 277 action.write_text('!export main\nrequest group operation arguments = t 10 (t group (t 1 (t operation arguments)))\nread reader = t 1 (t (request "stream" "read" (t reader 64)) (reply : case reply of\n ((t t), (("data", bytes), t)) = t 0 bytes\n _ = t 0 reply))\nmain = t "tricuIO" (t 2 (t 3 (owner : t 1\n (t (request "file" "open-reader" (t owner PATH))\n (reply : case reply of\n ((t t), (reader, t)) = read reader\n _ = t 0 reply)))))\n'.replace('PATH', json.dumps(str(message)))) 278 message.chmod(0) 279 try: 280 denied, _ = run([tricu, "eval", str(action), *flags, "--io", "--format", "decode"], cwd=root) 281 assert "permission" in denied and "hello native" not in denied 282 finally: 283 message.chmod(0o600) 284 run([tricu, "eval", str(action), *flags, "--io", "--format", "decode"], cwd=root, 285 contains="hello native") 286 run([tricu, "eval", str(action), *flags, "--io", "--allow-read", str(message), 287 "--format", "decode"], cwd=root, ok=False, contains="permission grants are not accepted") 288 run([tricu, "eval", str(action), *flags, "--compile-timeout", "0"], cwd=root, 289 ok=False, contains="--compile-timeout must be") 290 291 # A tiny, abstract-input-normalizable tool loops when the name is a 292 # concrete string. Verify cancellation rather than just option validation. 293 slow_source = root / "slow.tri" 294 slow_source.write_text('!export sourceBuildGraph\ny = ((mut wait fun : wait mut (x : fun (wait mut x)))\n' 295 ' (x : x x) (a0 a1 a2 : t (t a0) (t t a2) a1))\n' 296 'triage leaf stem fork = t (t leaf stem) fork\n' 297 'worker self input = triage t (child : t) (left right : self input) input\n' 298 'sourceBuildGraph directory root identity name = y worker name\n') 299 slow_bundle = root / "slow.arboricx" 300 run([tricu, "arboricx", "compile", "--file", str(slow_source), "--name", "sourceBuildGraph", 301 "--output", str(slow_bundle), "--store", str(root / "slow-bootstrap")], cwd=root) 302 slow_import, _ = run([tricu, "arboricx", "import", "--file", str(slow_bundle), 303 "--store", str(tools), *indexed_flags], cwd=root) 304 slow_pin = re.search(r"Created module alias .* -> ([0-9a-f]{64})", slow_import).group(1) 305 run([tricu, "eval", str(action), "--toolchain", slow_pin, "--tool-store", str(tools), 306 "--store", str(store), "--compile-timeout", "1"], cwd=root, ok=False, 307 contains="deadline exceeded", seconds=15) 308 309 # Valid digest but unsupported ABI, then corrupt manifest/tree bytes: all 310 # must fail before tool invocation, rather than recompiling source. 311 def tool_object(domain, payload): 312 digest = hashlib.sha256(domain.encode() + b"\0" + payload).hexdigest() 313 path = tools / "objects" / digest[:3] / digest 314 path.parent.mkdir(parents=True, exist_ok=True) 315 path.write_bytes(payload) 316 return digest, path 317 318 forged, _ = tool_object("arboricx.module-manifest.v1", 319 ("arboricx.module-manifest.v1\nexport\tsourceBuildGraph\tarboricx.tree-term.v1\t" 320 + compiler_tree + "\tbad.abi\n").encode()) 321 run([tricu, "eval", str(app), "--toolchain", forged, "--tool-store", str(tools), 322 "--store", str(store)], cwd=root, ok=False, contains="Unsupported module export ABI") 323 wrong_kind, _ = tool_object("arboricx.module-manifest.v1", 324 ("arboricx.module-manifest.v1\nexport\tsourceBuildGraph\tother.object.v1\t" 325 + compiler_tree + "\tarboricx.abi.tree.v1\n").encode()) 326 run([tricu, "eval", str(app), "--toolchain", wrong_kind, "--tool-store", str(tools), 327 "--store", str(store)], cwd=root, ok=False, contains="Unsupported module export object kind") 328 manifest_path = tools / "objects" / manifest[:3] / manifest 329 original = manifest_path.read_bytes() 330 manifest_path.write_bytes(original + b"corrupt") 331 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="manifest digest mismatch") 332 manifest_path.write_bytes(original) 333 tree_path = tools / "objects" / compiler_tree[:3] / compiler_tree 334 tree_path.write_bytes(tree_path.read_bytes() + b"corrupt") 335 run([tricu, "eval", str(app), *flags], cwd=root, ok=False, contains="tree digest mismatch") 336 print("Native module CLI: patterns, execution, rebuilds, diagnostics and integrity passed", flush=True)