test_api.py (6211B)
1 #!/usr/bin/env python3 2 """End-to-end parity test for the semif-api HTTP server. 3 4 Assumes uvicorn is already running on 127.0.0.1:8321 (started by validate.sh). 5 Compares API output against the CLI reference outputs field-by-field on the 6 deterministic fields (timing fields are excluded), then asserts the error 7 contract: every input problem is a 422 carrying the upstream message. 8 """ 9 import json 10 import sys 11 import urllib.request 12 13 BASE = "http://127.0.0.1:8321" 14 DIR = __import__("pathlib").Path(__file__).resolve().parent.parent / "examples" 15 16 # Timing fields are machine-specific and not compared; the model dict is 17 # narrowed to source+revision so parity does not depend on local 18 # torch/transformers versions. 19 COMPARE_FIELDS = ("id", "option_ids", "probabilities", "option_logits", 20 "input_tokens", "prompt_sha256", "prompt_version") 21 22 23 def comparable(row): 24 model = row.get("model") or {} 25 fields = {field: row.get(field) for field in COMPARE_FIELDS} 26 fields["model_source"] = model.get("source") 27 fields["model_revision"] = model.get("revision") 28 return fields 29 30 31 def get(path): 32 with urllib.request.urlopen(BASE + path, timeout=30) as response: 33 return json.load(response) 34 35 36 def post(path, payload): 37 request = urllib.request.Request( 38 BASE + path, data=json.dumps(payload).encode(), 39 headers={"Content-Type": "application/json"}) 40 try: 41 with urllib.request.urlopen(request, timeout=120) as response: 42 return json.load(response) 43 except urllib.error.HTTPError as error: 44 print(f"POST {path} -> HTTP {error.code}: {error.read().decode()}") 45 raise 46 47 48 def post_status(path, payload): 49 """POST and report (status, body) instead of raising, for expected failures.""" 50 request = urllib.request.Request( 51 BASE + path, data=json.dumps(payload).encode(), 52 headers={"Content-Type": "application/json"}) 53 try: 54 with urllib.request.urlopen(request, timeout=120) as response: 55 return response.status, json.load(response) 56 except urllib.error.HTTPError as error: 57 body = error.read().decode() 58 try: 59 return error.code, json.loads(body) 60 except json.JSONDecodeError: 61 return error.code, {"detail": body} 62 63 64 def diff_fields(a, b): 65 ca, cb = comparable(a), comparable(b) 66 return {field: (ca.get(field), cb.get(field)) 67 for field in ca if ca.get(field) != cb.get(field)} 68 69 70 def main(): 71 health = get("/healthz") 72 print(f"healthz: ok, model={health['model']['source']}@{health['model']['revision'][:12]}") 73 74 rows = [json.loads(line) for line in (DIR / "examples-shared.jsonl").read_text().splitlines()] 75 76 # 1) /decide must match the CLI direct-mode output exactly. 77 reference = {r["id"]: r for r in map(json.loads, (DIR / "reference-direct.jsonl").read_text().splitlines())} 78 failures = 0 79 for row in rows: 80 result = post("/decide", row) 81 fields = diff_fields(result, reference[row["id"]]) 82 if fields: 83 failures += 1 84 print(f"decide {row['id']}: MISMATCH {fields}") 85 else: 86 print(f"decide {row['id']}: exact match") 87 assert result["probability_status"].startswith("conditional option score") 88 89 # 2) /decide-batch must match the CLI shared-mode output exactly (same code path). 90 reference_shared = {r["id"]: r for r in map(json.loads, (DIR / "reference-shared.jsonl").read_text().splitlines())} 91 batch = post("/decide-batch", { 92 "state": rows[0]["state"], 93 "decisions": [{"id": r["id"], "question": r["question"], "options": r["options"]} for r in rows], 94 }) 95 if len(batch["results"]) != len(rows): 96 print(f"decide-batch: expected {len(rows)} results, got {len(batch['results'])}") 97 failures += 1 98 for result in batch["results"]: 99 fields = diff_fields(result, reference_shared[result["id"]]) 100 if fields: 101 failures += 1 102 print(f"decide-batch {result['id']}: MISMATCH {fields}") 103 else: 104 print(f"decide-batch {result['id']}: exact match") 105 print(f"batch timing: total={batch['timing']['total_seconds']:.3f}s " 106 f"batch_size={batch['timing']['batch_size']}") 107 108 # 3) Error contract (docs/usage.md): every input problem is 422 with the 109 # upstream message. The last three cases raise inside the scorer, so they 110 # are the ones that used to regress to a bare "Internal Server Error". 111 long_state = "no new information. " * 6000 112 pair = [{"id": "a", "description": "a"}, {"id": "b", "description": "b"}] 113 contract = [ 114 ("/decide", "one option", {"id": "e1", "state": "s", "question": "q?", 115 "options": [{"id": "only", "description": "one"}]}, "options must contain 2-16 entries"), 116 ("/decide", "duplicate option ids", {"id": "e2", "state": "s", "question": "q?", 117 "options": [{"id": "a", "description": "a"}, {"id": "a", "description": "b"}]}, 118 "Option IDs must be unique"), 119 ("/decide", "over max_tokens", {"id": "e3", "state": long_state, "question": "q?", 120 "options": pair}, "exceed limit"), 121 ("/decide-batch", "duplicate decision ids", {"state": "s", "decisions": [ 122 {"id": "dup", "question": "a?", "options": pair}, 123 {"id": "dup", "question": "b?", "options": pair}]}, "Decision IDs must be unique"), 124 ("/decide-batch", "empty decisions", {"state": "s", "decisions": []}, 125 "Shared scoring requires one nonempty exact state"), 126 ] 127 for path, label, payload, needle in contract: 128 status, body = post_status(path, payload) 129 detail = body.get("detail") if isinstance(body, dict) else None 130 if not isinstance(detail, str): 131 detail = json.dumps(body) 132 if status != 422 or needle not in detail: 133 failures += 1 134 print(f"contract {label}: MISMATCH got {status} {detail[:90]!r}, want 422 containing {needle!r}") 135 else: 136 print(f"contract {label}: 422 {detail[:64]}") 137 138 if failures: 139 print(f"\n{failures} PARITY FAILURE(S)") 140 sys.exit(1) 141 print("\nAPI parity: all exact matches") 142 143 144 if __name__ == "__main__": 145 main()