validate.sh (1891B)
1 #!/usr/bin/env bash 2 # Full validation: upstream unit tests, CLI direct + shared scoring, 3 # drift compare, HTTP API parity. 4 # Safe to run from any directory; re-execs into the nix dev shell if needed. 5 set -euo pipefail 6 cd "$(dirname "$(readlink -f "$0")")/.." 7 ROOT=$PWD 8 9 if [ "${IN_SEMIF_ROCM:-0}" != "1" ]; then 10 echo "[validate] entering nix dev shell..." 11 exec nix develop "$ROOT" --command "$0" "$@" 12 fi 13 14 PY=.venv/bin/python 15 SCORER=semif-score # from the nix env (semif package), not the venv 16 MODEL="Qwen/Qwen3.5-4B" 17 REV="851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a" 18 export CUDA_VISIBLE_DEVICES=0 19 20 echo "=== [1/4] upstream unit tests (mocked, mlx tests skipped) ===" 21 $PY -m pytest "$SEMIF_SRC/tests" -q -p no:cacheprovider \ 22 --ignore="$SEMIF_SRC/tests/test_mlx.py" \ 23 --ignore="$SEMIF_SRC/tests/test_mlx_evidence.py" 24 25 echo "=== [2/4] direct mode on shared-state input ===" 26 rm -f results-direct-on-shared.jsonl 27 $SCORER --mode direct --model "$MODEL" --revision "$REV" \ 28 --input examples/examples-shared.jsonl --output results-direct-on-shared.jsonl 29 30 echo "=== [3/4] shared mode + drift compare ===" 31 rm -f results-shared.jsonl 32 $SCORER --mode shared --model "$MODEL" --revision "$REV" \ 33 --input examples/examples-shared.jsonl --output results-shared.jsonl 34 $PY scripts/compare.py 35 36 echo "=== [4/4] HTTP API parity ===" 37 $PY -m uvicorn semif_api.app:app --host 127.0.0.1 --port 8321 > uvicorn.log 2>&1 & 38 SERVER_PID=$! 39 trap 'kill $SERVER_PID 2>/dev/null || true' EXIT 40 for attempt in $(seq 1 90); do 41 if $PY -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8321/healthz', timeout=2)" 2>/dev/null; then 42 break 43 fi 44 if ! kill -0 $SERVER_PID 2>/dev/null; then 45 echo "uvicorn died during startup; last log lines:" 46 tail -20 uvicorn.log 47 exit 1 48 fi 49 sleep 2 50 done 51 $PY tests/test_api.py 52 kill $SERVER_PID 2>/dev/null || true 53 trap - EXIT 54 55 echo "=== all done ==="