flake.nix (8173B)
1 { 2 description = "SemIf HTTP API (semif-api) + ROCm dev shell (gfx1151)"; 3 4 inputs = { 5 # Pin chosen for ROCm torch (gfx1151) prebuilt availability on cache.nixos.org. 6 nixpkgs.url = "github:NixOS/nixpkgs/e72e4f299401a3689d4b3d5fc6496b11db7064eb"; 7 # Upstream SemIf, consumed as a library dependency (not a fork). 8 semif-src = { 9 url = "github:TheoLeeCJ/SemIf/ca3ba65f142967030ecb453346e94d6f476a69df"; 10 flake = false; 11 }; 12 }; 13 14 outputs = { self, nixpkgs, semif-src }: 15 let 16 system = "x86_64-linux"; 17 pkgs = import nixpkgs { inherit system; config.allowUnfree = true; }; 18 py = pkgs.python3Packages; 19 20 # ROCm torch, prebuilt for gfx1151 on cache.nixos.org. 21 rocmTorch = py.torchWithRocm; 22 23 # torch is provided by rocmTorch; drop it from accelerate so the env 24 # never contains a second (plain CUDA) torch. 25 accelerate' = py.accelerate.overridePythonAttrs (old: { 26 propagatedBuildInputs = 27 builtins.filter (p: (p.pname or "") != "torch") 28 (old.propagatedBuildInputs or [ ]); 29 dependencies = 30 builtins.filter (p: (p.pname or "") != "torch") 31 (old.dependencies or [ ]); 32 }); 33 34 # Upstream SemIf built as a library. nixpkgs supplies the deps 35 # (nixpkgs does not resolve the pinned versions in its pyproject). 36 semif = py.buildPythonPackage { 37 pname = "semif-phase1"; 38 version = "0.1.0"; 39 src = semif-src; 40 pyproject = true; 41 # Upstream pins exact versions (torch==2.10.0 is CUDA-only); the nix 42 # env supplies newer compatible ones. 43 pythonRelaxDeps = true; 44 propagatedBuildInputs = with py; [ 45 rocmTorch 46 accelerate' 47 transformers 48 safetensors 49 huggingface-hub 50 tokenizers 51 numpy 52 sentencepiece 53 protobuf 54 ]; 55 pythonImportsCheck = [ "semif_phase1" ]; 56 }; 57 58 semif-api = py.buildPythonPackage { 59 pname = "semif-api"; 60 version = "0.1.0"; 61 src = ./semif-api; 62 pyproject = true; 63 propagatedBuildInputs = [ semif py.fastapi py.uvicorn ]; 64 pythonImportsCheck = [ "semif_api" ]; 65 }; 66 67 devPython = pkgs.python3.withPackages (ps: with ps; [ 68 semif 69 fastapi 70 uvicorn 71 pytest 72 httpx 73 pip 74 ]); 75 76 servePython = pkgs.python3.withPackages (ps: [ semif-api ps.uvicorn ]); 77 in 78 { 79 packages.${system} = { 80 inherit semif semif-api; 81 default = semif-api; 82 }; 83 84 devShells.${system}.default = pkgs.mkShell { 85 packages = [ devPython pkgs.git ]; 86 87 shellHook = '' 88 export IN_SEMIF_ROCM=1 89 export SEMIF_SRC=${semif-src} 90 export HF_HOME="$PWD/hf-cache" 91 if [ -d .venv ]; then 92 VENV_BASE=$(.venv/bin/python -c 'import sys; print(sys.base_prefix)' 2>/dev/null || echo none) 93 CUR_BASE=$(python -c 'import sys; print(sys.prefix)') 94 if [ "$VENV_BASE" != "$CUR_BASE" ]; then 95 echo "[semif-rocm] python env changed (stale venv), recreating..." 96 rm -rf .venv 97 fi 98 fi 99 if [ ! -d .venv ]; then 100 echo "[semif-rocm] creating venv (system site packages)..." 101 python -m venv --system-site-packages .venv 102 fi 103 if ! .venv/bin/python -c "import semif_api" 2>/dev/null; then 104 echo "[semif-rocm] installing semif-api (editable, no deps)..." 105 .venv/bin/python -m pip install -e ./semif-api --no-deps 106 fi 107 echo "[semif-rocm] torch: $(.venv/bin/python -c 'import torch; print(torch.__version__)') (ROCm/HIP: $(.venv/bin/python -c 'import torch; print(torch.version.hip is not None)'))" 108 echo "[semif-rocm] GPU visible to torch: $(.venv/bin/python -c 'import torch; print(torch.cuda.is_available())') (must be run on the host)" 109 ''; 110 }; 111 112 nixosModules.default = { config, lib, ... }: 113 let 114 cfg = config.services.semif-api; 115 serveScript = pkgs.writeShellScript "semif-api-serve" '' 116 export HF_HOME="''${STATE_DIRECTORY:-/var/lib/semif-api}/huggingface" 117 export CUDA_VISIBLE_DEVICES=''${CUDA_VISIBLE_DEVICES:-0} 118 exec ${semif-api}/bin/semif-serve 119 ''; 120 in 121 { 122 options.services.semif-api = { 123 enable = lib.mkEnableOption "semif-api semantic decision HTTP server"; 124 host = lib.mkOption { type = lib.types.str; default = "127.0.0.1"; }; 125 port = lib.mkOption { type = lib.types.port; default = 8321; }; 126 backend = lib.mkOption { type = lib.types.enum [ "torch" "llama" ]; default = "torch"; }; 127 maxTokens = lib.mkOption { type = lib.types.ints.positive; default = 4096; }; 128 llamaUrl = lib.mkOption { type = lib.types.str; default = "http://127.0.0.1:8080"; }; 129 llamaModel = lib.mkOption { 130 type = lib.types.str; 131 default = "Qwen3.5-4B"; 132 }; 133 llamaTimeout = lib.mkOption { type = lib.types.ints.positive; default = 600; }; 134 llamaNProbs = lib.mkOption { type = lib.types.ints.positive; default = 1024; }; 135 llamaMaxNProbs = lib.mkOption { type = lib.types.ints.positive; default = 16384; }; 136 llamaCachePrompt = lib.mkOption { type = lib.types.bool; default = true; }; 137 model = lib.mkOption { type = lib.types.str; default = "Qwen/Qwen3.5-4B"; }; 138 revision = lib.mkOption { 139 type = lib.types.str; 140 default = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"; 141 }; 142 user = lib.mkOption { type = lib.types.str; default = "semif"; }; 143 openFirewall = lib.mkOption { 144 type = lib.types.bool; 145 default = false; 146 description = "expose the API on the LAN (binds cfg.host; set host to 0.0.0.0 to listen on all interfaces)"; 147 }; 148 }; 149 150 config = lib.mkIf cfg.enable { 151 assertions = [{ 152 assertion = cfg.backend != "llama" || (cfg.llamaNProbs >= 16 && cfg.llamaMaxNProbs >= cfg.llamaNProbs); 153 message = "semif-api requires llamaMaxNProbs >= llamaNProbs >= 16"; 154 }]; 155 users.users.${cfg.user} = { 156 isSystemUser = true; 157 group = cfg.user; 158 extraGroups = [ "render" "video" ]; 159 }; 160 users.groups.${cfg.user} = { }; 161 162 systemd.services.semif-api = { 163 description = "SemIf semantic decision API (semif-api)"; 164 wantedBy = [ "multi-user.target" ]; 165 after = [ "network.target" ]; 166 167 environment = { 168 SEMIF_BACKEND = cfg.backend; 169 SEMIF_MAX_TOKENS = toString cfg.maxTokens; 170 SEMIF_LLAMA_URL = cfg.llamaUrl; 171 SEMIF_LLAMA_MODEL = cfg.llamaModel; 172 SEMIF_LLAMA_TIMEOUT = toString cfg.llamaTimeout; 173 SEMIF_LLAMA_N_PROBS = toString cfg.llamaNProbs; 174 SEMIF_LLAMA_MAX_N_PROBS = toString cfg.llamaMaxNProbs; 175 SEMIF_LLAMA_CACHE_PROMPT = lib.boolToString cfg.llamaCachePrompt; 176 SEMIF_MODEL = cfg.model; 177 SEMIF_REVISION = cfg.revision; 178 SEMIF_HOST = cfg.host; 179 SEMIF_PORT = toString cfg.port; 180 # System user has no home; MIOpen (ROCm) needs a writable 181 # kernel cache or every conv fails with miopenStatusUnknownError. 182 HOME = "%S/semif-api"; 183 XDG_CACHE_HOME = "%S/semif-api/.cache"; 184 }; 185 186 serviceConfig = { 187 User = cfg.user; 188 Group = cfg.user; 189 StateDirectory = "semif-api"; 190 StateDirectoryMode = "0750"; 191 ExecStart = serveScript; 192 Restart = "on-failure"; 193 RestartSec = 5; 194 }; 195 }; 196 197 networking.firewall.allowedTCPPorts = 198 lib.mkIf cfg.openFirewall [ cfg.port ]; 199 }; 200 }; 201 }; 202 }