Add Grafana & Loki & Promtail monitoring stack

application/containers.nix

@@ -1,6 +1,7 @@
 {
   virtualisation.oci-containers = {
     containers = {
+
       bind = {
         image = "sameersbn/bind:latest";
         ports =
@@ -8,6 +9,7 @@
         volumes = [ "/home/sezycei/srv/containerdata/bind/bind:/data" ];
         environmentFiles = [ /home/sezycei/srv/containerdata/bind/.env ];
       };
+
       jellyfin = {
         image = "linuxserver/jellyfin";
         ports = [ "0.0.0.0:8096:8096" "0.0.0.0:8920:8920" ];
@@ -26,6 +28,7 @@
           UMASK_SET = "022";
         };
       };
+
       legit = {
         image = "docker.matri.cx/legit";
         ports = [ "0.0.0.0:5121:8080" ];
@@ -37,6 +40,7 @@
         ];
         environment = { };
       };
+
       murmur = {
         image = "goofball222/murmur";
         ports = [ "0.0.0.0:64738:64738" "0.0.0.0:64738:64738/udp" ];
@@ -47,20 +51,7 @@
         ];
         environment = { };
       };
-      nginx = {
-        image = "nginx:alpine";
-        ports = [ "0.0.0.0:80:80" "0.0.0.0:443:443" "0.0.0.0:20222:20222" ];
-        volumes = [
-          "/home/sezycei/srv/web/www:/var/www/"
-          "/home/sezycei/srv/web/configuration/nginx.conf:/etc/nginx/nginx.conf"
-          "/home/sezycei/srv/web/configuration/htpasswd:/etc/nginx/htpasswd"
-          "/home/sezycei/srv/web/configuration/htpasswd-dock:/etc/nginx/htpasswd-dock"
-          "/home/sezycei/srv/web/configuration/sites-available:/etc/nginx/sites-enabled"
-          "/home/sezycei/srv/web/ssl/letsencrypt:/etc/letsencrypt"
-          "/home/sezycei/srv/web/ssl/dhparam.pem:/etc/ssl/certs/dhparam.pem"
-        ];
-        environment = { };
-      };
+
       purr = {
         image = "docker.matri.cx/purr";
         ports = [ "0.0.0.0:5195:3000" ];
@@ -70,6 +61,7 @@
         ];
         environment = { };
       };
+
       registry = {
         image = "registry:2";
         ports = [ "0.0.0.0:3001:5000" ];
@@ -80,6 +72,7 @@
         ];
         environment = { };
       };
+
       transmission = {
         image = "haugene/transmission-openvpn";
         ports = [ "0.0.0.0:9091:9091" ];
@@ -92,6 +85,7 @@
           [ /home/sezycei/srv/containerdata/transmission/.env ];
         extraOptions = [ "--cap-add=NET_ADMIN" "--privileged" ];
       };
+
     };
   };
 }

application/nginx.nix

@@ -0,0 +1,121 @@
+{ config, ... }: {
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    recommendedOptimisation = true;
+    recommendedGzipSettings = true;
+
+    virtualHosts."cache.matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/cache.matri.cx";
+      locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
+    };
+
+    virtualHosts."caitlynncox.com" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/caitlynncox.com";
+    };
+
+    virtualHosts."dallasmed65.com" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/dallasmed65.com";
+    };
+
+    virtualHosts."docker.matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/docker.matri.cx";
+      locations."/" = {
+        basicAuthFile = config.age.secrets.htpasswd-dock.path;
+        proxyPass = "http://192.168.0.130:3001";
+      };
+    };
+
+    virtualHosts."eversole.co" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/jame.su";
+      locations = {
+        "/.well-known/openpgpkey/hu/".extraConfig = ''
+          default_type "application/octet-stream";
+          add_header Access-Control-Allow-Origin * always;
+        '';
+        "/ip".proxyPass = "http://192.168.0.130:5001";
+        "/pw".proxyPass = "http://192.168.0.130:5002";
+      };
+    };
+
+    virtualHosts."git.eversole.co" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/git.eversole.co";
+      locations = {
+        "/James".extraConfig = ''
+          if ($request_uri ~* "([^/]*$)" ) {
+            return 301 https://$server_name/$1;
+          }
+        '';
+        "/".proxyPass = "http://192.168.0.130:5121";
+      };
+    };
+
+    # refer to /monitoring/nginx.nix
+    virtualHosts."graf.eversole.co" = {
+      root = "/var/www/graf.eversole.co";
+    };
+
+    virtualHosts."hydra.matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/hydra.matri.cx";
+      locations."/" = {
+        proxyPass = "http://192.168.0.130:3034";
+        extraConfig = ''
+          proxy_set_header X-Request-Base "https://hydra.matri.cx";
+        '';
+      };
+    };
+
+    virtualHosts."jame.su" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/jame.su";
+    };
+
+    virtualHosts."matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/matri.cx";
+    };
+
+    virtualHosts."media.matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/media.matri.cx";
+      locations."/".proxyPass = "http://192.168.0.130:8096";
+    };
+
+    virtualHosts."sezycei.com" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/sezycei.com";
+    };
+
+    virtualHosts."snakebelmont.com" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/snakebelmont.com";
+    };
+
+    virtualHosts."purr.eversole.co" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/purr.eversole.co";
+      locations."/".proxyPass = "http://192.168.0.130:5195";
+    };
+
+  };
+}

flake.nix

@@ -13,10 +13,11 @@
       pkgs = import nixpkgs { inherit system; };
       shell = import ./shell.nix { inherit agenix pkgs; };
       system = "x86_64-linux";
-    in {
+    in
+    {
       devShell.x86_64-linux = shell.dev;
 
-      formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt;
+      formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixpkgs-fmt;
 
       nixosConfigurations = {
         eve-psr-nix0 = nixpkgs.lib.nixosSystem {
@@ -28,7 +29,16 @@
               imports = [
                 agenix.nixosModules.default
 
-                ./system/containers.nix
+                ./application/containers.nix
+                ./application/nginx.nix
+
+                ./monitoring/nginx.nix
+                ./monitoring/grafana.nix
+                ./monitoring/prometheus.nix
+                ./monitoring/loki.nix
+                ./monitoring/promtail.nix
+
+                ./system/age-configuration.nix
                 ./system/hardware-configuration.nix
                 ./system/nix-configuration.nix
                 ./system/security.nix
@@ -39,14 +49,6 @@
                 ./user/users.nix
               ];
 
-              age = {
-                secrets = {
-                  keys.file = ./secrets/keys.age;
-                  sezycei.file = ./secrets/sezycei.age;
-                };
-                identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
-              };
-
               system.stateVersion = "22.11";
             })
           ];

monitoring/grafana.nix

@@ -0,0 +1,40 @@
+{ config, pkgs, ... }: {
+  services.grafana = {
+    enable = true;
+    provision = {
+      enable = true;
+      datasources.settings.datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          access = "proxy";
+          url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+        }
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+        }
+      ];
+    };
+    settings = {
+      analytics.reporting_enabled = false;
+      server = {
+        domain = "graf.matri.cx";
+        http_addr = "127.0.0.1";
+        http_port = 7000;
+        protocol = "http";
+        root_Url = "http://192.168.0.130:7000";
+      };
+      smtp = {
+        enabled = true;
+
+        from_address = "graf@matri.cx";
+        host = "box.eversole.co:465";
+        user = "graf@matri.cx";
+        password = "$__file{${config.age.secrets.graf-email.path}}";
+      };
+    };
+  };
+}

monitoring/loki.nix

@@ -0,0 +1,76 @@
+{ config, pkgs, ... }: {
+  services.loki = {
+    enable = true;
+    configuration = {
+      server.http_listen_port = 6999;
+      auth_enabled = false;
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-06-06";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      chunk_store_config = {
+        max_look_back_period = "0s";
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki";
+        shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+    };
+  };
+
+}

monitoring/nginx.nix

@@ -0,0 +1,76 @@
+{ config, ... }:
+let
+  hostIP = "192.168.0.130";
+in
+{
+  services.nginx = {
+
+    upstreams = {
+      "grafana" = {
+        servers = {
+          "127.0.0.1:${toString config.services.grafana.settings.server.http_port}" = { };
+        };
+      };
+      "prometheus" = {
+        servers = {
+          "127.0.0.1:${toString config.services.prometheus.port}" = { };
+        };
+      };
+      "loki" = {
+        servers = {
+          "127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = { };
+        };
+      };
+      "promtail" = {
+        servers = {
+          "127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = { };
+        };
+      };
+    };
+
+    virtualHosts."graf.matri.cx" = {
+      enableACME = true;
+      forceSSL = true;
+      root = "/var/www/graf.matri.cx";
+      locations."/" = {
+        proxyPass = "http://grafana";
+        proxyWebsockets = true;
+      };
+    };
+
+    virtualHosts.grafana = {
+      locations."/" = {
+        proxyPass = "http://grafana";
+        proxyWebsockets = true;
+      };
+      listen = [{
+        addr = hostIP;
+        port = 8010;
+      }];
+    };
+
+    virtualHosts.prometheus = {
+      locations."/".proxyPass = "http://prometheus";
+      listen = [{
+        addr = hostIP;
+        port = 8020;
+      }];
+    };
+
+    virtualHosts.loki = {
+      locations."/".proxyPass = "http://loki";
+      listen = [{
+        addr = hostIP;
+        port = 8030;
+      }];
+    };
+
+    virtualHosts.promtail = {
+      locations."/".proxyPass = "http://promtail";
+      listen = [{
+        addr = hostIP;
+        port = 8031;
+      }];
+    };
+  };
+}

monitoring/prometheus.nix

@@ -0,0 +1,21 @@
+{ config, pkgs, ... }: {
+  services.prometheus = {
+    enable = true;
+    port = 7001;
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 7002;
+      };
+    };
+    scrapeConfigs = [
+      {
+        job_name = "eve-psr-nix0";
+        static_configs = [{
+          targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
+        }];
+      }
+    ];
+  };
+}

monitoring/promtail.nix

@@ -0,0 +1,31 @@
+{ config, pkgs, ... }: {
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 6998;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+      }];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "12h";
+          labels = {
+            job = "systemd-journal";
+            host = "pihole";
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+}

secrets.nix

@@ -4,7 +4,11 @@ let
   eve-psr-nix0 =
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMyaPYK0HcKAjrD1g+FPqPEU9FJ0I6+iKYmQlWKE0zHp root@matri.cx";
   all = [ james eve-psr-nix0 ];
-in {
+in
+{
+  "secrets/cache-key.age" = { publicKeys = all; };
+  "secrets/graf-email.age" = { publicKeys = all; };
+  "secrets/htpasswd-dock.age" = { publicKeys = all; };
   "secrets/keys.age" = { publicKeys = all; };
   "secrets/sezycei.age" = { publicKeys = all; };
 }

secrets/cache-key.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 dQ70Fw F07dwZssmDV+ANluboxsd1jVAe6KbJUBvsDu/ZirtSw
+YYmBoDFmDhPnirUR8T5SXVP8xac6B0+udowSlH+rOpE
+-> ssh-ed25519 ZIoeGg WlwUu7iTrMsw+RGQky440KTypGrF/tjoGvVt2ZN0Egs
+CUj6fHmhV2Ve8xZDNv7wQTgwv0IzFetRe5crHaGc2h0
+-> XM/Q-grease kQNd w 6+y
+SjjodNRi+fxX5h4wCjguJrSuFnSrN9sXjs98kovd22jLvq8YFFSZuckHV6J1gV4
+--- Bxm/LWCg2AjL/zW79Zgd+PxTwIVvPozCZ10PQ5YYtok
+!\<5C><><EFBFBD>o<EFBFBD><6F>3<EFBFBD>nd<6E>J<EFBFBD><0E>sN:Et<45>&WN<57>D<EFBFBD><44>w:(<28>W#<23>]<5D><><EFBFBD>,<2C>'<27>P'<27><>F<EFBFBD>3<EFBFBD>`B<>x<EFBFBD><78><EFBFBD><EFBFBD><EFBFBD>9<>T|<7C><><EFBFBD>-<2D>G<>4<EFBFBD>1<EFBFBD><31>Z~IW<49>,<2C><>l<EFBFBD>^\<5C><><EFBFBD>G<EFBFBD>OJ0<4A><05>0CQ<43>Q<>k<EFBFBD>0<EFBFBD><30>W<EFBFBD><57><07>)<29>cP<63>/<2F><><EFBFBD>@

secrets/cache-pub.pem

@@ -0,0 +1 @@
+cache.matri.cx:Mli2r5oPpjsvO4/v4LLcVHhDgttvCp0HCQI3EJI2HmA=

secrets/graf-email.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 dQ70Fw HLDXiLVbc4vylLhXIqDyFMNcswKAzLCybCoZoB4bknM
+vrY7LWzxqGwbVugUlOubtDcF6rm080THRjlSNaw5j+c
+-> ssh-ed25519 ZIoeGg DYX9DqVS9bJNDWy9TD8B+Uoz0KonNUymh38afBYp41o
++yzCVPC8pSxj5HAwOSCOIRwQQtZG2Zl72u7syXzNlJI
+-> YP*A'-grease DT 9Xi":v <
+qByCkYW45b+kZbCQFPQ+MgmYlQENoKU7TR5WpzjFMVCkSA
+--- 7RBQJKjysOBdJeF6VZDpqosSMblP0KaQ0mSs9lDJ9rI
+C<EFBFBD><EFBFBD>𐳺<EFBFBD><EFBFBD><EFBFBD><EFBFBD><13>`<60><><EFBFBD>_<0E>)[><3E><><0F>4(kpт<70>g<EFBFBD>±<1E>!<21>6O<36><07>=a<><07>m<EFBFBD>

secrets/htpasswd-dock.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 dQ70Fw Vby/tt+kyZ9MAFtywTC7RLrc1c5oe4ni/pBHsCOVg08
+heI1GMGvOsgcglyteG8f2au61jQtECR7gl8HyDk9uYA
+-> ssh-ed25519 ZIoeGg 7UJnIWpJpVqEKIKQgSj5mBCCpHfW79f1ffCtStFQnwE
+TMsgdn3i0qfdKQ8/9kLZ/HnkJwULw80Zob1wiPmrTCQ
+-> "O0$E-grease Vp \(Q^DS W
+V2A
+--- JkBLGnIVayeKA28hR1ldJ8+GqQkIZV8VVOHYsZl9/T8
+<EFBFBD>mE<EFBFBD><EFBFBD><EFBFBD>]"<22><<3C><><EFBFBD>QC2<43>ö9[<5B><>~ycy!Ж~ٓ<<3C>ȸ<EFBFBD><C8B8><EFBFBD>'#<23><><EFBFBD>ܾ/<2F><><03>d<EFBFBD>Ag<><1C>L<EFBFBD><4C>C<EFBFBD>

shell.nix

@@ -6,7 +6,8 @@
       pkgs.terraform
     ];
     shellHook = ''
-      alias deploy="nixos-rebuild switch --target-host root@matri.cx --build-host root@matri.cx --flake .#eve-psr-nix0"
+      deploy() { nixos-rebuild switch --target-host root@matri.cx --build-host root@matri.cx --flake .#eve-psr-nix0 ; }
+      fmt() { nix fmt *.nix; for i in */; do cd $i; nix fmt *.nix; cd ..; done ; }
     '';
   };
 }

system/age-configuration.nix

@@ -0,0 +1,17 @@
+{ ... }: {
+  age = {
+    secrets = {
+      cache-key.file = ../secrets/cache-key.age;
+      graf-email = {
+        file = ../secrets/graf-email.age;
+        mode = "770";
+        owner = "grafana";
+        group = "grafana";
+      };
+      htpasswd-dock.file = ../secrets/htpasswd-dock.age;
+      keys.file = ../secrets/keys.age;
+      sezycei.file = ../secrets/sezycei.age;
+    };
+    identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+  };
+}

system/security.nix

@@ -1 +1,9 @@
-{ ... }: { security.sudo.wheelNeedsPassword = false; }
+{ ... }: {
+  security = {
+    sudo.wheelNeedsPassword = false;
+    acme = {
+      acceptTerms = true;
+      defaults.email = "james@eversole.co";
+    };
+  };
+}

system/services.nix

@@ -1,4 +1,4 @@
-{ ... }: {
+{ config, ... }: {
   services = {
     openssh = { enable = true; };
 
@@ -18,5 +18,10 @@
       notificationSender = "hydra@matri.cx";
       buildMachinesFiles = [ ];
     };
+
+    nix-serve = {
+      enable = true;
+      secretKeyFile = config.age.secrets.cache-key.path;
+    };
   };
 }

system/system.nix

@@ -9,7 +9,7 @@
   networking = {
     hostName = "eve-psr-nix0";
     firewall = {
-      allowedTCPPorts = [ 22 80 443 3034 ];
+      allowedTCPPorts = [ 22 80 443 ];
       allowedUDPPorts = [ 22 80 443 ];
     };
   };