Add metrics host with Grafana and Prometheus

flake.nix

@@ -10,7 +10,7 @@
 
   outputs = { self, nixpkgs, nixpkgs-unstable, nixos-generators, ... }@inputs: let
     hosts = import ./hosts.nix inputs;
-    helper = import ./helper.nix inputs; 
+    helper = import ./helper.nix inputs;
   in {
     colmena = {
       meta = {
@@ -23,11 +23,15 @@
         # The default is "nixpkgs" for "x86_64-linux" systems,
         # but it is overridden by the host-specific "hostNixpkgs" and "system" attributes.
         nodeNixpkgs = builtins.mapAttrs (name: host: host.pkgs) hosts;
+
+        specialArgs = {
+          inherit hosts;
+        };
       };
     } // builtins.mapAttrs (helper.generateColmenaHost) hosts;
 
     hydraJobs = {
-      nixConfigurations = builtins.mapAttrs (helper.generateNixConfiguration) hosts;
+      nixConfigurations = builtins.mapAttrs (host: helper.generateNixConfiguration host { inherit hosts; }) hosts;
     };
 
     # Generate a base VM image for Proxmox with `nix build .#base-proxmox`

helper.nix

@@ -14,7 +14,7 @@
     imports = modules ++ nixpkgs.lib.optional (builtins.pathExists ./hosts/${name}/secrets.nix) ./hosts/${name}/secrets.nix;
   };
 
-  generateNixConfiguration = name: {
+  generateNixConfiguration = name: specialArgs: {
     hostNixpkgs,
     system,
     modules,
@@ -26,6 +26,6 @@
     filter = name: host: (builtins.substring 0 1 name) != "_";
   in
     (nixpkgs.lib.filterAttrs filter (hostNixpkgs.lib.nixosSystem {
-      inherit system modules;
+      inherit system modules specialArgs;
     })).config.system.build.toplevel; # Builds the entire NixOS system, see: https://nixos.org/manual/nixos/stable/#sec-building-parts
 }

hosts/metrics/grafana.nix

@@ -1,11 +1,11 @@
-{ ... }:
+{ config, ... }:
 {
   services.grafana = {
     enable = true;
     settings = {
       server = {
-        domain = "grafana2.grzb.de";
+        domain = "grafana.grzb.de";
-        root_url = "https://grafana2.grzb.de";
+        root_url = "https://${config.services.grafana.settings.server.domain}";
       };
       security = {
         cookie_secure = true;
@@ -24,5 +24,13 @@
         startTLS_policy = "NoStartTLS";
       };
     };
+    provision.datasources.settings.datasources = [
+      {
+        name = "Prometheus";
+        type = "prometheus";
+        url = "http://localhost:${builtins.toString config.services.prometheus.port}";
+        isDefault = true;
+      }
+    ];
   };
 }

hosts/metrics/nginx.nix

@@ -3,7 +3,7 @@
   services.nginx = {
     enable = true;
     virtualHosts = {
-      "grafana2.grzb.de" = {
+      ${config.services.grafana.settings.server.domain} = {
         forceSSL = true;
         enableACME = true;
         listen = [
@@ -19,6 +19,7 @@
         ];
         locations."/" = {
           proxyPass = "http://${config.services.grafana.settings.server.http_addr}:${builtins.toString config.services.grafana.settings.server.http_port}";
+          proxyWebsockets = true;
         };
       };
     };

hosts/metrics/prometheus.nix

@@ -1,6 +1,19 @@
-{ ... }:
+{ hosts, ... }:
 {
   services.prometheus = {
     enable = true;
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = builtins.map (name: {
+          targets = [
+            "${name}.${hosts.${name}.site}.grzb.de:9100"
+          ];
+          labels = {
+            host = "${name}.${hosts.${name}.site}.grzb.de";
+          };
+        }) (builtins.attrNames hosts);
+      }
+    ];
   };
 }

hosts/web-public-2/virtualHosts/acme-challenge.nix

@@ -18,7 +18,7 @@
       proxyPass = "http://netbox.vs.grzb.de:80";
     };
   };
-  services.nginx.virtualHosts."grafana2.grzb.de" = {
+  services.nginx.virtualHosts."grafana.grzb.de" = {
     listen = [{ 
       addr = "0.0.0.0";
       port = 80;