applications/grafana.nix

@@ -24,6 +24,16 @@ in {
         type = types.str;
       };
     };
+
+    datasource = {
+      name = mkOption {
+        type = types.str;
+      };
+
+      url = mkOption {
+        type = types.str;
+      };
+    };
   };
 
   config = mkIf cfg.enable {
@@ -39,11 +49,31 @@ in {
       "${name}.skynet.ie"
     ];
 
+    age.secrets.grafana_pw = {
+      file = ../secrets/grafana/pw.age;
+      owner = "grafana";
+      group = "grafana";
+    };
+
     services.grafana = {
       enable = true;
       domain = "${name}.skynet.ie";
       port = port;
-      addr = cfg.host.ip;
+
+      settings.security.admin_password = "$__file{${config.age.secrets.grafana_pw.path}}";
+
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://localhost:${toString config.services.skynet.prometheus.server.port}";
+            isDefault = true;
+            editable = true;
+          }
+        ];
+      };
     };
 
     services.nginx.virtualHosts = {

applications/prometheus.nix

@@ -0,0 +1,75 @@
+{
+  nodes,
+  lib,
+  config,
+  ...
+}:
+with lib; let
+  name = "prometheus";
+  cfg = config.services.skynet."${name}";
+in {
+  imports = [];
+
+  options.services.skynet."${name}" = {
+    server = {
+      enable = mkEnableOption "Prometheus Server";
+      host = {
+        ip = mkOption {
+          type = types.str;
+        };
+
+        name = mkOption {
+          type = types.str;
+        };
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 9001;
+      };
+
+      other_nodes = mkOption {
+        type = types.listOf types.str;
+        default = [];
+        description = ''
+          To add other nodes outside of nix, specify ip and port that server should listen to here
+        '';
+      };
+    };
+
+    port_collecter = mkOption {
+      type = types.port;
+      default = 9002;
+    };
+  };
+
+  config = mkMerge [
+    {
+      services.prometheus.exporters.node = {
+        enable = true;
+        # most collectors are on by default see https://github.com/prometheus/node_exporter for more options
+        enabledCollectors = ["systemd"];
+        port = cfg.port_collecter;
+      };
+
+      # make sure the port is open
+      networking.firewall.allowedTCPPorts = [cfg.port_collecter];
+    }
+    (mkIf cfg.server.enable {
+      services.prometheus = {
+        enable = true;
+        port = cfg.server.port;
+        scrapeConfigs = [
+          {
+            job_name = "node_exporter";
+            static_configs = [
+              {
+                targets = (lib.attrsets.mapAttrsToList (key: value: "${value.config.deployment.targetHost}:${toString cfg.port_collecter}") nodes) ++ cfg.server.other_nodes;
+              }
+            ];
+          }
+        ];
+      };
+    })
+  ];
+}

machines/_base.nix

@@ -29,6 +29,9 @@ in {
 
     # every server will need the config to backup to
     ../applications/restic.nix
+
+    # every server will be monitored for grafana
+    ../applications/prometheus.nix
   ];
 
   options.skynet = {

machines/kitt.nix

@@ -9,6 +9,7 @@ Role:     LDAP Server
 Notes:
 */
 {
+  config,
   pkgs,
   lib,
   nodes,
@@ -25,6 +26,8 @@ in {
     ../applications/discord.nix
     ../applications/bitwarden/vaultwarden.nix
     ../applications/bitwarden/bitwarden_sync.nix
+    ../applications/grafana.nix
+    ../applications/prometheus.nix
   ];
 
   deployment = {
@@ -72,6 +75,24 @@ in {
   services.skynet_vaultwarden = {
     enable = true;
 
+    host = {
+      ip = ip_pub;
+      name = name;
+    };
+  };
+  services.skynet.prometheus = {
+    server = {
+      enable = true;
+      host = {
+        ip = ip_pub;
+        name = name;
+      };
+    };
+  };
+
+  services.skynet.grafana = {
+    enable = true;
+
     host = {
       ip = ip_pub;
       name = name;

machines/marvin.nix

@@ -25,7 +25,6 @@ Notes:
   groups_trusted = map (x: "@${x}") groups;
 in {
   imports = [
-    ../applications/grafana.nix
   ];
 
   deployment = {
@@ -50,15 +49,6 @@ in {
     sudo_groups = groups;
   };
 
-  services.skynet.grafana = {
-    enable = true;
-
-    host = {
-      ip = ip_pub;
-      name = name;
-    };
-  };
-
   skynet_dns.records = [
     {
       record = name;

secrets/grafana/pw.age

@@ -0,0 +1,13 @@
+age-encryption.org/v1
+-> ssh-ed25519 V1pwNA ly/9CnXtgQlXTbKcK+gD+v0Ck7rmGtNrA/S9XfBdg3s
+6skVNVJTgCf/EWlDbH6urfr4CUibVH/N+HcfIYPkzTo
+-> ssh-ed25519 4PzZog 7+Fc9ec8zvlKP6VGKJa3MRN6p9bUrA07/BlL8rSnp3w
+YgALG1b8QOmMqWuqr9iVxAal9cWFf8me0KT1Mg0onko
+-> ssh-ed25519 5Nd93w /lx/evI9jsXzHMxXYQMoavWucTMiGMXwxACpjXYFZlU
+nVWhQydOO8eaTYcR66u1MeH/glmwTDJnJM0I9tXUvV0
+-> ssh-ed25519 q8eJgg wYOxbUUXrTgY9XkUz02qtW8TaYJfNej9VBdwvfUWrT8
+/47DLKQGt1M3fJWDHo2Eg2ij4jCGd17ieYZ8gA/uYjY
+-> ssh-ed25519 IzAMqA FfUA/kyLBOFIHFUO+PSsdTwaRjGvfsq7OTMXYo7/WjM
+jEn8y+mncrOPmDzvsK90X2D/m8ZxmuIL8H0h27YP3hM
+--- ibLXLaT49j/Mb8CwbcL+Gjwy5GJ5YDX31JQFqfOIXRw
+ºôag9Òa“Yâ«Ò<C2AB>öä”<C3A4>GADóðgûÅi°^ýUaß±Fà YÏã@4>
<01>¬óÐàò£Ý*‚Š?úÉ„5»F-íã8à 

secrets/secrets.nix

@@ -69,6 +69,10 @@ let
     wheatly
   ];
 
+  grafana = [
+    kitt
+  ];
+
   # these need dns stuff
   webservers =
     [
@@ -150,4 +154,7 @@ in {
   "bitwarden/id.age".publicKeys = users ++ bitwarden;
   "bitwarden/secret.age".publicKeys = users ++ bitwarden;
   "bitwarden/details.age".publicKeys = users ++ bitwarden;
+
+  # grafana
+  "grafana/pw.age".publicKeys = users ++ grafana;
 }