Initial prometheus config

Also did provision config for grafana, could be done directly but went
through skynet.grafana config

applications/grafana.nix

@@ -24,6 +24,26 @@ in {
         type = types.str;
       };
     };
+
+    ip = mkOption {
+      type = types.str;
+      default = cfg.host.ip;
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = port;
+    };
+
+    datasource = {
+      name = mkOption {
+        type = types.str;
+      };
+
+      url = mkOption {
+        type = types.str;
+      };
+    };
   };
 
   config = mkIf cfg.enable {
@@ -44,6 +64,18 @@ in {
       domain = "${name}.skynet.ie";
       port = port;
       addr = cfg.host.ip;
+
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = cfg.datasource.name;
+            type = "Prometheus";
+            url = cfg.datasource.url;
+            isDefault = true;
+          }
+        ];
+      };
     };
 
     services.nginx.virtualHosts = {

applications/prometheus.nix

@@ -0,0 +1,67 @@
+{
+  lib,
+  config,
+  ...
+}:
+with lib; let
+  name = "prometheus";
+  cfg = config.services.skynet."${name}";
+in {
+  imports = [];
+
+  options.services.skynet."${name}" = {
+    server = {
+      enable = mkEnableOption "Prometheus Server";
+      host = {
+        ip = mkOption {
+          type = types.str;
+        };
+
+        name = mkOption {
+          type = types.str;
+        };
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 9001;
+      };
+    };
+
+    collecter_port = mkOption {
+      type = types.port;
+      default = 9002;
+    };
+
+    #list of servers passed in for monitoring
+    servers = mkOption {
+      type = types.listOf types.str;
+    };
+  };
+
+  config =
+    {
+      services.prometheus.exporters.node = {
+        enable = true;
+        # most collectors are on by default see docs for more options
+        enabledCollectors = ["systemd"];
+        port = cfg.collecter_port;
+      };
+    }
+    // mkIf cfg.server.enable {
+      services.prometheus = {
+        enable = true;
+        port = cfg.server.port;
+        scrapeConfigs = [
+          {
+            job_name = "node_exporter";
+            static_configs = [
+              {
+                targets = map (server: "${server}.skynet.ie:9002") cfg.servers;
+              }
+            ];
+          }
+        ];
+      };
+    };
+}

machines/_base.nix

@@ -29,6 +29,10 @@ in {
 
     # every server will need the config to backup to
     ../applications/restic.nix
+
+    # every server will be monitored for grafana
+    ../applications/prometheus.nix
+    #TODO: make sure no additional config needed for exporters ?
   ];
 
   options.skynet = {

machines/marvin.nix

@@ -10,6 +10,7 @@ Notes:
 */
 {
   pkgs,
+  config,
   lib,
   nodes,
   ...
@@ -26,6 +27,7 @@ Notes:
 in {
   imports = [
     ../applications/grafana.nix
+    ../applications/prometheus.nix
   ];
 
   deployment = {
@@ -50,6 +52,32 @@ in {
     sudo_groups = groups;
   };
 
+  services.skynet.prometheus.server = {
+    host = {
+      ip = ip_pub;
+      name = name;
+    };
+
+    port = 9001;
+
+    servers = [
+      "agentjones"
+      "cadie"
+      "earth"
+      "galatea"
+      "gir"
+      "glados"
+      "kitt"
+      "marvin"
+      "neuromancer"
+      "optimus"
+      "skynet"
+      "vendetta"
+      "vigil"
+      "wheatly"
+    ];
+  };
+
   services.skynet.grafana = {
     enable = true;
 
@@ -57,6 +85,12 @@ in {
       ip = ip_pub;
       name = name;
     };
+
+ # maybe just do provision config directly ? 
+    datasource = {
+      name = "Prometheus";
+      url = "localhost:${toString config.services.prometheus.port}";
+    };
   };
 
   skynet_dns.records = [