From 183f5a0e7df037e0e16cbb2f417ae7b932378157 Mon Sep 17 00:00:00 2001
From: daragh <daraghdownes12@gmail.com>
Date: Tue, 21 May 2024 06:16:35 +0100
Subject: [PATCH] Initial prometheus config Also did provision config for
 grafana, could be done directly but went through skynet.grafana config

---
 applications/grafana.nix    | 32 ++++++++++++++++++
 applications/prometheus.nix | 67 +++++++++++++++++++++++++++++++++++++
 machines/_base.nix          |  4 +++
 machines/marvin.nix         | 34 +++++++++++++++++++
 4 files changed, 137 insertions(+)
 create mode 100644 applications/prometheus.nix

diff --git a/applications/grafana.nix b/applications/grafana.nix
index e3057d1..fd06612 100644
--- a/applications/grafana.nix
+++ b/applications/grafana.nix
@@ -24,6 +24,26 @@ in {
         type = types.str;
       };
     };
+
+    ip = mkOption {
+      type = types.str;
+      default = cfg.host.ip;
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = port;
+    };
+
+    datasource = {
+      name = mkOption {
+        type = types.str;
+      };
+
+      url = mkOption {
+        type = types.str;
+      };
+    };
   };
 
   config = mkIf cfg.enable {
@@ -44,6 +64,18 @@ in {
       domain = "${name}.skynet.ie";
       port = port;
       addr = cfg.host.ip;
+
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = cfg.datasource.name;
+            type = "Prometheus";
+            url = cfg.datasource.url;
+            isDefault = true;
+          }
+        ];
+      };
     };
 
     services.nginx.virtualHosts = {
diff --git a/applications/prometheus.nix b/applications/prometheus.nix
new file mode 100644
index 0000000..fcda5c9
--- /dev/null
+++ b/applications/prometheus.nix
@@ -0,0 +1,67 @@
+{
+  lib,
+  config,
+  ...
+}:
+with lib; let
+  name = "prometheus";
+  cfg = config.services.skynet."${name}";
+in {
+  imports = [];
+
+  options.services.skynet."${name}" = {
+    server = {
+      enable = mkEnableOption "Prometheus Server";
+      host = {
+        ip = mkOption {
+          type = types.str;
+        };
+
+        name = mkOption {
+          type = types.str;
+        };
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 9001;
+      };
+    };
+
+    collecter_port = mkOption {
+      type = types.port;
+      default = 9002;
+    };
+
+    #list of servers passed in for monitoring
+    servers = mkOption {
+      type = types.listOf types.str;
+    };
+  };
+
+  config =
+    {
+      services.prometheus.exporters.node = {
+        enable = true;
+        # most collectors are on by default see docs for more options
+        enabledCollectors = ["systemd"];
+        port = cfg.collecter_port;
+      };
+    }
+    // mkIf cfg.server.enable {
+      services.prometheus = {
+        enable = true;
+        port = cfg.server.port;
+        scrapeConfigs = [
+          {
+            job_name = "node_exporter";
+            static_configs = [
+              {
+                targets = map (server: "${server}.skynet.ie:9002") cfg.servers;
+              }
+            ];
+          }
+        ];
+      };
+    };
+}
diff --git a/machines/_base.nix b/machines/_base.nix
index d83e75b..f46eef2 100644
--- a/machines/_base.nix
+++ b/machines/_base.nix
@@ -29,6 +29,10 @@ in {
 
     # every server will need the config to backup to
     ../applications/restic.nix
+
+    # every server will be monitored for grafana
+    ../applications/prometheus.nix
+    #TODO: make sure no additional config needed for exporters ?
   ];
 
   options.skynet = {
diff --git a/machines/marvin.nix b/machines/marvin.nix
index fdf59b1..963ff7c 100644
--- a/machines/marvin.nix
+++ b/machines/marvin.nix
@@ -10,6 +10,7 @@ Notes:
 */
 {
   pkgs,
+  config,
   lib,
   nodes,
   ...
@@ -26,6 +27,7 @@ Notes:
 in {
   imports = [
     ../applications/grafana.nix
+    ../applications/prometheus.nix
   ];
 
   deployment = {
@@ -50,6 +52,32 @@ in {
     sudo_groups = groups;
   };
 
+  services.skynet.prometheus.server = {
+    host = {
+      ip = ip_pub;
+      name = name;
+    };
+
+    port = 9001;
+
+    servers = [
+      "agentjones"
+      "cadie"
+      "earth"
+      "galatea"
+      "gir"
+      "glados"
+      "kitt"
+      "marvin"
+      "neuromancer"
+      "optimus"
+      "skynet"
+      "vendetta"
+      "vigil"
+      "wheatly"
+    ];
+  };
+
   services.skynet.grafana = {
     enable = true;
 
@@ -57,6 +85,12 @@ in {
       ip = ip_pub;
       name = name;
     };
+
+ # maybe just do provision config directly ? 
+    datasource = {
+      name = "Prometheus";
+      url = "localhost:${toString config.services.prometheus.port}";
+    };
   };
 
   skynet_dns.records = [