From 183f5a0e7df037e0e16cbb2f417ae7b932378157 Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 06:16:35 +0100 Subject: [PATCH 01/27] Initial prometheus config Also did provision config for grafana, could be done directly but went through skynet.grafana config --- applications/grafana.nix | 32 ++++++++++++++++++ applications/prometheus.nix | 67 +++++++++++++++++++++++++++++++++++++ machines/_base.nix | 4 +++ machines/marvin.nix | 34 +++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 applications/prometheus.nix diff --git a/applications/grafana.nix b/applications/grafana.nix index e3057d1..fd06612 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -24,6 +24,26 @@ in { type = types.str; }; }; + + ip = mkOption { + type = types.str; + default = cfg.host.ip; + }; + + port = mkOption { + type = types.port; + default = port; + }; + + datasource = { + name = mkOption { + type = types.str; + }; + + url = mkOption { + type = types.str; + }; + }; }; config = mkIf cfg.enable { @@ -44,6 +64,18 @@ in { domain = "${name}.skynet.ie"; port = port; addr = cfg.host.ip; + + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = cfg.datasource.name; + type = "Prometheus"; + url = cfg.datasource.url; + isDefault = true; + } + ]; + }; }; services.nginx.virtualHosts = { diff --git a/applications/prometheus.nix b/applications/prometheus.nix new file mode 100644 index 0000000..fcda5c9 --- /dev/null +++ b/applications/prometheus.nix @@ -0,0 +1,67 @@ +{ + lib, + config, + ... +}: +with lib; let + name = "prometheus"; + cfg = config.services.skynet."${name}"; +in { + imports = []; + + options.services.skynet."${name}" = { + server = { + enable = mkEnableOption "Prometheus Server"; + host = { + ip = mkOption { + type = types.str; + }; + + name = mkOption { + type = types.str; + }; + }; + + port = mkOption { + type = types.port; + default = 9001; + }; + }; + + collecter_port = mkOption { + type = types.port; + default = 9002; + }; + + #list of servers passed in for monitoring + servers = mkOption { + type = types.listOf types.str; + }; + }; + + config = + { + services.prometheus.exporters.node = { + enable = true; + # most collectors are on by default see docs for more options + enabledCollectors = ["systemd"]; + port = cfg.collecter_port; + }; + } + // mkIf cfg.server.enable { + services.prometheus = { + enable = true; + port = cfg.server.port; + scrapeConfigs = [ + { + job_name = "node_exporter"; + static_configs = [ + { + targets = map (server: "${server}.skynet.ie:9002") cfg.servers; + } + ]; + } + ]; + }; + }; +} diff --git a/machines/_base.nix b/machines/_base.nix index d83e75b..f46eef2 100644 --- a/machines/_base.nix +++ b/machines/_base.nix @@ -29,6 +29,10 @@ in { # every server will need the config to backup to ../applications/restic.nix + + # every server will be monitored for grafana + ../applications/prometheus.nix + #TODO: make sure no additional config needed for exporters ? ]; options.skynet = { diff --git a/machines/marvin.nix b/machines/marvin.nix index fdf59b1..963ff7c 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -10,6 +10,7 @@ Notes: */ { pkgs, + config, lib, nodes, ... @@ -26,6 +27,7 @@ Notes: in { imports = [ ../applications/grafana.nix + ../applications/prometheus.nix ]; deployment = { @@ -50,6 +52,32 @@ in { sudo_groups = groups; }; + services.skynet.prometheus.server = { + host = { + ip = ip_pub; + name = name; + }; + + port = 9001; + + servers = [ + "agentjones" + "cadie" + "earth" + "galatea" + "gir" + "glados" + "kitt" + "marvin" + "neuromancer" + "optimus" + "skynet" + "vendetta" + "vigil" + "wheatly" + ]; + }; + services.skynet.grafana = { enable = true; @@ -57,6 +85,12 @@ in { ip = ip_pub; name = name; }; + + # maybe just do provision config directly ? + datasource = { + name = "Prometheus"; + url = "localhost:${toString config.services.prometheus.port}"; + }; }; skynet_dns.records = [ -- 2.46.1 From 4637777e5c9d5c532870a4a70b838f536bcdaa45 Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 07:32:25 +0100 Subject: [PATCH 02/27] Fix servers list location --- applications/prometheus.nix | 9 ++++----- machines/marvin.nix | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index fcda5c9..c673600 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -26,17 +26,16 @@ in { type = types.port; default = 9001; }; + #list of servers passed in for monitoring + servers = mkOption { + type = types.listOf types.str; + }; }; collecter_port = mkOption { type = types.port; default = 9002; }; - - #list of servers passed in for monitoring - servers = mkOption { - type = types.listOf types.str; - }; }; config = diff --git a/machines/marvin.nix b/machines/marvin.nix index 963ff7c..3f25f48 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -86,7 +86,7 @@ in { name = name; }; - # maybe just do provision config directly ? + # maybe just do provision config directly ? datasource = { name = "Prometheus"; url = "localhost:${toString config.services.prometheus.port}"; -- 2.46.1 From 7f5f21dc8a796ce836f01120df20f4ddf43ab142 Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 07:34:28 +0100 Subject: [PATCH 03/27] Use port from cfg --- applications/prometheus.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index c673600..9ae7df8 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -56,7 +56,7 @@ in { job_name = "node_exporter"; static_configs = [ { - targets = map (server: "${server}.skynet.ie:9002") cfg.servers; + targets = map (server: "${server}.skynet.ie:{collecter_port}") cfg.servers; } ]; } -- 2.46.1 From cf600e2dc10e08d5ef351e836db7a6d64e45b41f Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 18:23:56 +0100 Subject: [PATCH 04/27] Using nodes instead of hardcoded server names Might not work probably did smnth wrong --- applications/prometheus.nix | 9 +++++---- machines/marvin.nix | 19 ++----------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index 9ae7df8..dbd7275 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -10,6 +10,11 @@ in { imports = []; options.services.skynet."${name}" = { + #list of servers passed in for monitoring + servers = mkOption { + type = types.listOf types.str; + }; + server = { enable = mkEnableOption "Prometheus Server"; host = { @@ -26,10 +31,6 @@ in { type = types.port; default = 9001; }; - #list of servers passed in for monitoring - servers = mkOption { - type = types.listOf types.str; - }; }; collecter_port = mkOption { diff --git a/machines/marvin.nix b/machines/marvin.nix index 3f25f48..12465f0 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -59,25 +59,10 @@ in { }; port = 9001; - - servers = [ - "agentjones" - "cadie" - "earth" - "galatea" - "gir" - "glados" - "kitt" - "marvin" - "neuromancer" - "optimus" - "skynet" - "vendetta" - "vigil" - "wheatly" - ]; }; + services.skynet.prometheus.servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; + services.skynet.grafana = { enable = true; -- 2.46.1 From be56e6b9e90e6c3ff511ac78e454a9ff49caa0bb Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 18:32:37 +0100 Subject: [PATCH 05/27] Reorganise prometheus/marvin config --- machines/marvin.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/machines/marvin.nix b/machines/marvin.nix index 12465f0..f0c7a39 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -52,16 +52,19 @@ in { sudo_groups = groups; }; - services.skynet.prometheus.server = { + services.skynet.prometheus = { + servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; + + server = { host = { ip = ip_pub; name = name; }; port = 9001; + }; }; - services.skynet.prometheus.servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; services.skynet.grafana = { enable = true; -- 2.46.1 From 82305d43ff1c941bc22294cd106ead312d7fb412 Mon Sep 17 00:00:00 2001 From: daragh Date: Tue, 21 May 2024 18:33:52 +0100 Subject: [PATCH 06/27] fmt --- machines/marvin.nix | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/machines/marvin.nix b/machines/marvin.nix index f0c7a39..0c240aa 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -53,19 +53,18 @@ in { }; services.skynet.prometheus = { - servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; + servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; - server = { - host = { - ip = ip_pub; - name = name; - }; + server = { + host = { + ip = ip_pub; + name = name; + }; - port = 9001; + port = 9001; }; }; - services.skynet.grafana = { enable = true; -- 2.46.1 From 9b3e7265dd4e9e29a8663bfd4e5b753022012fc7 Mon Sep 17 00:00:00 2001 From: daragh Date: Wed, 22 May 2024 01:51:21 +0100 Subject: [PATCH 07/27] Added new option to specify ip and port for other nodes, fixed server.name to go through deployment.hostname --- applications/prometheus.nix | 15 +++++++++------ machines/marvin.nix | 2 -- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index dbd7275..a431e1a 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -1,4 +1,5 @@ { + nodes, lib, config, ... @@ -10,11 +11,6 @@ in { imports = []; options.services.skynet."${name}" = { - #list of servers passed in for monitoring - servers = mkOption { - type = types.listOf types.str; - }; - server = { enable = mkEnableOption "Prometheus Server"; host = { @@ -31,6 +27,13 @@ in { type = types.port; default = 9001; }; + + other_nodes = mkOption { + type = types.listOf types.str; + description = '' + To add other nodes outside of nix, specify ip and port that server should listen to here + ''; + }; }; collecter_port = mkOption { @@ -57,7 +60,7 @@ in { job_name = "node_exporter"; static_configs = [ { - targets = map (server: "${server}.skynet.ie:{collecter_port}") cfg.servers; + targets = map (hostname: "${hostname}:${collecter_port}") lib.attrsets.mapAttrsToList (server: server.deployment.hostname) nodes ++ cfg.other_nodes; } ]; } diff --git a/machines/marvin.nix b/machines/marvin.nix index 0c240aa..191f0a5 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -53,8 +53,6 @@ in { }; services.skynet.prometheus = { - servers = lib.attrsets.mapAttrsToList (server: server.name) nodes; - server = { host = { ip = ip_pub; -- 2.46.1 From ca872275712c7ea590b63dc5861ac2baafb204c0 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 01:21:02 +0100 Subject: [PATCH 08/27] remove redundant option --- applications/grafana.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index fd06612..beb66a8 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -25,11 +25,6 @@ in { }; }; - ip = mkOption { - type = types.str; - default = cfg.host.ip; - }; - port = mkOption { type = types.port; default = port; -- 2.46.1 From 113084148c209fddfedba9d3b36ad4fe2cbc091c Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 01:21:29 +0100 Subject: [PATCH 09/27] Make map clearer, more parentheses --- applications/prometheus.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index a431e1a..67d1ca0 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -60,7 +60,7 @@ in { job_name = "node_exporter"; static_configs = [ { - targets = map (hostname: "${hostname}:${collecter_port}") lib.attrsets.mapAttrsToList (server: server.deployment.hostname) nodes ++ cfg.other_nodes; + targets = (map (hostname: "${hostname}:${collecter_port}") (lib.attrsets.mapAttrsToList (server: server.deployment.hostname)) nodes) ++ cfg.other_nodes; } ]; } -- 2.46.1 From 1ea703bfa13da4bbcfa4a897ce8434c9bbb00f6a Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 01:27:02 +0100 Subject: [PATCH 10/27] Removed redundant conf, rename portcollecter --- applications/prometheus.nix | 6 +++--- machines/marvin.nix | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index 67d1ca0..e4fd5e0 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -36,7 +36,7 @@ in { }; }; - collecter_port = mkOption { + port_collecter = mkOption { type = types.port; default = 9002; }; @@ -48,7 +48,7 @@ in { enable = true; # most collectors are on by default see docs for more options enabledCollectors = ["systemd"]; - port = cfg.collecter_port; + port = cfg.port_collecter; }; } // mkIf cfg.server.enable { @@ -60,7 +60,7 @@ in { job_name = "node_exporter"; static_configs = [ { - targets = (map (hostname: "${hostname}:${collecter_port}") (lib.attrsets.mapAttrsToList (server: server.deployment.hostname)) nodes) ++ cfg.other_nodes; + targets = (map (hostname: "${hostname}:${cfg.port_collecter}") (lib.attrsets.mapAttrsToList (server: server.deployment.hostname)) nodes) ++ cfg.other_nodes; } ]; } diff --git a/machines/marvin.nix b/machines/marvin.nix index 191f0a5..e4df5dc 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -58,8 +58,6 @@ in { ip = ip_pub; name = name; }; - - port = 9001; }; }; -- 2.46.1 From 9aeb7313b47f359029b7b28e96a520615c2ef94d Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 02:10:16 +0100 Subject: [PATCH 11/27] Moved grafana / prometheus to kitt --- machines/kitt.nix | 26 ++++++++++++++++++++++++++ machines/marvin.nix | 26 -------------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/machines/kitt.nix b/machines/kitt.nix index 5891571..88f383c 100644 --- a/machines/kitt.nix +++ b/machines/kitt.nix @@ -9,6 +9,7 @@ Role: LDAP Server Notes: */ { + config, pkgs, lib, nodes, @@ -25,6 +26,8 @@ in { ../applications/discord.nix ../applications/bitwarden/vaultwarden.nix ../applications/bitwarden/bitwarden_sync.nix + ../applications/grafana.nix + ../applications/prometheus.nix ]; deployment = { @@ -77,4 +80,27 @@ in { name = name; }; }; + services.skynet.prometheus = { + server = { + host = { + ip = ip_pub; + name = name; + }; + }; + }; + + services.skynet.grafana = { + enable = true; + + host = { + ip = ip_pub; + name = name; + }; + + # maybe just do provision config directly ? + datasource = { + name = "Prometheus"; + url = "localhost:${toString config.services.prometheus.port}"; + }; + }; } diff --git a/machines/marvin.nix b/machines/marvin.nix index e4df5dc..3bb6d1e 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -26,8 +26,6 @@ Notes: groups_trusted = map (x: "@${x}") groups; in { imports = [ - ../applications/grafana.nix - ../applications/prometheus.nix ]; deployment = { @@ -52,30 +50,6 @@ in { sudo_groups = groups; }; - services.skynet.prometheus = { - server = { - host = { - ip = ip_pub; - name = name; - }; - }; - }; - - services.skynet.grafana = { - enable = true; - - host = { - ip = ip_pub; - name = name; - }; - - # maybe just do provision config directly ? - datasource = { - name = "Prometheus"; - url = "localhost:${toString config.services.prometheus.port}"; - }; - }; - skynet_dns.records = [ { record = name; -- 2.46.1 From fd3beade9b94c43b9a25ca0247219aaa1050a901 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 02:13:06 +0100 Subject: [PATCH 12/27] Added entry in secrets.nix for grafana --- secrets/secrets.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 7fa8397..6c865b3 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -69,6 +69,10 @@ let wheatly ]; + grafana = [ + kitt + ]; + # these need dns stuff webservers = [ @@ -150,4 +154,7 @@ in { "bitwarden/id.age".publicKeys = users ++ bitwarden; "bitwarden/secret.age".publicKeys = users ++ bitwarden; "bitwarden/details.age".publicKeys = users ++ bitwarden; + + # grafana + "grafana/pw.age".publicKeys = users++ grafana; } -- 2.46.1 From 40e4fe5ac487aab5e3f2d05d63dd9633d9cad675 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 02:13:14 +0100 Subject: [PATCH 13/27] fmt --- secrets/secrets.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 6c865b3..0ba160e 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -70,7 +70,7 @@ let ]; grafana = [ - kitt + kitt ]; # these need dns stuff @@ -156,5 +156,5 @@ in { "bitwarden/details.age".publicKeys = users ++ bitwarden; # grafana - "grafana/pw.age".publicKeys = users++ grafana; + "grafana/pw.age".publicKeys = users ++ grafana; } -- 2.46.1 From 23f77caef60dc506217631b99975d83e0af31301 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 02:39:36 +0100 Subject: [PATCH 14/27] feat: setup the password for grafana --- applications/grafana.nix | 4 ++++ secrets/grafana/pw.age | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 secrets/grafana/pw.age diff --git a/applications/grafana.nix b/applications/grafana.nix index beb66a8..84af996 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -54,12 +54,16 @@ in { "${name}.skynet.ie" ]; + age.secrets.grafana_pw.file = ../secrets/grafana/pw.age; + services.grafana = { enable = true; domain = "${name}.skynet.ie"; port = port; addr = cfg.host.ip; + settings.security.admin_password = "$__file{${config.age.secrets.grafana_pw.path}}"; + provision = { enable = true; datasources.settings.datasources = [ diff --git a/secrets/grafana/pw.age b/secrets/grafana/pw.age new file mode 100644 index 0000000..6a01432 --- /dev/null +++ b/secrets/grafana/pw.age @@ -0,0 +1,13 @@ +age-encryption.org/v1 +-> ssh-ed25519 V1pwNA ly/9CnXtgQlXTbKcK+gD+v0Ck7rmGtNrA/S9XfBdg3s +6skVNVJTgCf/EWlDbH6urfr4CUibVH/N+HcfIYPkzTo +-> ssh-ed25519 4PzZog 7+Fc9ec8zvlKP6VGKJa3MRN6p9bUrA07/BlL8rSnp3w +YgALG1b8QOmMqWuqr9iVxAal9cWFf8me0KT1Mg0onko +-> ssh-ed25519 5Nd93w /lx/evI9jsXzHMxXYQMoavWucTMiGMXwxACpjXYFZlU +nVWhQydOO8eaTYcR66u1MeH/glmwTDJnJM0I9tXUvV0 +-> ssh-ed25519 q8eJgg wYOxbUUXrTgY9XkUz02qtW8TaYJfNej9VBdwvfUWrT8 +/47DLKQGt1M3fJWDHo2Eg2ij4jCGd17ieYZ8gA/uYjY +-> ssh-ed25519 IzAMqA FfUA/kyLBOFIHFUO+PSsdTwaRjGvfsq7OTMXYo7/WjM +jEn8y+mncrOPmDzvsK90X2D/m8ZxmuIL8H0h27YP3hM +--- ibLXLaT49j/Mb8CwbcL+Gjwy5GJ5YDX31JQFqfOIXRw +ag9 aYҍ䔁GADgi^UaFY@4> *?Ʉ5F-8 \ No newline at end of file -- 2.46.1 From 061453e5d19b1bdc0f95a4c29b6f159661963600 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 02:44:39 +0100 Subject: [PATCH 15/27] remove dead code --- applications/grafana.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index 84af996..ca13ff4 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -25,11 +25,6 @@ in { }; }; - port = mkOption { - type = types.port; - default = port; - }; - datasource = { name = mkOption { type = types.str; -- 2.46.1 From 03ae1c5101558dfefea4fd21f079ce1e15bf4484 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 02:57:10 +0100 Subject: [PATCH 16/27] Remove config from marvin --- machines/marvin.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/machines/marvin.nix b/machines/marvin.nix index 3bb6d1e..1c4f57b 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -10,7 +10,6 @@ Notes: */ { pkgs, - config, lib, nodes, ... -- 2.46.1 From 0f75f119184c6fc351b4369471a260d2df9b58ce Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:07:07 +0100 Subject: [PATCH 17/27] fix: this was blocking teh web interface --- applications/grafana.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index ca13ff4..6d27d40 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -55,7 +55,6 @@ in { enable = true; domain = "${name}.skynet.ie"; port = port; - addr = cfg.host.ip; settings.security.admin_password = "$__file{${config.age.secrets.grafana_pw.path}}"; -- 2.46.1 From aba1a41d4df2a598dc4f672f3bfe84f0dd43bd86 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:07:19 +0100 Subject: [PATCH 18/27] fix: file permissions --- applications/grafana.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index 6d27d40..a1c67b7 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -49,7 +49,11 @@ in { "${name}.skynet.ie" ]; - age.secrets.grafana_pw.file = ../secrets/grafana/pw.age; + age.secrets.grafana_pw = { + file = ../secrets/grafana/pw.age; + owner = "grafana"; + group = "grafana"; + }; services.grafana = { enable = true; -- 2.46.1 From 62ead11aada617e5ce98b5c0313ee4a8949e0dae Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:07:37 +0100 Subject: [PATCH 19/27] fix: have to actually enable it --- machines/kitt.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/kitt.nix b/machines/kitt.nix index 88f383c..04d450c 100644 --- a/machines/kitt.nix +++ b/machines/kitt.nix @@ -82,6 +82,7 @@ in { }; services.skynet.prometheus = { server = { + enable = true; host = { ip = ip_pub; name = name; -- 2.46.1 From 15271c1d09b73270b446f27fe92bb5a9d8403dcf Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:08:50 +0100 Subject: [PATCH 20/27] fix: this does need a default --- applications/prometheus.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index e4fd5e0..207863b 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -30,6 +30,7 @@ in { other_nodes = mkOption { type = types.listOf types.str; + default = []; description = '' To add other nodes outside of nix, specify ip and port that server should listen to here ''; -- 2.46.1 From b8c6e153a4f124fc88b43e7da7b1e84d54c6757f Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:10:15 +0100 Subject: [PATCH 21/27] fix: set the type of protocol --- machines/kitt.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/kitt.nix b/machines/kitt.nix index 04d450c..6d8eca5 100644 --- a/machines/kitt.nix +++ b/machines/kitt.nix @@ -101,7 +101,7 @@ in { # maybe just do provision config directly ? datasource = { name = "Prometheus"; - url = "localhost:${toString config.services.prometheus.port}"; + url = "http://localhost:${toString config.services.skynet.prometheus.server.port}"; }; }; } -- 2.46.1 From 9148963c1f6d3ee6569d084737247450d0863b70 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 04:20:44 +0100 Subject: [PATCH 22/27] fix: final set of changes to get it working --- applications/grafana.nix | 3 ++- applications/prometheus.nix | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index a1c67b7..4e42f81 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -67,9 +67,10 @@ in { datasources.settings.datasources = [ { name = cfg.datasource.name; - type = "Prometheus"; + type = "prometheus"; url = cfg.datasource.url; isDefault = true; + editable = true; } ]; }; diff --git a/applications/prometheus.nix b/applications/prometheus.nix index 207863b..7c53f77 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -61,7 +61,7 @@ in { job_name = "node_exporter"; static_configs = [ { - targets = (map (hostname: "${hostname}:${cfg.port_collecter}") (lib.attrsets.mapAttrsToList (server: server.deployment.hostname)) nodes) ++ cfg.other_nodes; + targets = (lib.attrsets.mapAttrsToList (key: value: "${value.config.deployment.targetHost}:${toString cfg.port_collecter}") nodes) ++ cfg.server.other_nodes; } ]; } -- 2.46.1 From 963a189bcbaa93177df7c6bf21d6b4730d4b7a63 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 04:34:19 +0100 Subject: [PATCH 23/27] Removed provision config away from kitt --- applications/grafana.nix | 4 ++-- machines/kitt.nix | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/applications/grafana.nix b/applications/grafana.nix index 4e42f81..be8e948 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -66,9 +66,9 @@ in { enable = true; datasources.settings.datasources = [ { - name = cfg.datasource.name; + name = "Prometheus"; type = "prometheus"; - url = cfg.datasource.url; + url = "http://localhost:${toString config.services.skynet.prometheus.server.port}"; isDefault = true; editable = true; } diff --git a/machines/kitt.nix b/machines/kitt.nix index 6d8eca5..f036fe0 100644 --- a/machines/kitt.nix +++ b/machines/kitt.nix @@ -97,11 +97,5 @@ in { ip = ip_pub; name = name; }; - - # maybe just do provision config directly ? - datasource = { - name = "Prometheus"; - url = "http://localhost:${toString config.services.skynet.prometheus.server.port}"; - }; }; } -- 2.46.1 From 147bd86ad56dbd9363eea6a6b95f17092cd48ac0 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 21:48:23 +0100 Subject: [PATCH 24/27] fix: get the attributes merging correctly --- applications/prometheus.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index 7c53f77..cd0dfcc 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -43,7 +43,7 @@ in { }; }; - config = + config = mkMerge [ { services.prometheus.exporters.node = { enable = true; @@ -52,7 +52,7 @@ in { port = cfg.port_collecter; }; } - // mkIf cfg.server.enable { + (mkIf cfg.server.enable { services.prometheus = { enable = true; port = cfg.server.port; @@ -67,5 +67,6 @@ in { } ]; }; - }; + }) + ]; } -- 2.46.1 From f7dd90e92ba331a40c03f7bfc25a7888da5324ca Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Thu, 23 May 2024 22:04:15 +0100 Subject: [PATCH 25/27] fix: needed to open teh ports to be able to get the data --- applications/prometheus.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index cd0dfcc..76281e3 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -51,6 +51,9 @@ in { enabledCollectors = ["systemd"]; port = cfg.port_collecter; }; + + # make sure the port is open + networking.firewall.allowedTCPPorts = [cfg.port_collecter]; } (mkIf cfg.server.enable { services.prometheus = { -- 2.46.1 From 889bb0dab6cb03db010e4aebce3bdcde70778ea0 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 22:34:02 +0100 Subject: [PATCH 26/27] doc: added link to node exporter options --- applications/prometheus.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/prometheus.nix b/applications/prometheus.nix index 76281e3..eb15c48 100644 --- a/applications/prometheus.nix +++ b/applications/prometheus.nix @@ -47,7 +47,7 @@ in { { services.prometheus.exporters.node = { enable = true; - # most collectors are on by default see docs for more options + # most collectors are on by default see https://github.com/prometheus/node_exporter for more options enabledCollectors = ["systemd"]; port = cfg.port_collecter; }; -- 2.46.1 From c0816ccce43bbe6b18ac6e43787c1f5e23f30bb9 Mon Sep 17 00:00:00 2001 From: daragh Date: Thu, 23 May 2024 22:35:31 +0100 Subject: [PATCH 27/27] remove todo --- machines/_base.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/machines/_base.nix b/machines/_base.nix index f46eef2..63acbb5 100644 --- a/machines/_base.nix +++ b/machines/_base.nix @@ -32,7 +32,6 @@ in { # every server will be monitored for grafana ../applications/prometheus.nix - #TODO: make sure no additional config needed for exporters ? ]; options.skynet = { -- 2.46.1