diff --git a/applications/grafana.nix b/applications/grafana.nix index e3057d1..be8e948 100644 --- a/applications/grafana.nix +++ b/applications/grafana.nix @@ -24,6 +24,16 @@ in { type = types.str; }; }; + + datasource = { + name = mkOption { + type = types.str; + }; + + url = mkOption { + type = types.str; + }; + }; }; config = mkIf cfg.enable { @@ -39,11 +49,31 @@ in { "${name}.skynet.ie" ]; + age.secrets.grafana_pw = { + file = ../secrets/grafana/pw.age; + owner = "grafana"; + group = "grafana"; + }; + services.grafana = { enable = true; domain = "${name}.skynet.ie"; port = port; - addr = cfg.host.ip; + + settings.security.admin_password = "$__file{${config.age.secrets.grafana_pw.path}}"; + + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:${toString config.services.skynet.prometheus.server.port}"; + isDefault = true; + editable = true; + } + ]; + }; }; services.nginx.virtualHosts = { diff --git a/applications/prometheus.nix b/applications/prometheus.nix new file mode 100644 index 0000000..eb15c48 --- /dev/null +++ b/applications/prometheus.nix @@ -0,0 +1,75 @@ +{ + nodes, + lib, + config, + ... +}: +with lib; let + name = "prometheus"; + cfg = config.services.skynet."${name}"; +in { + imports = []; + + options.services.skynet."${name}" = { + server = { + enable = mkEnableOption "Prometheus Server"; + host = { + ip = mkOption { + type = types.str; + }; + + name = mkOption { + type = types.str; + }; + }; + + port = mkOption { + type = types.port; + default = 9001; + }; + + other_nodes = mkOption { + type = types.listOf types.str; + default = []; + description = '' + To add other nodes outside of nix, specify ip and port that server should listen to here + ''; + }; + }; + + port_collecter = mkOption { + type = types.port; + default = 9002; + }; + }; + + config = mkMerge [ + { + services.prometheus.exporters.node = { + enable = true; + # most collectors are on by default see https://github.com/prometheus/node_exporter for more options + enabledCollectors = ["systemd"]; + port = cfg.port_collecter; + }; + + # make sure the port is open + networking.firewall.allowedTCPPorts = [cfg.port_collecter]; + } + (mkIf cfg.server.enable { + services.prometheus = { + enable = true; + port = cfg.server.port; + scrapeConfigs = [ + { + job_name = "node_exporter"; + static_configs = [ + { + targets = (lib.attrsets.mapAttrsToList (key: value: "${value.config.deployment.targetHost}:${toString cfg.port_collecter}") nodes) ++ cfg.server.other_nodes; + } + ]; + } + ]; + }; + }) + ]; +} diff --git a/machines/_base.nix b/machines/_base.nix index d83e75b..63acbb5 100644 --- a/machines/_base.nix +++ b/machines/_base.nix @@ -29,6 +29,9 @@ in { # every server will need the config to backup to ../applications/restic.nix + + # every server will be monitored for grafana + ../applications/prometheus.nix ]; options.skynet = { diff --git a/machines/kitt.nix b/machines/kitt.nix index 5891571..f036fe0 100644 --- a/machines/kitt.nix +++ b/machines/kitt.nix @@ -9,6 +9,7 @@ Role: LDAP Server Notes: */ { + config, pkgs, lib, nodes, @@ -25,6 +26,8 @@ in { ../applications/discord.nix ../applications/bitwarden/vaultwarden.nix ../applications/bitwarden/bitwarden_sync.nix + ../applications/grafana.nix + ../applications/prometheus.nix ]; deployment = { @@ -72,6 +75,24 @@ in { services.skynet_vaultwarden = { enable = true; + host = { + ip = ip_pub; + name = name; + }; + }; + services.skynet.prometheus = { + server = { + enable = true; + host = { + ip = ip_pub; + name = name; + }; + }; + }; + + services.skynet.grafana = { + enable = true; + host = { ip = ip_pub; name = name; diff --git a/machines/marvin.nix b/machines/marvin.nix index fdf59b1..1c4f57b 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -25,7 +25,6 @@ Notes: groups_trusted = map (x: "@${x}") groups; in { imports = [ - ../applications/grafana.nix ]; deployment = { @@ -50,15 +49,6 @@ in { sudo_groups = groups; }; - services.skynet.grafana = { - enable = true; - - host = { - ip = ip_pub; - name = name; - }; - }; - skynet_dns.records = [ { record = name; diff --git a/secrets/grafana/pw.age b/secrets/grafana/pw.age new file mode 100644 index 0000000..6a01432 --- /dev/null +++ b/secrets/grafana/pw.age @@ -0,0 +1,13 @@ +age-encryption.org/v1 +-> ssh-ed25519 V1pwNA ly/9CnXtgQlXTbKcK+gD+v0Ck7rmGtNrA/S9XfBdg3s +6skVNVJTgCf/EWlDbH6urfr4CUibVH/N+HcfIYPkzTo +-> ssh-ed25519 4PzZog 7+Fc9ec8zvlKP6VGKJa3MRN6p9bUrA07/BlL8rSnp3w +YgALG1b8QOmMqWuqr9iVxAal9cWFf8me0KT1Mg0onko +-> ssh-ed25519 5Nd93w /lx/evI9jsXzHMxXYQMoavWucTMiGMXwxACpjXYFZlU +nVWhQydOO8eaTYcR66u1MeH/glmwTDJnJM0I9tXUvV0 +-> ssh-ed25519 q8eJgg wYOxbUUXrTgY9XkUz02qtW8TaYJfNej9VBdwvfUWrT8 +/47DLKQGt1M3fJWDHo2Eg2ij4jCGd17ieYZ8gA/uYjY +-> ssh-ed25519 IzAMqA FfUA/kyLBOFIHFUO+PSsdTwaRjGvfsq7OTMXYo7/WjM +jEn8y+mncrOPmDzvsK90X2D/m8ZxmuIL8H0h27YP3hM +--- ibLXLaT49j/Mb8CwbcL+Gjwy5GJ5YDX31JQFqfOIXRw +ag9 aYҍ䔁GADgi^UaFY@4> *?Ʉ5F-8 \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 7fa8397..0ba160e 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -69,6 +69,10 @@ let wheatly ]; + grafana = [ + kitt + ]; + # these need dns stuff webservers = [ @@ -150,4 +154,7 @@ in { "bitwarden/id.age".publicKeys = users ++ bitwarden; "bitwarden/secret.age".publicKeys = users ++ bitwarden; "bitwarden/details.age".publicKeys = users ++ bitwarden; + + # grafana + "grafana/pw.age".publicKeys = users ++ grafana; }