From 2c5938437d325c17f605fb869b247dc3b9b7c84e Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Mon, 1 Sep 2025 09:42:47 +0100 Subject: [PATCH 1/9] feat: packaged up the main application --- applications/itd/splunk/package.nix | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 applications/itd/splunk/package.nix diff --git a/applications/itd/splunk/package.nix b/applications/itd/splunk/package.nix new file mode 100644 index 0000000..aa5db9c --- /dev/null +++ b/applications/itd/splunk/package.nix @@ -0,0 +1,43 @@ +{ + stdenv, + lib, + fetchurl, + #, alsa-lib + openssl, + zlib, + autoPatchelfHook, +}: +stdenv.mkDerivation rec { + pname = "splunk-forwarder"; + version = "9.1.7"; + version_long = "${version}-e17104057ef0"; + + src = fetchurl { + url = "https://download.splunk.com/products/universalforwarder/releases/${version}/linux/splunkforwarder-${version_long}-Linux-x86_64.tgz"; + hash = "sha256-EWpfenEhExe5qDscqb7ZQUwahYgAgPQ+APLlweRQAoc="; + }; + + nativeBuildInputs = [autoPatchelfHook]; + + buildInputs = [ + # alsaLib + openssl + zlib + ]; + + sourceRoot = "."; + + installPhase = '' + runHook preInstall + # gets unzipped and we just pop it all into teh output + # might add the patching of the path vars in the scripts to be here + cp -R ./splunkforwarder $out + runHook postInstall + ''; + + meta = with lib; { + homepage = "https://splunk.com"; + description = "Logging Service"; + platforms = platforms.linux; + }; +} From d2712117e5656ab78b830e47a247631da90bd049 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Mon, 1 Sep 2025 09:43:41 +0100 Subject: [PATCH 2/9] feat: module in current state, not fully complete yet but working well enough --- applications/itd/splunk/module.nix | 165 +++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 applications/itd/splunk/module.nix diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix new file mode 100644 index 0000000..e186929 --- /dev/null +++ b/applications/itd/splunk/module.nix @@ -0,0 +1,165 @@ +{ + config, + pkgs, + lib, + ... +}: +with lib; let + name = "splunk"; + cfg = config.services.itd."${name}"; + + package = pkgs.callPackage ./package.nix {}; +in { + imports = []; + + options.services.itd."${name}" = { + enable = mkEnableOption "ITD Splunk"; + + user = { + user = mkOption { + type = types.str; + default = "splunk"; + }; + group = mkOption { + type = types.str; + default = "splunk"; + }; + + home = mkOption { + type = types.str; + default = "/etc/itd/splunk"; + }; + }; + }; + + config = mkIf cfg.enable { + # create user and group for the program + users.groups."${cfg.user.group}" = {}; + users.users."${cfg.user.user}" = { + createHome = true; + isNormalUser = true; + home = cfg.user.home; + group = cfg.user.group; + # X11 is to ensure the directory can be traversed + homeMode = "711"; + }; + + # might not be required + # networking.firewall.allowedTCPPorts = [8089]; + + # set up the core files + systemd.services."${name}_prestart" = { + wantedBy = [ + # "gitlab.target" + ]; + partOf = [ + # "gitlab.target" + ]; + path = with pkgs; [util-linux]; + serviceConfig = { + Type = "oneshot"; + User = "root"; + Group = "root"; + # User = cfg.user.user; + # Group = cfg.user.group; + # TimeoutSec = "infinity"; + # Restart = "on-failure"; + # WorkingDirectory = "${cfg.package}/share/gitlab"; + # Slice = "system-gitlab.slice"; + RemainAfterExit = true; + ExecStart = let + # from $SPLUNK_HOME/etc/splunk-launch.conf.default + splunk_launch = pkgs.writeText "splunk-launch" '' + SPLUNK_HOME=${cfg.user.home} + SPLUNK_SERVER_NAME=SplunkForwarder + PYTHONHTTPSVERIFY=0 + PYTHONUTF8=1 + ''; + + etc_passwd = pkgs.writeText "passwd" '' + :admin:$6$XOFhpq0Y1Ul1pejO$DUkXU6xSAKYrl3nLTPFS7LJqlnWC4r97SSikjCzv7XNEG6lgP32TZHyLDY/aJs8ZYyblZ5spfmnlMvh8Lxndi/::Administrator:admin:changeme@example.com:::20329 + ''; + + deployment_client = pkgs.writeText "deploymentclient" '' + [deployment-client] + clientName = UL_client + [target-broker:deploymentServer] + targetUri = soc-deployment.heanet.ie:8089 + ''; + in + pkgs.writeShellScript "splunk-config" '' + set -o errexit -o pipefail -o nounset + shopt -s inherit_errexit + + umask u=rwx,g=rx,o= + + # start with a clean slate, it yells at ye if ye try to boot it up without doing this + rm -rf ${cfg.user.home}/* + + chmod 771 -R ${cfg.user.home} + + # pull in all relevent files + cp -f -R ${package}/* ${cfg.user.home} + + chmod 771 -R ${cfg.user.home} + chown ${cfg.user.user}:${cfg.user.group} -R ${cfg.user.home} + + # if [ ! -f ${cfg.user.home}/etc/splunk-launch.conf ]; then + export SPLUNK_HOME=${cfg.user.home} + + cp ${splunk_launch} $SPLUNK_HOME/etc/splunk-launch.conf + + cp ${etc_passwd} $SPLUNK_HOME/etc/passwd + + # ensure the dir exists + mkdir -p $SPLUNK_HOME/etc/apps/000_ul_umbrio_essentials/local/ + cp ${deployment_client} $SPLUNK_HOME/etc/apps/000_ul_umbrio_essentials/local/deploymentclient.conf + + + # Splunk is doing a crime by deciding what teh path is, need to comment out this line + # in /bin/pid_check.sh + # PATH=/usr/xpg4/bin:/usr/bin:/bin + # PATH= + sed -i -e 's/PATH=/#PATH=/g' ${cfg.user.home}/bin/pid_check.sh + + # make sure user permissions are correct + chmod 771 -R ${cfg.user.home} + chown ${cfg.user.user}:${cfg.user.group} -R ${cfg.user.home} + + # run it to see if that works + runuser -u ${cfg.user.user} -- $SPLUNK_HOME/bin/splunk start --accept-license + + # $SPLUNK_HOME/bin/splunk enable boot-start + # /etc/systemd/system/multi-user.target.wants/SplunkForwarder.service + # fi + ''; + }; + }; + + systemd.services."${name}" = { + wantedBy = [ + ]; + requires = [ + "${name}_prestart.service" + ]; + path = with pkgs; [util-linux toybox]; + serviceConfig = { + Type = "oneshot"; + User = cfg.user.user; + Group = cfg.user.group; + TimeoutSec = "infinity"; + Restart = "on-failure"; + # WorkingDirectory = "${cfg.package}/share/gitlab"; + # Slice = "system-gitlab.slice"; + RemainAfterExit = true; + + Environment = "SPLUNK_HOME=${cfg.user.home}"; + ExecStart = "${package}/bin/splunk _internal_launch_under_systemd"; + + # no so sure about these + NoNewPrivileges = "yes"; + AmbientCapabilities = "CAP_DAC_READ_SEARCH"; + }; + }; + }; +} From f8826c5d2f60fa98a8ca8a15608ea0fdab6cb114 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Mon, 1 Sep 2025 09:44:40 +0100 Subject: [PATCH 3/9] dbg: using marvin to test out teh new logging (slightly depressing) --- machines/marvin.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/machines/marvin.nix b/machines/marvin.nix index 175d61d..abb15d1 100644 --- a/machines/marvin.nix +++ b/machines/marvin.nix @@ -30,6 +30,7 @@ Notes: groups_trusted = map (x: "@${x}") groups; in { imports = [ + ../applications/itd/splunk/module.nix ]; deployment = { @@ -58,4 +59,8 @@ in { host = host; backup.enable = true; }; + + services.itd.splunk = { + enable = true; + }; } From b742a09c439ec76ab99a2a709eccca6895fe028b Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Tue, 2 Sep 2025 10:38:54 +0100 Subject: [PATCH 4/9] fix: move the fixing of the path to the package and not teh module --- applications/itd/splunk/module.nix | 7 ------- applications/itd/splunk/package.nix | 9 +++++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix index e186929..3c841fb 100644 --- a/applications/itd/splunk/module.nix +++ b/applications/itd/splunk/module.nix @@ -115,13 +115,6 @@ in { mkdir -p $SPLUNK_HOME/etc/apps/000_ul_umbrio_essentials/local/ cp ${deployment_client} $SPLUNK_HOME/etc/apps/000_ul_umbrio_essentials/local/deploymentclient.conf - - # Splunk is doing a crime by deciding what teh path is, need to comment out this line - # in /bin/pid_check.sh - # PATH=/usr/xpg4/bin:/usr/bin:/bin - # PATH= - sed -i -e 's/PATH=/#PATH=/g' ${cfg.user.home}/bin/pid_check.sh - # make sure user permissions are correct chmod 771 -R ${cfg.user.home} chown ${cfg.user.user}:${cfg.user.group} -R ${cfg.user.home} diff --git a/applications/itd/splunk/package.nix b/applications/itd/splunk/package.nix index aa5db9c..c5a21ef 100644 --- a/applications/itd/splunk/package.nix +++ b/applications/itd/splunk/package.nix @@ -2,7 +2,7 @@ stdenv, lib, fetchurl, - #, alsa-lib + gnused, openssl, zlib, autoPatchelfHook, @@ -23,6 +23,7 @@ stdenv.mkDerivation rec { # alsaLib openssl zlib + gnused ]; sourceRoot = "."; @@ -30,8 +31,12 @@ stdenv.mkDerivation rec { installPhase = '' runHook preInstall # gets unzipped and we just pop it all into teh output - # might add the patching of the path vars in the scripts to be here cp -R ./splunkforwarder $out + + # Splunk is doing a crime by deciding what teh path is, need to comment out this line in /bin/pid_check.sh + # PATH=/usr/xpg4/bin:/usr/bin:/bin + find $out -type f -name "*.sh" -print0 | xargs -0 sed -i -e 's/PATH=/#PATH=/g' + runHook postInstall ''; From f47e95c61de10eefdc195ee07c0e3308baa9f02c Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Fri, 5 Sep 2025 21:35:37 +0100 Subject: [PATCH 5/9] fix: service no longer stalling CPUShares and MemoryLimit are depricated Sep 05 20:30:42 marvin systemd[1]: /etc/systemd/system/splunk.service:9: Unit uses CPUShares=; please use CPUWeight= instead. Support for CPUShares= will be removed soon. Sep 05 20:30:42 marvin systemd[1]: /etc/systemd/system/splunk.service:18: Unit uses MemoryLimit=; please use MemoryMax= instead. Support for MemoryLimit= will be removed soon. --- applications/itd/splunk/module.nix | 34 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix index 3c841fb..30dcb19 100644 --- a/applications/itd/splunk/module.nix +++ b/applications/itd/splunk/module.nix @@ -45,7 +45,7 @@ in { }; # might not be required - # networking.firewall.allowedTCPPorts = [8089]; + networking.firewall.allowedTCPPorts = [8089]; # set up the core files systemd.services."${name}_prestart" = { @@ -137,21 +137,31 @@ in { ]; path = with pkgs; [util-linux toybox]; serviceConfig = { - Type = "oneshot"; - User = cfg.user.user; - Group = cfg.user.group; - TimeoutSec = "infinity"; - Restart = "on-failure"; - # WorkingDirectory = "${cfg.package}/share/gitlab"; - # Slice = "system-gitlab.slice"; - RemainAfterExit = true; - + Type = "simple"; + Restart="always"; Environment = "SPLUNK_HOME=${cfg.user.home}"; ExecStart = "${package}/bin/splunk _internal_launch_under_systemd"; - - # no so sure about these + KillMode="mixed"; + KillSignal="SIGINT"; + TimeoutStopSec=360; + LimitNOFILE=65536; + LimitRTPRIO=99; + SuccessExitStatus="51 52"; + RestartPreventExitStatus=51; + RestartForceExitStatus=52; + User = cfg.user.user; + Group = cfg.user.group; NoNewPrivileges = "yes"; AmbientCapabilities = "CAP_DAC_READ_SEARCH"; + + # ExecStartPre=-/bin/bash -c "chown -R splunk:splunk /opt/splunkforwarder" + Delegate="true"; +# CPUShares=1024; +# MemoryLimit=3973632000; + PermissionsStartOnly="true"; + # ExecStartPost=-/bin/bash -c "chown -R splunk:splunk /sys/fs/cgroup/cpu/system.slice/%n" + # ExecStartPost=-/bin/bash -c "chown -R splunk:splunk /sys/fs/cgroup/memory/system.slice/%n" + }; }; }; From 534dabca19804f05944222b10b4d32018cd62bbf Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Fri, 5 Sep 2025 22:11:46 +0100 Subject: [PATCH 6/9] feat: set ACL's --- applications/itd/splunk/module.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix index 30dcb19..8500a41 100644 --- a/applications/itd/splunk/module.nix +++ b/applications/itd/splunk/module.nix @@ -47,6 +47,21 @@ in { # might not be required networking.firewall.allowedTCPPorts = [8089]; + # need to set access controls + systemd.tmpfiles.rules = [ + "a /var/log - - - - u:splunk:rx" + "a /var/log/auth.log - - - - splunk:r" + "a /var/log/messages - - - - u:splunk:r" + "a /var/log/secure - - - - u:splunk:r" + "a /var/log/audit - - - - u:splunk:rx" + "a /var/log/audit.log - - - - u:splunk:r" + "a /var/log/audit/audit.log - - - - u:splunk:r" + "a /root - - - - u:splunk:rx" + "a /root/.bash_history - - - - u:splunk:r" + "a /home/* - - - - u:splunk:rx" + "a /home/*/.bash_history - - - - u:splunk:r" + ]; + # set up the core files systemd.services."${name}_prestart" = { wantedBy = [ From 0075fca9860a357857b5ca11907d6efb32b7b1da Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Fri, 5 Sep 2025 22:30:33 +0100 Subject: [PATCH 7/9] feat: needed to update nixpkgs to gain access to ``security.auditd.settings``. --- applications/acme.nix | 1 - flake.lock | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/applications/acme.nix b/applications/acme.nix index 1031cf5..97aff51 100644 --- a/applications/acme.nix +++ b/applications/acme.nix @@ -27,7 +27,6 @@ in { age.secrets.acme.file = ../secrets/dns_certs.secret.age; security.acme = { - preliminarySelfsigned = false; acceptTerms = true; defaults = { diff --git a/flake.lock b/flake.lock index 81c5faa..285b680 100644 --- a/flake.lock +++ b/flake.lock @@ -910,11 +910,11 @@ }, "nixpkgs_7": { "locked": { - "lastModified": 1751271578, - "narHash": "sha256-P/SQmKDu06x8yv7i0s8bvnnuJYkxVGBWLWHaU+tt4YY=", + "lastModified": 1756787288, + "narHash": "sha256-rw/PHa1cqiePdBxhF66V7R+WAP8WekQ0mCDG4CFqT8Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3016b4b15d13f3089db8a41ef937b13a9e33a8df", + "rev": "d0fc30899600b9b3466ddb260fd83deb486c32f1", "type": "github" }, "original": { From 1fa89834d03983b416fee719d5686f3c8feaff77 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Fri, 5 Sep 2025 22:35:09 +0100 Subject: [PATCH 8/9] feat: enabled auditd --- applications/itd/splunk/module.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix index 8500a41..52d0e4b 100644 --- a/applications/itd/splunk/module.nix +++ b/applications/itd/splunk/module.nix @@ -62,6 +62,13 @@ in { "a /home/*/.bash_history - - - - u:splunk:r" ]; + security.auditd = { + enable = true; + settings = { + log_group = cfg.user.group; + }; + }; + # set up the core files systemd.services."${name}_prestart" = { wantedBy = [ From a3b9d89b1a10b245417487adf25ce8701e503281 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Fri, 5 Sep 2025 22:45:32 +0100 Subject: [PATCH 9/9] fmt: formatted --- applications/itd/splunk/module.nix | 57 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/applications/itd/splunk/module.nix b/applications/itd/splunk/module.nix index 52d0e4b..304d29e 100644 --- a/applications/itd/splunk/module.nix +++ b/applications/itd/splunk/module.nix @@ -49,24 +49,24 @@ in { # need to set access controls systemd.tmpfiles.rules = [ - "a /var/log - - - - u:splunk:rx" - "a /var/log/auth.log - - - - splunk:r" - "a /var/log/messages - - - - u:splunk:r" - "a /var/log/secure - - - - u:splunk:r" - "a /var/log/audit - - - - u:splunk:rx" - "a /var/log/audit.log - - - - u:splunk:r" - "a /var/log/audit/audit.log - - - - u:splunk:r" - "a /root - - - - u:splunk:rx" - "a /root/.bash_history - - - - u:splunk:r" - "a /home/* - - - - u:splunk:rx" - "a /home/*/.bash_history - - - - u:splunk:r" + "a /var/log - - - - u:splunk:rx" + "a /var/log/auth.log - - - - u:splunk:r " + "a /var/log/messages - - - - u:splunk:r " + "a /var/log/secure - - - - u:splunk:r " + "a /var/log/audit - - - - u:splunk:rx" + "a /var/log/audit.log - - - - u:splunk:r " + "a /var/log/audit/audit.log - - - - u:splunk:r " + "a /root - - - - u:splunk:rx" + "a /root/.bash_history - - - - u:splunk:r " + "a /home/* - - - - u:splunk:rx" + "a /home/*/.bash_history - - - - u:splunk:r " ]; security.auditd = { - enable = true; - settings = { - log_group = cfg.user.group; - }; + enable = true; + settings = { + log_group = cfg.user.group; + }; }; # set up the core files @@ -160,30 +160,29 @@ in { path = with pkgs; [util-linux toybox]; serviceConfig = { Type = "simple"; - Restart="always"; + Restart = "always"; Environment = "SPLUNK_HOME=${cfg.user.home}"; ExecStart = "${package}/bin/splunk _internal_launch_under_systemd"; - KillMode="mixed"; - KillSignal="SIGINT"; - TimeoutStopSec=360; - LimitNOFILE=65536; - LimitRTPRIO=99; - SuccessExitStatus="51 52"; - RestartPreventExitStatus=51; - RestartForceExitStatus=52; + KillMode = "mixed"; + KillSignal = "SIGINT"; + TimeoutStopSec = 360; + LimitNOFILE = 65536; + LimitRTPRIO = 99; + SuccessExitStatus = "51 52"; + RestartPreventExitStatus = 51; + RestartForceExitStatus = 52; User = cfg.user.user; Group = cfg.user.group; NoNewPrivileges = "yes"; AmbientCapabilities = "CAP_DAC_READ_SEARCH"; # ExecStartPre=-/bin/bash -c "chown -R splunk:splunk /opt/splunkforwarder" - Delegate="true"; -# CPUShares=1024; -# MemoryLimit=3973632000; - PermissionsStartOnly="true"; + Delegate = "true"; + # CPUShares=1024; + # MemoryLimit=3973632000; + PermissionsStartOnly = "true"; # ExecStartPost=-/bin/bash -c "chown -R splunk:splunk /sys/fs/cgroup/cpu/system.slice/%n" # ExecStartPost=-/bin/bash -c "chown -R splunk:splunk /sys/fs/cgroup/memory/system.slice/%n" - }; }; };