From 834ec7cf947f518701be367cefce77f8d7bcd66c Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 18:34:24 +0100 Subject: [PATCH 01/12] dns: set teh right paths for the rDNS Closes #9 Related to 646a21f0 --- applications/dns.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/applications/dns.nix b/applications/dns.nix index 72d8837..2f3cf8a 100644 --- a/applications/dns.nix +++ b/applications/dns.nix @@ -94,7 +94,7 @@ ${format_records sort_records_srv 17} # https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/4/html/reference_guide/s2-bind-configuration-zone-reverse # config for our reverse dnspointers (not properly working) get_config_file_rev = (domain: -''$ORIGIN 64-127.99.1.193.in-addr.arpa. +''$ORIGIN 64-64.99.1.193.in-addr.arpa. $TTL 60 ; 1 minute ; hostmaster@skynet.ie is an email address that recieves stuff related to dns @ IN SOA ${nameserver}.skynet.ie. hostmaster.skynet.ie. ( @@ -320,7 +320,7 @@ in { (create_entry_zone "skynet.ie" extraConfig.owned ) // (create_entry_zone "ulcompsoc.ie" extraConfig.owned ) // - #(create_entry_zone "99.1.193.in-addr.arpa" extraConfig.reverse ) // + (create_entry_zone "64-64.99.1.193.in-addr.arpa" extraConfig.reverse ) // (create_entry_zone "conradcollins.net" extraConfig.old )// (create_entry_zone "edelharty.net" extraConfig.old ); @@ -330,7 +330,7 @@ in { (create_entry_etc "skynet.ie" "owned") // (create_entry_etc "ulcompsoc.ie" "owned") // - #(create_entry_etc "99.1.193.in-addr.arpa" "reverse") // + (create_entry_etc "64-64.99.1.193.in-addr.arpa" "reverse") // (create_entry_etc "conradcollins.net" "old") // (create_entry_etc "edelharty.net" "old"); From b0d7c5173658b8cf773734d37ac980d27aee9cad Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 19:56:30 +0100 Subject: [PATCH 02/12] gitlab: updated teh runners to have a token each as they should --- applications/gitlab_runner.nix | 12 ++++++------ secrets/gitlab/runners/runner01.age | Bin 730 -> 722 bytes secrets/gitlab/runners/runner02.age | Bin 0 -> 739 bytes secrets/secrets.nix | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 secrets/gitlab/runners/runner02.age diff --git a/applications/gitlab_runner.nix b/applications/gitlab_runner.nix index 45186b0..b470701 100644 --- a/applications/gitlab_runner.nix +++ b/applications/gitlab_runner.nix @@ -47,8 +47,8 @@ pkgs.gitlab-runner ]; - age.secrets.runner_nix.file = ../secrets/gitlab/runners/runner01.age; - age.secrets.runner_general.file = ../secrets/gitlab/runners/runner01.age; + age.secrets.runner_01_nix.file = ../secrets/gitlab/runners/runner01.age; + age.secrets.runner_02_general.file = ../secrets/gitlab/runners/runner02.age; boot.kernel.sysctl."net.ipv4.ip_forward" = true; # 1 @@ -67,9 +67,9 @@ # might make a function later to have multiple runners, might never need it though runner_nix = { cloneUrl = cfg.runner.gitlab; - description = cfg.runner.description; + description = "For Nix only"; registrationFlags = [ "--docker-host" "tcp://127.0.0.1:2375" ]; - registrationConfigFile = config.age.secrets.runner_nix.path; + registrationConfigFile = config.age.secrets.runner_01_nix.path; dockerImage = cfg.runner.docker.image; # from https://nixos.wiki/wiki/Gitlab_runner @@ -106,9 +106,9 @@ runner_general = { cloneUrl = cfg.runner.gitlab; - description = cfg.runner.description; + description = "General Runner"; registrationFlags = [ "--docker-host" "tcp://127.0.0.1:2375" ]; - registrationConfigFile = config.age.secrets.runner_general.path; + registrationConfigFile = config.age.secrets.runner_02_general.path; dockerImage = cfg.runner.docker.image; }; }; diff --git a/secrets/gitlab/runners/runner01.age b/secrets/gitlab/runners/runner01.age index 1744c77d0da6bb457324c77cd50dcc637ba1b659..81f1aca14ed751dc26224049fcd9eac7ce1a6bd7 100644 GIT binary patch literal 722 zcmZ9_OKZ~r003Z7&=EZN4;GlRL)N8fle7Vm<<%zXV@=znO+_(D)4bX=%R5uNc$nG$-uaAUTV3G~_Q)X)G$SWcNCxVl+@s;7YrVRUpoM2nalwWS=YMjl&7y@2tGDx7sY6G3Ro z9gg7Xd8%?9CTz42!7Jt>#Tb?=0);{W*I~&&~W+o@w>mb96* XZ^x$ZcaJ`%FRwjw?q6DvmtXw@v`6|{ literal 730 zcmZ9|OKZ~r007{N7jy7&*hLVc7dNw-rdgYYsVr@iHfxeTvOFEbG|QuF9!=8bb@2h} z2EuS4?4}^%9}x87PJ%}T^&l#G*~x49Pgri9i~xd$)=8=s6!YaE(Qpstv-S$#3wTPg@qp@i_6o8R zqP56V4r@WxShg(CbWD9ZCOK&!_F%X~*Q=t#(BzP|4r^J=@B+!WAFt%6xu|<{~jd1JO80{#StQV(jt>=#hLz>|w9V&AF zC($eV{L;DfQo#xhEi%9Yi26&Ts|uKNHES)U06WA`VIX31Q*v63-1J?E|}QB;r9OSKR6`*&;S4c diff --git a/secrets/gitlab/runners/runner02.age b/secrets/gitlab/runners/runner02.age new file mode 100644 index 0000000000000000000000000000000000000000..241398043bb832a01637089bee59b68ea922e019 GIT binary patch literal 739 zcmZ9|$%@lp003aolX1h7qKpJYEri;1Nt?{5Owu;fq-nZk`BT)Hq?7zzvb0UJ;!zM! zb5KMPM7+3NJ*apWR0NNT3VIP=z?&d=oG19;)1jF)-QgmcEytdF-dn(2m`AdgP~RVA z&Fum~=#)!WeHC!cYr!klZ_ER z$zwxU!(%>IQ)ZRYFswKmxXmH1528UFMa?v>6u_wJYMd!swWhtaQw7>nn!-U5w#aba z+O5q=lZOz+1xg#6o?K?;3JrM5#r1-N0YY?#22r2SVpDcG6~rnVAqiv}nxsnYpc1N# zCyj>N3D8>SVurx(E27VQ$^Nj7j(JqLN+G|MT6lxggK33yx@XBB3+^ygL0FY1meE Date: Wed, 9 Aug 2023 20:12:43 +0100 Subject: [PATCH 03/12] ci: improve it so some thigns only run on main --- .gitlab-ci.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdc2e47..1b9823a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ update: - $UPDATE_FLAKE == "yes" # every commit on main will build and deploy -.build_template: &build +.build_template: &builder tags: - nix before_script: @@ -54,8 +54,6 @@ update: - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - git pull origin main only: - refs: - - main changes: - applications/**/* - machines/**/* @@ -63,21 +61,29 @@ update: - flake.* - .gitlab-ci.yml +# deploy items only run on main +.deploy_template: &deployment + only: + refs: + - main + build: - <<: *build + <<: *builder stage: test script: - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- build # dns always has to be deployed first deploy_dns: - <<: *build + <<: *builder + <<: *deployment stage: deploy script: - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-dns deploy_core: - <<: *build + <<: *builder + <<: *deployment stage: deploy needs: - deploy_dns @@ -85,7 +91,8 @@ deploy_core: - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-core deploy_active: - <<: *build + <<: *builder + <<: *deployment stage: deploy needs: - deploy_dns @@ -93,7 +100,8 @@ deploy_active: - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active deploy_gitlab: - <<: *build + <<: *builder + <<: *deployment stage: deploy_gitlab script: - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-gitlab From d630c724882b98d1ff2ed003ac6fb084f6e2a5e8 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:27:04 +0100 Subject: [PATCH 04/12] ci: only run the scripts that need envs to be added on main --- .gitlab-ci.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b9823a..195fb79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,19 +40,24 @@ update: variables: - $UPDATE_FLAKE == "yes" +.scripts_base: &scripts_base + # load nix environment + - . "$HOME/.nix-profile/etc/profile.d/nix.sh" + - git pull origin main + +.scripts_deploy: &scripts_deploy + # setup ssh key + - eval $(ssh-agent -s) + - echo "$DEPLOY_KEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + # every commit on main will build and deploy .build_template: &builder tags: - nix before_script: - # setup ssh key - - eval $(ssh-agent -s) - - echo "$DEPLOY_KEY" | tr -d '\r' | ssh-add - > /dev/null - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - # load nix environment - - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - - git pull origin main + - *scripts_base only: changes: - applications/**/* @@ -63,6 +68,9 @@ update: # deploy items only run on main .deploy_template: &deployment + before_script: + - *scripts_deploy + - *scripts_base only: refs: - main From 8e489c1616787344d4adc155f77a4dafc227f67c Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:31:31 +0100 Subject: [PATCH 05/12] ci: pull the current branch --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 195fb79..493c9b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ update: .scripts_base: &scripts_base # load nix environment - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - - git pull origin main + - git pull origin $CI_COMMIT_REF_NAME .scripts_deploy: &scripts_deploy # setup ssh key From 8a2c97ab2cbb48d65551aa8b262bf6760123bf62 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:45:33 +0100 Subject: [PATCH 06/12] ci: install colmena --- .gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 493c9b9..3533038 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ update: - git config --global user.email "${CI_EMAIL}" - git config --global user.name "${CI_USERNAME}" # the part that updates the flake - - nix --experimental-features 'nix-command flakes' flake lock --update-input $PACKAGE_NAME + - nix flake lock --update-input $PACKAGE_NAME - git add flake.lock - git commit -m "[skip ci] Updated flake for $PACKAGE_NAME" || echo "No changes, nothing to commit" # we have a custom domain @@ -44,6 +44,7 @@ update: # load nix environment - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - git pull origin $CI_COMMIT_REF_NAME + - nix-shell -p colmena .scripts_deploy: &scripts_deploy # setup ssh key @@ -79,7 +80,7 @@ build: <<: *builder stage: test script: - - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- build + - colmena build # dns always has to be deployed first deploy_dns: @@ -87,7 +88,7 @@ deploy_dns: <<: *deployment stage: deploy script: - - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-dns + - colmena apply --on @active-dns deploy_core: <<: *builder @@ -96,7 +97,7 @@ deploy_core: needs: - deploy_dns script: - - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-core + - colmena apply --on @active-core deploy_active: <<: *builder @@ -105,12 +106,12 @@ deploy_active: needs: - deploy_dns script: - - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active + - colmena apply --on @active deploy_gitlab: <<: *builder <<: *deployment stage: deploy_gitlab script: - - nix --experimental-features 'nix-command flakes' run nixpkgs#colmena -- apply --on @active-gitlab + - colmena apply --on @active-gitlab when: manual \ No newline at end of file From 51c9761380874a9887319c03b7472708928a8e4f Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:47:47 +0100 Subject: [PATCH 07/12] ci: pre-install colmena --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3533038..9080b30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,6 @@ update: # load nix environment - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - git pull origin $CI_COMMIT_REF_NAME - - nix-shell -p colmena .scripts_deploy: &scripts_deploy # setup ssh key From e9941ddfdde86e3f21d23721a5387403078ca328 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:56:22 +0100 Subject: [PATCH 08/12] ci: further testing --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9080b30..6bf96bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,8 +42,9 @@ update: .scripts_base: &scripts_base # load nix environment - - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - git pull origin $CI_COMMIT_REF_NAME + - . "$HOME/.nix-profile/etc/profile.d/nix.sh" + - nix-shell -p colmena .scripts_deploy: &scripts_deploy # setup ssh key From e53050aa116947c3d8ccc7bf7b0e1e065aa605d6 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:57:35 +0100 Subject: [PATCH 09/12] ci: try nix-env --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bf96bd..e2a5568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ update: # load nix environment - git pull origin $CI_COMMIT_REF_NAME - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - - nix-shell -p colmena + - nix-env -iA colmena .scripts_deploy: &scripts_deploy # setup ssh key From 1b6dd39e71c3c5ab081b9ae83c5be1525ab75824 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 20:59:29 +0100 Subject: [PATCH 10/12] ci: try the tarball --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e2a5568..705c6bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ update: # load nix environment - git pull origin $CI_COMMIT_REF_NAME - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - - nix-env -iA colmena + - nix-env -if https://github.com/zhaofengli/colmena/tarball/main .scripts_deploy: &scripts_deploy # setup ssh key From c1573134ea422fe614aa960e97a1e9f3f3cae001 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 21:08:39 +0100 Subject: [PATCH 11/12] ci: lock in a specific version --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 705c6bd..cf7c5d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ update: # load nix environment - git pull origin $CI_COMMIT_REF_NAME - . "$HOME/.nix-profile/etc/profile.d/nix.sh" - - nix-env -if https://github.com/zhaofengli/colmena/tarball/main + - nix-env -if https://github.com/zhaofengli/colmena/tarball/v0.4.0 .scripts_deploy: &scripts_deploy # setup ssh key From 12e25162eef17a0ededee1533f3147f73eecb643 Mon Sep 17 00:00:00 2001 From: Brendan Golden Date: Wed, 9 Aug 2023 21:09:13 +0100 Subject: [PATCH 12/12] ci: switch over to unstable --- applications/gitlab_runner.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/gitlab_runner.nix b/applications/gitlab_runner.nix index b470701..48559da 100644 --- a/applications/gitlab_runner.nix +++ b/applications/gitlab_runner.nix @@ -90,7 +90,7 @@ mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root mkdir -p -m 0700 "$HOME/.nix-defexpr" . ${pkgs.nix}/etc/profile.d/nix-daemon.sh - ${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixos-23.05 nixpkgs # 3 + ${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixos-unstable nixpkgs # 3 ${pkgs.nix}/bin/nix-channel --update nixpkgs ${pkgs.nix}/bin/nix-env -i ${concatStringsSep " " (with pkgs; [ nix cacert git openssh ])} '';