Merge branch '#10_wolves_database' into 'main'

feat: added the function to get teh data and updated teh times tehy run at

See merge request compsoc1/skynet/ldap/backend!8

Cargo.toml

@@ -4,6 +4,8 @@ version = "0.1.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "update_data"
 
 [[bin]]
 name = "update_groups"

flake.nix

@@ -67,6 +67,44 @@
           USERS_LIFETIME  = lib.strings.concatStringsSep "," cfg.users.lifetime;
           USERS_BANNED    = lib.strings.concatStringsSep "," cfg.users.banned;
         };
+        
+        service_name = script: lib.strings.sanitizeDerivationName("${cfg.prefix}${cfg.user}@${script}");
+        
+        # oneshot scripts to run
+        serviceGenerator = builtins.mapAttrs (script: time: nameValuePair (service_name script) {
+          description = "Service for ${desc} ${script}";
+          wantedBy    = [ ];
+          after       = [ "network-online.target" ];
+          environment = environment_config;
+          
+          serviceConfig = {
+            Type            = "oneshot";
+            DynamicUser     = true;
+            ExecStart       = "${self.defaultPackage."${system}"}/bin/${script}";
+            EnvironmentFile = "${cfg.envFile}";
+          };
+        }) scripts;
+        
+        # each timer will run the above service
+        timerGenerator = builtins.mapAttrs (script: time: nameValuePair (service_name script) {
+          description = "Timer for ${desc} ${script}";
+          
+          wantedBy    = [ "timers.target" ];
+          partOf      = [ "${service_name script}.service" ];
+          timerConfig = {
+            OnCalendar  = time;
+            Unit        = "${service_name script}.service";
+            Persistent  = true;
+          };
+        }) scripts;
+        
+        # modify these
+        scripts = {
+          "new_data"      = "*:0,15,30,45";  
+          "new_users"     = "*:5,20,35,50";        
+          "update_groups" = "*:10";
+        };
+
       in { 
         options.services."${package_name}" = {
           enable = mkEnableOption "enable ${package_name}";
@@ -145,6 +183,13 @@
             description = "The home for the user";
           };
           
+          prefix = mkOption rec {
+            type = types.str;
+            default = "skynet_";
+            example = default;
+            description = "The prefix used to name service/folders";
+          };
+            
         };
 
         config = mkIf cfg.enable {
@@ -158,7 +203,9 @@
             group         = "${cfg.user}";
           };
           
-          systemd.services."${cfg.user}" = {
+          systemd.services = {
+            # main service
+            "${cfg.user}" = {
               description = desc;
               wantedBy    = [ "multi-user.target" ];
               after       = [ "network-online.target" ];
@@ -174,59 +221,10 @@
                 EnvironmentFile = "${cfg.envFile}";
               };
             };
+          } // serviceGenerator;
           
-          # for updating the data
+          # timers to run the above services
-          systemd.services."${cfg.user}_update" = {
+          systemd.timers = timerGenerator;
-            description = "${desc} Update groups";
-            wantedBy    = [ ];
-            after       = [ "network-online.target" ];
-            environment = environment_config;
-            
-            serviceConfig = {
-              Type            = "oneshot";
-              DynamicUser     = true;
-              ExecStart       = "${self.defaultPackage."${system}"}/bin/update_groups";
-              EnvironmentFile = "${cfg.envFile}";
-            };
-          };
-           
-          systemd.timers."${cfg.user}_update" = {
-            description = "Run the update script for ${desc}";
-            
-            wantedBy    = [ "timers.target" ];
-            partOf      = [ "${cfg.user}_update.service" ];
-            timerConfig = {
-              # every hour
-              OnCalendar  = "*-*-* *:00:00";
-              Unit        = "${cfg.user}_update.service";
-            };
-          };
-          
-          # for new users
-          systemd.services."${cfg.user}_new_users" = {
-            description = "${desc} Get new users";
-            wantedBy    = [ ];
-            after       = [ "network-online.target" ];
-            environment = environment_config;
-            
-            serviceConfig = {
-              Type            = "oneshot";
-              DynamicUser     = true;
-              ExecStart       = "${self.defaultPackage."${system}"}/bin/new_users";
-              EnvironmentFile = "${cfg.envFile}";
-            };
-          };
-           
-          systemd.timers."${cfg.user}_new_users" = {
-            description = "Run the new users script for ${desc}";
-            wantedBy    = [ "timers.target" ];
-            partOf      = [ "${cfg.user}_new_users.service" ];
-            timerConfig = {
-              # every 15 min
-              OnCalendar  = "*:0/15";
-              Unit        = "${cfg.user}_new_users.service";
-            };
-          };
           
         };
         

src/bin/new_users.rs

@@ -4,7 +4,7 @@ use lettre::{
     Message, SmtpTransport, Transport,
 };
 use maud::html;
-use skynet_ldap_backend::{db_init, get_config, get_now_iso, random_string, read_csv, Accounts, AccountsNew, Config, Record};
+use skynet_ldap_backend::{db_init, get_config, get_now_iso, get_wolves, random_string, AccountWolves, Accounts, AccountsNew, Config};
 use sqlx::{Pool, Sqlite};
 
 #[async_std::main]
@@ -12,8 +12,7 @@ async fn main() {
     let config = get_config();
     let db = db_init(&config).await.unwrap();
 
-    if let Ok(records) = read_csv(&config) {
+    for record in get_wolves(&db).await {
-        for record in records {
         // skynet emails not permitted
         if record.email.trim().ends_with("@skynet.ie") {
             continue;
@@ -39,7 +38,6 @@ async fn main() {
             }
         }
     }
-    }
 }
 
 async fn check(db: &Pool<Sqlite>, mail: &str) -> bool {
@@ -75,7 +73,7 @@ async fn check_pending(db: &Pool<Sqlite>, mail: &str) -> bool {
 }
 
 // using https://github.com/lettre/lettre/blob/57886c367d69b4d66300b322c94bd910b1eca364/examples/maud_html.rs
-fn send_mail(config: &Config, record: &Record, auth: &str) -> Result<Response, lettre::transport::smtp::Error> {
+fn send_mail(config: &Config, record: &AccountWolves, auth: &str) -> Result<Response, lettre::transport::smtp::Error> {
     let recipient = &record.name_first;
     let mail = &record.email;
     let url_base = "https://sso.skynet.ie";
@@ -180,7 +178,7 @@ fn send_mail(config: &Config, record: &Record, auth: &str) -> Result<Response, l
     mailer.send(&email)
 }
 
-async fn save_to_db(db: &Pool<Sqlite>, record: &Record, auth: &str) -> Result<Option<AccountsNew>, sqlx::Error> {
+async fn save_to_db(db: &Pool<Sqlite>, record: &AccountWolves, auth: &str) -> Result<Option<AccountsNew>, sqlx::Error> {
     sqlx::query_as::<_, AccountsNew>(
         "
         INSERT OR REPLACE INTO accounts_new (mail, auth_code, date_iso, date_expiry, name_first, name_surname, id_student)

src/bin/update_data.rs

@@ -0,0 +1,162 @@
+use ldap3::{LdapConn, Scope, SearchEntry};
+use skynet_ldap_backend::{db_init, get_config, AccountWolves, Accounts, Config};
+use sqlx::{Pool, Sqlite};
+
+#[async_std::main]
+async fn main() -> tide::Result<()> {
+    let config = get_config();
+    let db = db_init(&config).await.unwrap();
+
+    update_wolves(&config, &db).await;
+    update_ldap(&config, &db).await;
+
+    Ok(())
+}
+
+async fn update_wolves(config: &Config, db: &Pool<Sqlite>) {
+    let mut records = vec![];
+
+    if let Ok(accounts) = get_csv(config) {
+        for account in accounts {
+            records.push(AccountWolves::from(account));
+        }
+    }
+
+    for account in records {
+        update_account(db, &account).await;
+    }
+}
+
+async fn update_ldap(config: &Config, db: &Pool<Sqlite>) {
+    let mut ldap = LdapConn::new(&config.ldap_host).unwrap();
+
+    ldap.simple_bind(&config.ldap_admin, &config.ldap_admin_pw).unwrap().success().unwrap();
+
+    // use this to pre load a large chunk of data
+    if let Ok(x) = ldap.search("ou=users,dc=skynet,dc=ie", Scope::OneLevel, "(objectClass=*)", vec!["uid", "uidNumber", "skDiscord", "skMemberOf", "mail", "skID", "userPassword"]) {
+        if let Ok((rs, _res)) = x.success() {
+            for entry in rs {
+                let tmp = SearchEntry::construct(entry);
+
+                let mut tmp_account = Accounts {
+                    user: "".to_string(),
+                    uid: 0,
+                    discord: None,
+                    mail: "".to_string(),
+                    student_id: "".to_string(),
+                    enabled: false,
+                    secure: false,
+                };
+
+                // pull out the required info
+                if tmp.attrs.contains_key("uid") && !tmp.attrs["uid"].is_empty() {
+                    tmp_account.user = tmp.attrs["uid"][0].clone();
+                }
+                if tmp.attrs.contains_key("uidNumber") && !tmp.attrs["uidNumber"].is_empty() {
+                    tmp_account.uid = tmp.attrs["uidNumber"][0].clone().parse().unwrap_or(0);
+                }
+                if tmp.attrs.contains_key("skDiscord") && !tmp.attrs["skDiscord"].is_empty() {
+                    tmp_account.discord = Option::from(tmp.attrs["skDiscord"][0].clone());
+                }
+                if tmp.attrs.contains_key("mail") && !tmp.attrs["mail"].is_empty() {
+                    tmp_account.mail = tmp.attrs["mail"][0].clone();
+                }
+                if tmp.attrs.contains_key("skID") && !tmp.attrs["skID"].is_empty() {
+                    tmp_account.student_id = tmp.attrs["skID"][0].clone();
+                }
+                if tmp.attrs.contains_key("skMemberOf") && !tmp.attrs["skMemberOf"].is_empty() && tmp.attrs["skMemberOf"].contains(&String::from("cn=skynet-users-linux,ou=groups,dc=skynet,dc=ie")) {
+                    tmp_account.enabled = true;
+                }
+                if tmp.attrs.contains_key("userPassword") && !tmp.attrs["userPassword"].is_empty() {
+                    tmp_account.secure = tmp.attrs["userPassword"][0].starts_with("{SSHA512}")
+                }
+
+                if !tmp_account.user.is_empty() {
+                    sqlx::query_as::<_, Accounts>(
+                        "
+                        INSERT OR REPLACE INTO accounts (user, uid, discord, mail, student_id, enabled, secure)
+                        VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+                        ",
+                    )
+                    .bind(&tmp_account.user)
+                    .bind(tmp_account.uid)
+                    .bind(&tmp_account.discord)
+                    .bind(&tmp_account.mail)
+                    .bind(&tmp_account.student_id)
+                    .bind(tmp_account.enabled)
+                    .bind(tmp_account.secure)
+                    .fetch_optional(db)
+                    .await
+                    .ok();
+                }
+            }
+        }
+    }
+
+    // done with ldap
+    ldap.unbind().unwrap();
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct RecordCSV {
+    #[serde(rename = "MemID")]
+    mem_id: String,
+    #[serde(rename = "Student Num")]
+    id_student: String,
+    #[serde(rename = "Contact Email")]
+    email: String,
+    #[serde(rename = "Expiry")]
+    expiry: String,
+    #[serde(rename = "First Name")]
+    name_first: String,
+    #[serde(rename = "Last Name")]
+    name_second: String,
+}
+impl From<RecordCSV> for AccountWolves {
+    fn from(input: RecordCSV) -> Self {
+        AccountWolves {
+            id_wolves: input.mem_id,
+            id_student: input.id_student,
+            email: input.email,
+            expiry: input.expiry,
+            name_first: input.name_first,
+            name_second: input.name_second,
+        }
+    }
+}
+
+fn get_csv(config: &Config) -> Result<Vec<RecordCSV>, Box<dyn std::error::Error>> {
+    let mut records: Vec<RecordCSV> = vec![];
+
+    if let Ok(mut rdr) = csv::Reader::from_path(format!("{}/{}", &config.home, &config.csv)) {
+        for result in rdr.deserialize() {
+            // Notice that we need to provide a type hint for automatic
+            // deserialization.
+            let record: RecordCSV = result?;
+            if record.mem_id.is_empty() {
+                continue;
+            }
+            records.push(record);
+        }
+    }
+
+    Ok(records)
+}
+
+async fn update_account(db: &Pool<Sqlite>, account: &AccountWolves) {
+    sqlx::query_as::<_, AccountWolves>(
+        "
+        INSERT OR REPLACE INTO accounts_wolves (id_wolves, id_student, email, expiry, name_first, name_second)
+        VALUES (?1, ?2, ?3, ?4, ?5, ?6)
+        ",
+    )
+    .bind(&account.id_wolves)
+    .bind(&account.id_student)
+    .bind(&account.email)
+    .bind(&account.expiry)
+    .bind(&account.name_first)
+    .bind(&account.name_second)
+    .fetch_optional(db)
+    .await
+    .ok();
+}

src/bin/update_groups.rs

@@ -1,5 +1,5 @@
 use ldap3::{LdapConn, Mod};
-use skynet_ldap_backend::{db_init, get_config, get_now_iso, read_csv, Accounts, Config};
+use skynet_ldap_backend::{db_init, get_config, get_now_iso, get_wolves, Accounts, Config};
 use sqlx::{Pool, Sqlite};
 use std::{collections::HashSet, env, error::Error};
 
@@ -116,9 +116,7 @@ async fn from_csv(config: &Config) -> Result<HashSet<String>, Box<dyn Error>> {
 
     let mut uids = HashSet::new();
 
-    let records = read_csv(config)?;
+    for record in get_wolves(&db).await {
-
-    for record in records {
         // only import users if it is actually active.
         if record.expiry < get_now_iso(true) {
             continue;

src/lib.rs

@@ -1,7 +1,6 @@
 pub mod methods;
 use chrono::{Datelike, SecondsFormat, Utc};
 use dotenvy::dotenv;
-use ldap3::{LdapConn, Scope, SearchEntry};
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
 use sqlx::{
     sqlite::{SqliteConnectOptions, SqlitePoolOptions},
@@ -14,6 +13,16 @@ use std::{
 };
 use tide::prelude::*;
 
+#[derive(Debug, Deserialize, Serialize, sqlx::FromRow)]
+pub struct AccountWolves {
+    pub id_wolves: String,
+    pub id_student: String,
+    pub email: String,
+    pub expiry: String,
+    pub name_first: String,
+    pub name_second: String,
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize, sqlx::FromRow)]
 pub struct AccountsNew {
     pub mail: String,
@@ -44,6 +53,19 @@ pub async fn db_init(config: &Config) -> Result<Pool<Sqlite>, Error> {
         .connect_with(SqliteConnectOptions::from_str(&format!("sqlite://{}", database))?.create_if_missing(true))
         .await?;
 
+    sqlx::query(
+        "CREATE TABLE IF NOT EXISTS accounts_wolves (
+            id_wolves text primary key,
+            id_student text not null,
+            email text not null,
+            expiry text not null,
+            name_first text not null,
+            name_surname integer not null
+         )",
+    )
+    .execute(&pool)
+    .await?;
+
     sqlx::query(
         "CREATE TABLE IF NOT EXISTS accounts_new (
             mail text primary key,
@@ -83,8 +105,6 @@ pub async fn db_init(config: &Config) -> Result<Pool<Sqlite>, Error> {
         .execute(&pool)
         .await?;
 
-    update_accounts(&pool, config).await;
-
     Ok(pool)
 }
 
@@ -176,111 +196,19 @@ pub fn get_config() -> Config {
     config
 }
 
-async fn update_accounts(pool: &Pool<Sqlite>, config: &Config) {
-    let mut ldap = LdapConn::new(&config.ldap_host).unwrap();
-
-    ldap.simple_bind(&config.ldap_admin, &config.ldap_admin_pw).unwrap().success().unwrap();
-
-    // use this to pre load a large chunk of data
-    if let Ok(x) = ldap.search("ou=users,dc=skynet,dc=ie", Scope::OneLevel, "(objectClass=*)", vec!["uid", "uidNumber", "skDiscord", "skMemberOf", "mail", "skID", "userPassword"]) {
-        if let Ok((rs, _res)) = x.success() {
-            for entry in rs {
-                let tmp = SearchEntry::construct(entry);
-
-                let mut tmp_account = Accounts {
-                    user: "".to_string(),
-                    uid: 0,
-                    discord: None,
-                    mail: "".to_string(),
-                    student_id: "".to_string(),
-                    enabled: false,
-                    secure: false,
-                };
-
-                // pull out the required info
-                if tmp.attrs.contains_key("uid") && !tmp.attrs["uid"].is_empty() {
-                    tmp_account.user = tmp.attrs["uid"][0].clone();
-                }
-                if tmp.attrs.contains_key("uidNumber") && !tmp.attrs["uidNumber"].is_empty() {
-                    tmp_account.uid = tmp.attrs["uidNumber"][0].clone().parse().unwrap_or(0);
-                }
-                if tmp.attrs.contains_key("skDiscord") && !tmp.attrs["skDiscord"].is_empty() {
-                    tmp_account.discord = Option::from(tmp.attrs["skDiscord"][0].clone());
-                }
-                if tmp.attrs.contains_key("mail") && !tmp.attrs["mail"].is_empty() {
-                    tmp_account.mail = tmp.attrs["mail"][0].clone();
-                }
-                if tmp.attrs.contains_key("skID") && !tmp.attrs["skID"].is_empty() {
-                    tmp_account.student_id = tmp.attrs["skID"][0].clone();
-                }
-                if tmp.attrs.contains_key("skMemberOf") && !tmp.attrs["skMemberOf"].is_empty() && tmp.attrs["skMemberOf"].contains(&String::from("cn=skynet-users-linux,ou=groups,dc=skynet,dc=ie")) {
-                    tmp_account.enabled = true;
-                }
-                if tmp.attrs.contains_key("userPassword") && !tmp.attrs["userPassword"].is_empty() {
-                    tmp_account.secure = tmp.attrs["userPassword"][0].starts_with("{SSHA512}")
-                }
-
-                if !tmp_account.user.is_empty() {
-                    sqlx::query_as::<_, Accounts>(
-                        "
-                        INSERT OR REPLACE INTO accounts (user, uid, discord, mail, student_id, enabled, secure)
-                        VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
-                        ",
-                    )
-                    .bind(&tmp_account.user)
-                    .bind(tmp_account.uid)
-                    .bind(&tmp_account.discord)
-                    .bind(&tmp_account.mail)
-                    .bind(&tmp_account.student_id)
-                    .bind(tmp_account.enabled)
-                    .bind(tmp_account.secure)
-                    .fetch_optional(pool)
-                    .await
-                    .ok();
-                }
-            }
-        }
-    }
-
-    // done with ldap
-    ldap.unbind().unwrap();
-}
-
-#[derive(Debug, serde::Deserialize)]
-pub struct Record {
-    #[serde(rename = "MemID")]
-    pub mem_id: String,
-    #[serde(rename = "Student Num")]
-    pub id_student: String,
-    #[serde(rename = "Contact Email")]
-    pub email: String,
-    #[serde(rename = "Expiry")]
-    pub expiry: String,
-    #[serde(rename = "First Name")]
-    pub name_first: String,
-    #[serde(rename = "Last Name")]
-    pub name_second: String,
-}
-
-pub fn read_csv(config: &Config) -> Result<Vec<Record>, Box<dyn std::error::Error>> {
-    let mut records: Vec<Record> = vec![];
-
-    if let Ok(mut rdr) = csv::Reader::from_path(format!("{}/{}", &config.home, &config.csv)) {
-        for result in rdr.deserialize() {
-            // Notice that we need to provide a type hint for automatic
-            // deserialization.
-            let record: Record = result?;
-            if record.mem_id.is_empty() {
-                continue;
-            }
-            records.push(record);
-        }
-    }
-
-    Ok(records)
-}
-
 // from https://rust-lang-nursery.github.io/rust-cookbook/algorithms/randomness.html#create-random-passwords-from-a-set-of-alphanumeric-characters
 pub fn random_string(len: usize) -> String {
     thread_rng().sample_iter(&Alphanumeric).take(len).map(char::from).collect()
 }
+
+pub async fn get_wolves(db: &Pool<Sqlite>) -> Vec<AccountWolves> {
+    sqlx::query_as::<_, AccountWolves>(
+        r#"
+        SELECT *
+        FROM accounts_wolves
+        "#,
+    )
+    .fetch_all(db)
+    .await
+    .unwrap_or(vec![])
+}