Skynet/ldap_backend
6
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch '#10_wolves_database' into 'main'

Browse source 
feat: added the function to get teh data and updated teh times tehy run at

See merge request compsoc1/skynet/ldap/backend!8
This commit is contained in:
Brendan Golden 2023-08-05 21:36:30 +00:00
commit ae7b73176c
6 changed files with 287 additions and 201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.toml
View file

@ -4,6 +4,8 @@ version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[[bin]]
name = "update_data"
[[bin]]
name = "update_groups"

132
flake.nix
View file

@ -67,6 +67,44 @@
USERS_LIFETIME = lib.strings.concatStringsSep "," cfg.users.lifetime;
USERS_BANNED = lib.strings.concatStringsSep "," cfg.users.banned;
};
service_name = script: lib.strings.sanitizeDerivationName("${cfg.prefix}${cfg.user}@${script}");
# oneshot scripts to run
serviceGenerator = builtins.mapAttrs (script: time: nameValuePair (service_name script) {
description = "Service for ${desc} ${script}";
wantedBy = [ ];
after = [ "network-online.target" ];
environment = environment_config;
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
ExecStart = "${self.defaultPackage."${system}"}/bin/${script}";
EnvironmentFile = "${cfg.envFile}";
};
}) scripts;
# each timer will run the above service
timerGenerator = builtins.mapAttrs (script: time: nameValuePair (service_name script) {
description = "Timer for ${desc} ${script}";
wantedBy = [ "timers.target" ];
partOf = [ "${service_name script}.service" ];
timerConfig = {
OnCalendar = time;
Unit = "${service_name script}.service";
Persistent = true;
};
}) scripts;
# modify these
scripts = {
"new_data" = "*:0,15,30,45";
"new_users" = "*:5,20,35,50";
"update_groups" = "*:10";
};
in {
options.services."${package_name}" = {
enable = mkEnableOption "enable ${package_name}";
@ -144,6 +182,13 @@
default = "/etc/skynet_${package_name}";
description = "The home for the user";
};
prefix = mkOption rec {
type = types.str;
default = "skynet_";
example = default;
description = "The prefix used to name service/folders";
};
};
@ -158,75 +203,28 @@
group = "${cfg.user}";
};
systemd.services."${cfg.user}" = {
description = desc;
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ ];
environment = environment_config;
serviceConfig = {
# because we are storing data we need a home for it
User = "${cfg.user}";
Group = "${cfg.user}";
Restart = "always";
ExecStart = "${self.defaultPackage."${system}"}/bin/${package_name}";
EnvironmentFile = "${cfg.envFile}";
systemd.services = {
# main service
"${cfg.user}" = {
description = desc;
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ ];
environment = environment_config;
serviceConfig = {
# because we are storing data we need a home for it
User = "${cfg.user}";
Group = "${cfg.user}";
Restart = "always";
ExecStart = "${self.defaultPackage."${system}"}/bin/${package_name}";
EnvironmentFile = "${cfg.envFile}";
};
};
};
} // serviceGenerator;
# for updating the data
systemd.services."${cfg.user}_update" = {
description = "${desc} Update groups";
wantedBy = [ ];
after = [ "network-online.target" ];
environment = environment_config;
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
ExecStart = "${self.defaultPackage."${system}"}/bin/update_groups";
EnvironmentFile = "${cfg.envFile}";
};
};
systemd.timers."${cfg.user}_update" = {
description = "Run the update script for ${desc}";
wantedBy = [ "timers.target" ];
partOf = [ "${cfg.user}_update.service" ];
timerConfig = {
# every hour
OnCalendar = "*-*-* *:00:00";
Unit = "${cfg.user}_update.service";
};
};
# for new users
systemd.services."${cfg.user}_new_users" = {
description = "${desc} Get new users";
wantedBy = [ ];
after = [ "network-online.target" ];
environment = environment_config;
serviceConfig = {
Type = "oneshot";
DynamicUser = true;
ExecStart = "${self.defaultPackage."${system}"}/bin/new_users";
EnvironmentFile = "${cfg.envFile}";
};
};
systemd.timers."${cfg.user}_new_users" = {
description = "Run the new users script for ${desc}";
wantedBy = [ "timers.target" ];
partOf = [ "${cfg.user}_new_users.service" ];
timerConfig = {
# every 15 min
OnCalendar = "*:0/15";
Unit = "${cfg.user}_new_users.service";
};
};
# timers to run the above services
systemd.timers = timerGenerator;
};

44
src/bin/new_users.rs
View file

@ -4,7 +4,7 @@ use lettre::{
Message, SmtpTransport, Transport,
};
use maud::html;
use skynet_ldap_backend::{db_init, get_config, get_now_iso, random_string, read_csv, Accounts, AccountsNew, Config, Record};
use skynet_ldap_backend::{db_init, get_config, get_now_iso, get_wolves, random_string, AccountWolves, Accounts, AccountsNew, Config};
use sqlx::{Pool, Sqlite};
#[async_std::main]
@ -12,31 +12,29 @@ async fn main() {
let config = get_config();
let db = db_init(&config).await.unwrap();
if let Ok(records) = read_csv(&config) {
for record in records {
// skynet emails not permitted
if record.email.trim().ends_with("@skynet.ie") {
continue;
}
for record in get_wolves(&db).await {
// skynet emails not permitted
if record.email.trim().ends_with("@skynet.ie") {
continue;
}
// check if the email is already in the db
if !check(&db, &record.email).await {
continue;
}
// check if the email is already in the db
if !check(&db, &record.email).await {
continue;
}
// generate a auth key
let auth = random_string(50);
// generate a auth key
let auth = random_string(50);
match send_mail(&config, &record, &auth) {
Ok(_) => match save_to_db(&db, &record, &auth).await {
Ok(_) => {}
Err(e) => {
println!("Unable to save to db {} {e:?}", &record.email);
}
},
match send_mail(&config, &record, &auth) {
Ok(_) => match save_to_db(&db, &record, &auth).await {
Ok(_) => {}
Err(e) => {
println!("Unable to send mail to {} {e:?}", &record.email);
println!("Unable to save to db {} {e:?}", &record.email);
}
},
Err(e) => {
println!("Unable to send mail to {} {e:?}", &record.email);
}
}
}
@ -75,7 +73,7 @@ async fn check_pending(db: &Pool<Sqlite>, mail: &str) -> bool {
}
// using https://github.com/lettre/lettre/blob/57886c367d69b4d66300b322c94bd910b1eca364/examples/maud_html.rs
fn send_mail(config: &Config, record: &Record, auth: &str) -> Result<Response, lettre::transport::smtp::Error> {
fn send_mail(config: &Config, record: &AccountWolves, auth: &str) -> Result<Response, lettre::transport::smtp::Error> {
let recipient = &record.name_first;
let mail = &record.email;
let url_base = "https://sso.skynet.ie";
@ -180,7 +178,7 @@ fn send_mail(config: &Config, record: &Record, auth: &str) -> Result<Response, l
mailer.send(&email)
}
async fn save_to_db(db: &Pool<Sqlite>, record: &Record, auth: &str) -> Result<Option<AccountsNew>, sqlx::Error> {
async fn save_to_db(db: &Pool<Sqlite>, record: &AccountWolves, auth: &str) -> Result<Option<AccountsNew>, sqlx::Error> {
sqlx::query_as::<_, AccountsNew>(
"
INSERT OR REPLACE INTO accounts_new (mail, auth_code, date_iso, date_expiry, name_first, name_surname, id_student)

162
src/bin/update_data.rs Normal file
View file

@ -0,0 +1,162 @@
use ldap3::{LdapConn, Scope, SearchEntry};
use skynet_ldap_backend::{db_init, get_config, AccountWolves, Accounts, Config};
use sqlx::{Pool, Sqlite};
#[async_std::main]
async fn main() -> tide::Result<()> {
let config = get_config();
let db = db_init(&config).await.unwrap();
update_wolves(&config, &db).await;
update_ldap(&config, &db).await;
Ok(())
}
async fn update_wolves(config: &Config, db: &Pool<Sqlite>) {
let mut records = vec![];
if let Ok(accounts) = get_csv(config) {
for account in accounts {
records.push(AccountWolves::from(account));
}
}
for account in records {
update_account(db, &account).await;
}
}
async fn update_ldap(config: &Config, db: &Pool<Sqlite>) {
let mut ldap = LdapConn::new(&config.ldap_host).unwrap();
ldap.simple_bind(&config.ldap_admin, &config.ldap_admin_pw).unwrap().success().unwrap();
// use this to pre load a large chunk of data
if let Ok(x) = ldap.search("ou=users,dc=skynet,dc=ie", Scope::OneLevel, "(objectClass=*)", vec!["uid", "uidNumber", "skDiscord", "skMemberOf", "mail", "skID", "userPassword"]) {
if let Ok((rs, _res)) = x.success() {
for entry in rs {
let tmp = SearchEntry::construct(entry);
let mut tmp_account = Accounts {
user: "".to_string(),
uid: 0,
discord: None,
mail: "".to_string(),
student_id: "".to_string(),
enabled: false,
secure: false,
};
// pull out the required info
if tmp.attrs.contains_key("uid") && !tmp.attrs["uid"].is_empty() {
tmp_account.user = tmp.attrs["uid"][0].clone();
}
if tmp.attrs.contains_key("uidNumber") && !tmp.attrs["uidNumber"].is_empty() {
tmp_account.uid = tmp.attrs["uidNumber"][0].clone().parse().unwrap_or(0);
}
if tmp.attrs.contains_key("skDiscord") && !tmp.attrs["skDiscord"].is_empty() {
tmp_account.discord = Option::from(tmp.attrs["skDiscord"][0].clone());
}
if tmp.attrs.contains_key("mail") && !tmp.attrs["mail"].is_empty() {
tmp_account.mail = tmp.attrs["mail"][0].clone();
}
if tmp.attrs.contains_key("skID") && !tmp.attrs["skID"].is_empty() {
tmp_account.student_id = tmp.attrs["skID"][0].clone();
}
if tmp.attrs.contains_key("skMemberOf") && !tmp.attrs["skMemberOf"].is_empty() && tmp.attrs["skMemberOf"].contains(&String::from("cn=skynet-users-linux,ou=groups,dc=skynet,dc=ie")) {
tmp_account.enabled = true;
}
if tmp.attrs.contains_key("userPassword") && !tmp.attrs["userPassword"].is_empty() {
tmp_account.secure = tmp.attrs["userPassword"][0].starts_with("{SSHA512}")
}
if !tmp_account.user.is_empty() {
sqlx::query_as::<_, Accounts>(
"
INSERT OR REPLACE INTO accounts (user, uid, discord, mail, student_id, enabled, secure)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
",
)
.bind(&tmp_account.user)
.bind(tmp_account.uid)
.bind(&tmp_account.discord)
.bind(&tmp_account.mail)
.bind(&tmp_account.student_id)
.bind(tmp_account.enabled)
.bind(tmp_account.secure)
.fetch_optional(db)
.await
.ok();
}
}
}
}
// done with ldap
ldap.unbind().unwrap();
}
#[derive(Debug, serde::Deserialize)]
struct RecordCSV {
#[serde(rename = "MemID")]
mem_id: String,
#[serde(rename = "Student Num")]
id_student: String,
#[serde(rename = "Contact Email")]
email: String,
#[serde(rename = "Expiry")]
expiry: String,
#[serde(rename = "First Name")]
name_first: String,
#[serde(rename = "Last Name")]
name_second: String,
}
impl From<RecordCSV> for AccountWolves {
fn from(input: RecordCSV) -> Self {
AccountWolves {
id_wolves: input.mem_id,
id_student: input.id_student,
email: input.email,
expiry: input.expiry,
name_first: input.name_first,
name_second: input.name_second,
}
}
}
fn get_csv(config: &Config) -> Result<Vec<RecordCSV>, Box<dyn std::error::Error>> {
let mut records: Vec<RecordCSV> = vec![];
if let Ok(mut rdr) = csv::Reader::from_path(format!("{}/{}", &config.home, &config.csv)) {
for result in rdr.deserialize() {
// Notice that we need to provide a type hint for automatic
// deserialization.
let record: RecordCSV = result?;
if record.mem_id.is_empty() {
continue;
}
records.push(record);
}
}
Ok(records)
}
async fn update_account(db: &Pool<Sqlite>, account: &AccountWolves) {
sqlx::query_as::<_, AccountWolves>(
"
INSERT OR REPLACE INTO accounts_wolves (id_wolves, id_student, email, expiry, name_first, name_second)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
",
)
.bind(&account.id_wolves)
.bind(&account.id_student)
.bind(&account.email)
.bind(&account.expiry)
.bind(&account.name_first)
.bind(&account.name_second)
.fetch_optional(db)
.await
.ok();
}

6
src/bin/update_groups.rs
View file

@ -1,5 +1,5 @@
use ldap3::{LdapConn, Mod};
use skynet_ldap_backend::{db_init, get_config, get_now_iso, read_csv, Accounts, Config};
use skynet_ldap_backend::{db_init, get_config, get_now_iso, get_wolves, Accounts, Config};
use sqlx::{Pool, Sqlite};
use std::{collections::HashSet, env, error::Error};
@ -116,9 +116,7 @@ async fn from_csv(config: &Config) -> Result<HashSet<String>, Box<dyn Error>> {
let mut uids = HashSet::new();
let records = read_csv(config)?;
for record in records {
for record in get_wolves(&db).await {
// only import users if it is actually active.
if record.expiry < get_now_iso(true) {
continue;

142
src/lib.rs
View file

@ -1,7 +1,6 @@
pub mod methods;
use chrono::{Datelike, SecondsFormat, Utc};
use dotenvy::dotenv;
use ldap3::{LdapConn, Scope, SearchEntry};
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use sqlx::{
sqlite::{SqliteConnectOptions, SqlitePoolOptions},
@ -14,6 +13,16 @@ use std::{
};
use tide::prelude::*;
#[derive(Debug, Deserialize, Serialize, sqlx::FromRow)]
pub struct AccountWolves {
pub id_wolves: String,
pub id_student: String,
pub email: String,
pub expiry: String,
pub name_first: String,
pub name_second: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, sqlx::FromRow)]
pub struct AccountsNew {
pub mail: String,
@ -44,6 +53,19 @@ pub async fn db_init(config: &Config) -> Result<Pool<Sqlite>, Error> {
.connect_with(SqliteConnectOptions::from_str(&format!("sqlite://{}", database))?.create_if_missing(true))
.await?;
sqlx::query(
"CREATE TABLE IF NOT EXISTS accounts_wolves (
id_wolves text primary key,
id_student text not null,
email text not null,
expiry text not null,
name_first text not null,
name_surname integer not null
)",
)
.execute(&pool)
.await?;
sqlx::query(
"CREATE TABLE IF NOT EXISTS accounts_new (
mail text primary key,
@ -83,8 +105,6 @@ pub async fn db_init(config: &Config) -> Result<Pool<Sqlite>, Error> {
.execute(&pool)
.await?;
update_accounts(&pool, config).await;
Ok(pool)
}
@ -176,111 +196,19 @@ pub fn get_config() -> Config {
config
}
async fn update_accounts(pool: &Pool<Sqlite>, config: &Config) {
let mut ldap = LdapConn::new(&config.ldap_host).unwrap();
ldap.simple_bind(&config.ldap_admin, &config.ldap_admin_pw).unwrap().success().unwrap();
// use this to pre load a large chunk of data
if let Ok(x) = ldap.search("ou=users,dc=skynet,dc=ie", Scope::OneLevel, "(objectClass=*)", vec!["uid", "uidNumber", "skDiscord", "skMemberOf", "mail", "skID", "userPassword"]) {
if let Ok((rs, _res)) = x.success() {
for entry in rs {
let tmp = SearchEntry::construct(entry);
let mut tmp_account = Accounts {
user: "".to_string(),
uid: 0,
discord: None,
mail: "".to_string(),
student_id: "".to_string(),
enabled: false,
secure: false,
};
// pull out the required info
if tmp.attrs.contains_key("uid") && !tmp.attrs["uid"].is_empty() {
tmp_account.user = tmp.attrs["uid"][0].clone();
}
if tmp.attrs.contains_key("uidNumber") && !tmp.attrs["uidNumber"].is_empty() {
tmp_account.uid = tmp.attrs["uidNumber"][0].clone().parse().unwrap_or(0);
}
if tmp.attrs.contains_key("skDiscord") && !tmp.attrs["skDiscord"].is_empty() {
tmp_account.discord = Option::from(tmp.attrs["skDiscord"][0].clone());
}
if tmp.attrs.contains_key("mail") && !tmp.attrs["mail"].is_empty() {
tmp_account.mail = tmp.attrs["mail"][0].clone();
}
if tmp.attrs.contains_key("skID") && !tmp.attrs["skID"].is_empty() {
tmp_account.student_id = tmp.attrs["skID"][0].clone();
}
if tmp.attrs.contains_key("skMemberOf") && !tmp.attrs["skMemberOf"].is_empty() && tmp.attrs["skMemberOf"].contains(&String::from("cn=skynet-users-linux,ou=groups,dc=skynet,dc=ie")) {
tmp_account.enabled = true;
}
if tmp.attrs.contains_key("userPassword") && !tmp.attrs["userPassword"].is_empty() {
tmp_account.secure = tmp.attrs["userPassword"][0].starts_with("{SSHA512}")
}
if !tmp_account.user.is_empty() {
sqlx::query_as::<_, Accounts>(
"
INSERT OR REPLACE INTO accounts (user, uid, discord, mail, student_id, enabled, secure)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
",
)
.bind(&tmp_account.user)
.bind(tmp_account.uid)
.bind(&tmp_account.discord)
.bind(&tmp_account.mail)
.bind(&tmp_account.student_id)
.bind(tmp_account.enabled)
.bind(tmp_account.secure)
.fetch_optional(pool)
.await
.ok();
}
}
}
}
// done with ldap
ldap.unbind().unwrap();
}
#[derive(Debug, serde::Deserialize)]
pub struct Record {
#[serde(rename = "MemID")]
pub mem_id: String,
#[serde(rename = "Student Num")]
pub id_student: String,
#[serde(rename = "Contact Email")]
pub email: String,
#[serde(rename = "Expiry")]
pub expiry: String,
#[serde(rename = "First Name")]
pub name_first: String,
#[serde(rename = "Last Name")]
pub name_second: String,
}
pub fn read_csv(config: &Config) -> Result<Vec<Record>, Box<dyn std::error::Error>> {
let mut records: Vec<Record> = vec![];
if let Ok(mut rdr) = csv::Reader::from_path(format!("{}/{}", &config.home, &config.csv)) {
for result in rdr.deserialize() {
// Notice that we need to provide a type hint for automatic
// deserialization.
let record: Record = result?;
if record.mem_id.is_empty() {
continue;
}
records.push(record);
}
}
Ok(records)
}
// from https://rust-lang-nursery.github.io/rust-cookbook/algorithms/randomness.html#create-random-passwords-from-a-set-of-alphanumeric-characters
pub fn random_string(len: usize) -> String {
thread_rng().sample_iter(&Alphanumeric).take(len).map(char::from).collect()
}
pub async fn get_wolves(db: &Pool<Sqlite>) -> Vec<AccountWolves> {
sqlx::query_as::<_, AccountWolves>(
r#"
SELECT *
FROM accounts_wolves
"#,
)
.fetch_all(db)
.await
.unwrap_or(vec![])
}