diff --git a/src/common/set_roles.rs b/src/common/set_roles.rs
index c51c52e..fbbc828 100644
--- a/src/common/set_roles.rs
+++ b/src/common/set_roles.rs
@@ -329,15 +329,11 @@ pub mod committee {
         None => {
           vec![]
         }
-        Some(x) => {
-          let mut tmp = x.to_owned();
-          if !tmp.is_empty() {
-            tmp.push(committee_member);
-          }
-          tmp
-        }
+        Some(x) => x.to_owned(),
       };
 
+      let on_committee = !roles_required.is_empty();
+
       let mut roles_rem = vec![];
       let mut roles_add = vec![];
       // get a list of all the roles to remove from someone
@@ -346,14 +342,25 @@ pub mod committee {
       for role in &roles_current {
         roles_current_id.push(role.id.to_owned());
         if !roles_required.contains(&role.id) {
-          roles_rem.push(role.id.to_owned());
+          if role.id == committee_member {
+            if !on_committee {
+              roles_rem.push(role.id.to_owned());
+            }
+          }
         }
       }
 
-      if !roles_required.is_empty() {
+      let has_committee_role = roles_current_id.contains(&committee_member);
+
+      if on_committee && !has_committee_role {
         // if there are committee roles then give the general purporse role
         roles_add.push(committee_member);
+      }
+      if !on_committee && has_committee_role {
+        roles_rem.push(committee_member);
+      }
 
+      if !roles_required.is_empty() {
         if let Some(x) = roles_db.get_mut(&0) {
           x.count += 1;
         }
@@ -372,8 +379,6 @@ pub mod committee {
       if !roles_add.is_empty() {
         // these roles are flavor roles, only there to make folks mentionable
         member.add_roles(&ctx, &roles_add).await.unwrap_or_default();
-      } else {
-        member.remove_roles(&ctx, &[committee_member]).await.unwrap_or_default();
       }
     }