automate project fairshare calculation

coldfront/core/resource/management/commands/add_resource_defaults.py

@@ -33,6 +33,7 @@ def handle(self, *args, **options):
             ('GPU Count', 'Int'),
             ('Features', 'Text'),
             ('slurm_integration', 'Text'),
+            ('NodeType', 'Text'),
             # UBCCR
             ('Core Count', 'Int'),
             # ('expiry_time', 'Int'),
@@ -63,6 +64,9 @@ def handle(self, *args, **options):
             )
 
         for resource_type, description in (
+            # FASRC
+            ('Supergroup', 'Compute Supergroup'),
+            # UBCCR
             ('Storage', 'Network Storage'),
             ('Storage Tier', 'Storage Tier',),
             ('Cloud', 'Cloud Computing'),

coldfront/core/resource/templates/resource_detail.html

@@ -205,7 +205,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
       {# {% if user_sync_dt %}#}
       {# <span class="float-right">Last Sync: {{user_sync_dt}}</span>#}
       {# {% endif %}#}
-      {% if 'Cluster' in resource.resource_type.name %}
+      {% if 'Cluster' in resource.resource_type.name or 'Supergroup' in resource.resource_type.name %}
         <div class="float-right">
           <a class="btn btn-danger" href="{% url 'resource-allocations-edit' resource.pk %}" role="button">
             <i class="fas fa-edit" aria-hidden="true"></i> Edit Resource Allocations
@@ -222,7 +222,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
               <tr>
                 <th scope="col">Project</th>
                 <th scope="col">Users</th>
-                {% if 'Cluster' in resource.resource_type.name %}
+                {% if 'Cluster' in resource.resource_type.name or 'Supergroup' in resource.resource_type.name %}
                   <th scope="col">CPU Hours</th>
                   <th scope="col">Percent Usage</th>
                   <th scope="col">RawShare</th>
@@ -256,7 +256,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
                     </a>
                   </td>
                   <td>{{ allocation.user_count }}</td>
-                  {% if 'Cluster' in resource.resource_type.name %}
+                  {% if 'Cluster' in resource.resource_type.name or 'Supergroup' in resource.resource_type.name %}
                     <td data-sort="{{allocation.usage}}" name="usage">
                       {{ allocation.usage}}
                     </td>

coldfront/plugins/ldap/signals.py

@@ -12,11 +12,13 @@
     project_reactivate_projectuser,
 )
 from coldfront.core.project.models import (
+    Project,
+    ProjectUser,
     ProjectUserRoleChoice,
     ProjectUserStatusChoice,
-    ProjectUser,
 )
-from coldfront.plugins.ldap.utils import LDAPConn
+from coldfront.core.resource.models import Resource
+from coldfront.plugins.ldap.utils import LDAPConn, LDAPException
 
 if 'coldfront.plugins.sftocf' in import_from_settings('INSTALLED_APPS', []):
     from coldfront.plugins.sftocf.signals import (
@@ -115,6 +117,83 @@ def update_new_project(sender, **kwargs):
                     extra={ 'category': 'database_change:ProjectUser', 'status': 'success' }
         )
 
+# @receiver(slurmrest_supergroup_membership_update)
+def update_supergroup_membership(sender, **kwargs):
+    """Update ColdFront Supergroup allocation list based on AD group membership
+    Supergroups are ColdFront Resources that correspond to slurm accounts and to AD groups.
+    They need to be linked to the ColdFront cluster Allocations that belong to the Projects that
+    correspond to the Supergroup's AD group members.
+    """
+    # get supergroup name from kwargs
+    supergroup_name = kwargs['supergroup_name']
+    # get corresponding AD group and supergroup Resource
+    try:
+        ad_conn = LDAPConn()
+        members = ad_conn.return_group_group_members(supergroup_name)
+    except Exception as e:
+        logger.exception(
+            "error encountered retrieving members and manager for Supergroup %s: %s",
+            supergroup_name, e
+        )
+        raise LDAPException(f"ldap error: {e}") from e
+
+    supergroup_obj = Resource.objects.get(name=supergroup_name)
+    # collect Projects corresponding to AD group membership
+    for member in members:
+        if not ad_conn.check_group_validity(member):
+            logger.warning(
+                "skipping invalid Supergroup %s member %s",
+                supergroup_name, member['sAMAccountName'][0]
+            )
+            continue
+        try:
+            project_obj = Project.objects.get(
+                title=member['sAMAccountName'][0],
+            )
+        except Exception as e:
+            logger.warning(
+                "issue retrieving Project for Supergroup %s member %s: %s",
+                supergroup_name, member['sAMAccountName'][0], e
+            )
+            continue
+        # get allocation corresponding to the Project and supergroup's parent resource
+        try:
+            allocation_obj = project_obj.allocation_set.get(
+                resources=supergroup_obj.parent_resource,
+            )
+        except Exception as e:
+            logger.warning(
+                "issue retrieving Allocation for Supergroup %s member Project %s: %s",
+                supergroup_name, project_obj.title, e
+            )
+            continue
+        # link allocation to supergroup if not already linked
+        if supergroup_obj not in allocation_obj.resources.all():
+            allocation_obj.resources.add(supergroup_obj)
+            logger.info(
+                "linked Supergroup %s to Allocation for Project %s",
+                supergroup_name, project_obj.title,
+                extra={ 'category': 'ldap:Supergroup', 'status': 'success' }
+            )
+
+    # remove allocations no longer linked to AD group members
+    for allocation in supergroup_obj.allocation_set.all():
+        project_title = allocation.project.title
+        keep_linked = True
+        if not ad_conn.check_group_validity({'sAMAccountName':[project_title]}):
+            keep_linked = False
+        # check if project_title is in membership
+        member_usernames = [m['sAMAccountName'][0] for m in members]
+        if project_title not in member_usernames:
+            keep_linked = False
+        if not keep_linked:
+            allocation.resources.remove(supergroup_obj)
+            logger.info(
+                "removed Supergroup %s from Allocation for Project %s",
+                supergroup_name, project_title,
+                extra={ 'category': 'ldap:Supergroup', 'status': 'success' }
+            )
+
 @receiver(project_filter_users_to_remove)
 def filter_project_users_to_remove(sender, **kwargs):
     users_to_remove = kwargs['users_to_remove']

coldfront/plugins/ldap/utils.py

@@ -88,6 +88,25 @@ def member_in_group(self, member_dn, group_dn):
         group = self.search_groups({'distinguishedName': group_dn, 'member': member_dn})
         return bool(group)
 
+    def check_group_validity(self, group_entry):
+        """Check if group entry is valid.
+
+        A valid group has a valid manager.
+        """
+        # run checks on the groups to ensure they have valid managers and aren't disabled
+        try:
+            _, manager = self.return_group_members_manager(group_entry['sAMAccountName'][0])
+        except LDAPException as e:
+            logger.warning('group %s invalid: %s', group_entry['sAMAccountName'][0], e)
+            return False
+        if not manager:
+            return False
+        if not user_valid(manager):
+            return False
+        return True
+
+
+
     def search(self, attr_search_dict, search_base, attributes=ALL_ATTRIBUTES):
         """Run an LDAP search.
 
@@ -172,14 +191,16 @@ def return_user_by_name(self, username, return_as='dict', attributes=ALL_ATTRIBU
             raise ValueError(f"no users returned for username {username}")
         return user[0]
 
-    def return_group_by_name(self, groupname, return_as='dict', attributes=ALL_ATTRIBUTES):
+    def return_group_by_name(self, samaccountname, return_as='dict', attributes=ALL_ATTRIBUTES):
         group = self.search_groups(
-            {"sAMAccountName": groupname}, return_as=return_as, attributes=attributes
+            {"sAMAccountName": samaccountname}, return_as=return_as, attributes=attributes
         )
         if len(group) > 1:
+            logger.error('multiple groups with same sAMAccountName: %s', samaccountname)
             raise ValueError("too many groups in value returned")
         if not group:
-            raise ValueError("no groups returned")
+            logger.error('no groups found with sAMAccountName: %s', samaccountname)
+            raise ValueError("no matching groups returned")
         return group[0]
 
     def add_user_to_group(self, user_name, group_name):
@@ -324,12 +345,15 @@ def return_group_members_manager(self, samaccountname):
         )
         if len(group_entries) > 1:
             logger.error('multiple groups with same sAMAccountName: %s', samaccountname)
-            return 'multiple groups with same sAMAccountName'
+            raise LDAPException('multiple groups with same sAMAccountName')
         if not group_entries:
             logger.error('no groups found with sAMAccountName: %s', samaccountname)
-            return 'no matching groups found'
+            raise LDAPException('no matching groups found')
         group_entry = group_entries[0]
-        return self.manager_members_from_group(group_entry)
+        try:
+            return self.manager_members_from_group(group_entry)
+        except LDAPException as e:
+            raise
 
     def manager_members_from_group(self, group_entry):
         group_dn = group_entry['distinguishedName'][0]
@@ -343,17 +367,26 @@ def manager_members_from_group(self, group_entry):
             group_manager_dn = group_entry['managedBy'][0]
         except Exception as e:
             logger.error('no manager specified for group %s', group_dn)
-            return 'no manager specified'
+            raise LDAPException('no manager specified') from e
         manager_attr_list = user_attr_list + ['memberOf']
         group_manager = self.search_users(
             {'distinguishedName': group_manager_dn}, attributes=manager_attr_list
         )
         logger.debug('group_manager:\n%s', group_manager)
         if not group_manager:
             logger.error('no ADUser manager found for group %s', group_dn)
-            return 'no ADUser manager found'
+            raise LDAPException('no ADUser manager found')
         return (group_members, group_manager[0])
 
+    def return_group_group_members(self, samaccountname):
+        """return group entries that are members of the specified group."""
+        logger.debug('return_group_group_members for Group %s', samaccountname)
+        group = self.return_group_by_name(samaccountname)
+        group_dn = group['distinguishedName'][0]
+        group_members = self.search_groups({'memberOf': group_dn})
+        logger.debug('group_members:\n%s', group_members)
+        return group_members
+
 
 def user_valid(user):
     return user['userAccountControl'][0] in [512, 66048]