Allow using validate_allocations to validate Keycloak group memberships

.github/workflows/test-functional-keycloak.yaml

@@ -0,0 +1,35 @@
+name: test-functional-keycloak
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.12
+
+      - name: Upgrade and install packages
+        run: |
+          bash ./ci/setup-ubuntu.sh
+
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
+      - name: Install ColdFront and plugin
+        run: |
+          ./ci/setup.sh
+
+      - name: Run functional tests
+        run: |
+          ./ci/run_functional_tests_keycloak.sh

ci/run_functional_tests_keycloak.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -xe
+
+if [[ ! "${CI}" == "true" ]]; then
+    source /tmp/coldfront_venv/bin/activate
+fi
+
+export DJANGO_SETTINGS_MODULE="local_settings"
+export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="coldfront"
+export KEYCLOAK_CLIENT_SECRET="nomoresecret"
+
+coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.keycloak
+coverage report

ci/setup-keycloak.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -xe
+
+sudo docker rm -f keycloak
+
+sudo docker run -d --name keycloak \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \
+    -p 8080:8080 \
+    -p 8443:8443 \
+    quay.io/keycloak/keycloak:25.0 start-dev
+
+# wait for keycloak to be ready
+until curl -fsS http://localhost:8080/realms/master; do
+    echo "Waiting for Keycloak to be ready..."
+    sleep 5
+done
+
+# Create client and add admin role to client's service account
+ACCESS_TOKEN=$(curl -X POST "http://localhost:8080/realms/master/protocol/openid-connect/token" \
+     -d "username=admin" \
+     -d "password=nomoresecret" \
+     -d "grant_type=password" \
+     -d "client_id=admin-cli" \
+     -d "scope=openid" \
+| jq -r '.access_token')
+
+
+curl -X POST "http://localhost:8080/admin/realms/master/clients" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "clientId": "coldfront",
+        "secret": "nomoresecret",
+        "redirectUris": ["http://localhost:8080/*"],
+        "serviceAccountsEnabled": true
+    }'
+
+COLDFRONT_CLIENT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients?clientId=coldfront" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.[0].id')
+
+
+COLDFRONT_SERVICE_ACCOUNT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients/$COLDFRONT_CLIENT_ID/service-account-user" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+| jq -r '.id')
+
+ADMIN_ROLE_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/roles/admin" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.id')
+
+# Add admin role to the service account user
+curl -X POST "http://localhost:8080/admin/realms/master/users/$COLDFRONT_SERVICE_ACCOUNT_ID/role-mappings/realm" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '[
+            {
+                "id": "'$ADMIN_ROLE_ID'",
+                "name": "admin"
+            }
+        ]'

requirements.txt

@@ -12,3 +12,4 @@ python-novaclient
 python-neutronclient
 python-swiftclient
 pytz
+requests

src/coldfront_plugin_cloud/attributes.py

@@ -24,7 +24,6 @@ class CloudAllocationAttribute:
 RESOURCE_API_URL = "OpenShift API Endpoint URL"
 RESOURCE_IDENTITY_NAME = "OpenShift Identity Provider Name"
 RESOURCE_ROLE = "Role for User in Project"
-RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
 
 RESOURCE_FEDERATION_PROTOCOL = "OpenStack Federation Protocol"
 RESOURCE_IDP = "OpenStack Identity Provider"
@@ -35,6 +34,8 @@ class CloudAllocationAttribute:
 
 RESOURCE_EULA_URL = "EULA URL"
 RESOURCE_CLUSTER_NAME = "Internal Cluster Name"
+RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
+RESOURCE_KEYCLOAK_GROUP_TEMPLATE = "Template String for Keycloak Group Names"
 
 RESOURCE_ATTRIBUTES = [
     CloudResourceAttribute(name=RESOURCE_AUTH_URL),
@@ -45,6 +46,7 @@ class CloudAllocationAttribute:
     CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_ROLE),
     CloudResourceAttribute(name=RESOURCE_QUOTA_RESOURCES),
+    CloudResourceAttribute(name=RESOURCE_KEYCLOAK_GROUP_TEMPLATE),
     CloudResourceAttribute(name=RESOURCE_USER_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_EULA_URL),
     CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK),

src/coldfront_plugin_cloud/kc_client.py

@@ -0,0 +1,118 @@
+import os
+import functools
+
+import requests
+from pydantic import BaseModel, ConfigDict, RootModel
+
+
+class KeyCloakGroup(BaseModel):
+    """Keycloak group response model"""
+
+    model_config = ConfigDict(extra="allow")
+    id: str
+    name: str
+
+
+class GroupResponse(RootModel):
+    """Wrapper for group list responses"""
+
+    root: list[KeyCloakGroup]
+
+
+class KeyCloakUser(BaseModel):
+    """Keycloak user response model"""
+
+    model_config = ConfigDict(extra="allow")
+    id: str
+    username: str
+
+
+class UserResponse(RootModel):
+    """Wrapper for user list responses"""
+
+    root: list[KeyCloakUser]
+
+
+class KeyCloakAPIClient:
+    def __init__(self):
+        self.base_url = os.getenv("KEYCLOAK_BASE_URL")
+        self.realm = os.getenv("KEYCLOAK_REALM")
+        self.client_id = os.getenv("KEYCLOAK_CLIENT_ID")
+        self.client_secret = os.getenv("KEYCLOAK_CLIENT_SECRET")
+
+        self.token_url = (
+            f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token"
+        )
+
+    @functools.cached_property
+    def api_client(self):
+        params = {
+            "grant_type": "client_credentials",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+        }
+        r = requests.post(self.token_url, data=params)
+        r.raise_for_status()
+        headers = {
+            "Authorization": ("Bearer %s" % r.json()["access_token"]),
+            "Content-Type": "application/json",
+        }
+        session = requests.session()
+        session.headers.update(headers)
+        return session
+
+    def create_group(self, group_name):
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        payload = {"name": group_name}
+        response = self.api_client.post(url, json=payload)
+
+        # If group already exists, ignore and move on
+        if response.status_code not in (201, 409):
+            response.raise_for_status()
+
+    def get_group_id(self, group_name) -> str | None:
+        """Return None if group not found"""
+        query = {
+            "search": group_name,
+            "exact": "true",
+        }
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        r = self.api_client.get(url, params=query)
+        r.raise_for_status()
+        groups = GroupResponse.model_validate(r.json())
+        return groups.root[0].id if groups.root else None
+
+    def get_user_id(self, cf_username) -> str | None:
+        """Return None if user not found"""
+        # (Quan) Coldfront usernames map to Keycloak usernames
+        # https://github.com/nerc-project/coldfront-plugin-cloud/pull/249#discussion_r2953393852
+        query = {"username": cf_username, "exact": "true"}
+        url = f"{self.base_url}/admin/realms/{self.realm}/users"
+        r = self.api_client.get(url, params=query)
+        r.raise_for_status()
+        users = UserResponse.model_validate(r.json())
+        return users.root[0].id if users.root else None
+
+    def add_user_to_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.put(url)
+        r.raise_for_status()
+
+    def remove_user_from_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.delete(url)
+        r.raise_for_status()
+
+    def get_user_groups(self, user_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        groups = GroupResponse.model_validate(r.json())
+        return [group.name for group in groups.root]
+
+    def get_group_members(self, group_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups/{group_id}/members"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        users = UserResponse.model_validate(r.json())
+        return [user.username for user in users.root]

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -3,11 +3,13 @@
 from coldfront_plugin_cloud import attributes
 from coldfront_plugin_cloud import utils
 from coldfront_plugin_cloud import tasks
+from coldfront_plugin_cloud import signals
 
 from django.core.management.base import BaseCommand
 from coldfront.core.resource.models import Resource
 from coldfront.core.allocation.models import (
     Allocation,
+    AllocationUser,
 )
 from keystoneauth1.exceptions import http
 from kubernetes.dynamic import exceptions as k8s_exceptions
@@ -46,6 +48,45 @@ def check_institution_specific_code(self, allocation, apply):
                 utils.set_attribute_on_allocation(allocation, attr, "N/A")
                 logger.warning(f'Attribute "{attr}" added to allocation {alloc_str}')
 
+    def validate_keycloak_group_memberships(self, allocation: Allocation, apply: bool):
+        kc_client = tasks.get_kc_client()
+        resource = allocation.resources.first()
+        group_template = resource.get_attribute(
+            attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE
+        )
+        if not group_template:
+            logger.info(
+                f"Keycloak enabled but no group name template specified for resource {resource.name}. Skipping validation"
+            )
+            return
+
+        allocation_users: list[AllocationUser] = allocation.allocationuser_set.all()
+        allocation_usernames = set(au.user.username for au in allocation_users)
+
+        group_name = tasks._get_keycloak_group_name(allocation, group_template)
+        group_id = kc_client.get_group_id(group_name)
+        if group_id:
+            group_usernames = set(kc_client.get_group_members(group_id))
+        else:
+            group_usernames = set()
+
+        to_add = [
+            au for au in allocation_users if au.user.username not in group_usernames
+        ]
+        to_remove = group_usernames - allocation_usernames
+
+        for au in to_add:
+            logger.info(
+                f"Adding user {au.user.username} to Keycloak group {group_name}"
+            )
+            if apply:
+                tasks.add_user_to_keycloak(au.pk)
+        for username in to_remove:
+            logger.info(f"Removing user {username} from Keycloak group {group_name}")
+            if apply:
+                user_id = kc_client.get_user_id(username)
+                kc_client.remove_user_from_group(user_id, group_id)
+
     def handle(self, *args, **options):
         for resource_name in self.PLUGIN_RESOURCE_NAMES:
             resource = Resource.objects.filter(resource_type__name=resource_name)
@@ -70,6 +111,11 @@ def handle(self, *args, **options):
                     )
                     continue
 
+                if signals.is_keycloak_enabled():
+                    self.validate_keycloak_group_memberships(
+                        allocation, options["apply"]
+                    )
+
                 # Check project exists in remote cluster
                 try:
                     allocator.get_project(project_id)

src/coldfront_plugin_cloud/signals.py

@@ -6,8 +6,10 @@
 from coldfront_plugin_cloud.tasks import (
     activate_allocation,
     add_user_to_allocation,
+    add_user_to_keycloak,
     disable_allocation,
     remove_user_from_allocation,
+    remove_user_from_keycloak,
 )
 from coldfront.core.allocation.signals import (
     allocation_activate,
@@ -25,6 +27,10 @@ def is_async():
     return os.getenv("REDIS_HOST")
 
 
+def is_keycloak_enabled():
+    return os.getenv("KEYCLOAK_BASE_URL")
+
+
 @receiver(allocation_activate)
 @receiver(allocation_change_approved)
 def activate_allocation_receiver(sender, **kwargs):
@@ -48,11 +54,18 @@ def activate_allocation_user_receiver(sender, **kwargs):
     allocation_user_pk = kwargs.get("allocation_user_pk")
     if is_async():
         async_task(add_user_to_allocation, allocation_user_pk)
+        if is_keycloak_enabled():
+            async_task(add_user_to_keycloak, allocation_user_pk)
     else:
         add_user_to_allocation(allocation_user_pk)
+        if is_keycloak_enabled():
+            add_user_to_keycloak(allocation_user_pk)
 
 
 @receiver(allocation_remove_user)
 def allocation_remove_user_receiver(sender, **kwargs):
     allocation_user_pk = kwargs.get("allocation_user_pk")
     remove_user_from_allocation(allocation_user_pk)
+
+    if is_keycloak_enabled():
+        remove_user_from_keycloak(allocation_user_pk)