Allow pushing user-allocation membership to Keycloak

A Keycloak admin client has been added
When `activate_allocation` is called, the user is added
to a Keycloak group named after the project ID on the remote cluster.
If the user does not already exist in Keycloak, the case is ignored for now

When `deactivate_allocation` is called, the user is removed from the Keycloak group

Unit tests have been updated to remove dependancy on Keycloak

A comment in `validate_allocations` has been updated to
reflect the more restrictive validation behavior, where users on cluster projects
will be removed if they are not part of the Coldfront allocation (rather
than if they are registered on Coldfront at all). This is relevant
for functional tests for this new feature.

Author: QuanMPhm

.github/workflows/test-functional-microshift.yaml

@@ -35,6 +35,10 @@ jobs:
         run: |
           bash ./ci/setup-oc-client.sh
 
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
       - name: Install Microshift
         run: |
           ./ci/microshift.sh

.github/workflows/test-functional-microstack.yaml

@@ -18,6 +18,10 @@ jobs:
         with:
           python-version: 3.12
 
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
       - name: Install ColdFront and plugin
         run: |
           python -m pip install --upgrade pip

ci/run_functional_tests_openshift.sh

@@ -5,6 +5,12 @@
 # Tests expect the resource to be name Devstack
 set -xe
 
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="admin-cli"
+export KEYCLOAK_ADMIN_USER="admin"
+export KEYCLOAK_ADMIN_PASSWORD="nomoresecret"
+
 export OPENSHIFT_MICROSHIFT_TOKEN="$(oc create token -n onboarding onboarding-serviceaccount)"
 export OPENSHIFT_MICROSHIFT_VERIFY="false"
 

ci/run_functional_tests_openstack.sh

@@ -5,6 +5,12 @@
 # Tests expect the resource to be name Devstack
 set -xe
 
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="admin-cli"
+export KEYCLOAK_ADMIN_USER="admin"
+export KEYCLOAK_ADMIN_PASSWORD="nomoresecret"
+
 export CREDENTIAL_NAME=$(openssl rand -base64 12)
 
 export OPENSTACK_DEVSTACK_APPLICATION_CREDENTIAL_SECRET=$(

ci/setup-keycloak.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xe
+
+sudo docker run -d --name keycloak \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \
+    -p 8080:8080 \
+    -p 8443:8443 \
+    quay.io/keycloak/keycloak:25.0 start-dev

requirements.txt

@@ -8,3 +8,4 @@ python-keystoneclient
 python-novaclient
 python-neutronclient
 python-swiftclient
+requests

src/coldfront_plugin_cloud/base.py

@@ -1,11 +1,14 @@
 import abc
 import functools
 from typing import NamedTuple
+import logging
 
 from coldfront.core.allocation import models as allocation_models
 from coldfront.core.resource import models as resource_models
 
-from coldfront_plugin_cloud import attributes
+from coldfront_plugin_cloud import attributes, kc_client
+
+logger = logging.getLogger(__name__)
 
 
 class ResourceAllocator(abc.ABC):
@@ -25,11 +28,30 @@ def __init__(
         self.resource = resource
         self.allocation = allocation
 
+    @functools.cached_property
+    def kc_admin_client(self):
+        return kc_client.KeyCloakAPIClient()
+
     def get_or_create_federated_user(self, username):
         if not (user := self.get_federated_user(username)):
             user = self.create_federated_user(username)
         return user
 
+    def assign_role_on_user(self, username, project_id):
+        self.kc_admin_client.create_group(project_id)
+        if user_id := self.kc_admin_client.get_user_id(username):
+            group_id = self.kc_admin_client.get_group_id(project_id)
+            self.kc_admin_client.add_user_to_group(user_id, group_id)
+        else:
+            logger.warning(
+                f"User {username} not found in Keycloak, cannot add to group."
+            )
+
+    def remove_role_from_user(self, username, project_id):
+        user_id = self.kc_admin_client.get_user_id(username)
+        group_id = self.kc_admin_client.get_group_id(project_id)
+        self.kc_admin_client.remove_user_from_group(user_id, group_id)
+
     @functools.cached_property
     def auth_url(self):
         return self.resource.get_attribute(attributes.RESOURCE_AUTH_URL).rstrip("/")
@@ -69,11 +91,3 @@ def create_federated_user(self, unique_id):
     @abc.abstractmethod
     def get_federated_user(self, unique_id):
         pass
-
-    @abc.abstractmethod
-    def assign_role_on_user(self, username, project_id):
-        pass
-
-    @abc.abstractmethod
-    def remove_role_from_user(self, username, project_id):
-        pass

src/coldfront_plugin_cloud/kc_client.py

@@ -0,0 +1,86 @@
+import os
+import functools
+
+import requests
+
+
+class KeyCloakAPIClient:
+    def __init__(self):
+        self.base_url = os.getenv("KEYCLOAK_BASE_URL")
+        self.realm = os.getenv("KEYCLOAK_REALM")
+        self.admin_user = os.getenv("KEYCLOAK_ADMIN_USER")
+        self.admin_password = os.getenv("KEYCLOAK_ADMIN_PASSWORD")
+        self.client_id = os.getenv("KEYCLOAK_CLIENT_ID", "admin-cli")
+
+        self.token_url = (
+            f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token"
+        )
+
+    @functools.cached_property
+    def api_client(self):
+        params = {
+            "grant_type": "password",
+            "client_id": self.client_id,
+            "username": self.admin_user,
+            "password": self.admin_password,
+            "scope": "openid",
+        }
+        r = requests.post(self.token_url, data=params).json()
+        headers = {
+            "Authorization": ("Bearer %s" % r["access_token"]),
+            "Content-Type": "application/json",
+        }
+        session = requests.session()
+        session.headers.update(headers)
+        return session
+
+    def create_group(self, group_name):
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        payload = {"name": group_name}
+        response = self.api_client.post(url, json=payload)
+
+        # If group already exists, ignore and move on
+        if response.status_code not in (201, 409):
+            response.raise_for_status()
+
+    def create_user(self, cf_username):
+        """Helper function to create user in Keycloak, for testing purposes only"""
+        url = f"{self.base_url}/admin/realms/{self.realm}/users"
+        payload = {
+            "username": cf_username,
+            "enabled": True,
+            "email": cf_username,
+        }
+        r = self.api_client.post(url, json=payload)
+        r.raise_for_status()
+
+    def get_group_id(self, group_name) -> str | None:
+        """Return None if group not found"""
+        query = f"search={group_name}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups?{query}"
+        r = self.api_client.get(url).json()
+        return r[0]["id"] if r else None
+
+    def get_user_id(self, cf_username) -> str | None:
+        """Return None if user not found"""
+        # TODO (Quan): Confirm that Coldfront usernames map to Keycloak emails, not email, or something else?
+        query = f"email={cf_username}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/users?{query}"
+        r = self.api_client.get(url).json()
+        return r[0]["id"] if r else None
+
+    def add_user_to_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.put(url)
+        r.raise_for_status()
+
+    def remove_user_from_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.delete(url)
+        r.raise_for_status()
+
+    def get_user_groups(self, user_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        return [group["name"] for group in r.json()]

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -50,15 +50,15 @@ def sync_users(project_id, allocation, allocator, apply):
                 if apply:
                     tasks.add_user_to_allocation(coldfront_user.pk)
 
-        # remove users that are in the resource but not in coldfront
+        # remove users that are in the resource but not in coldfront allocation
         users = set(
             [coldfront_user.user.username for coldfront_user in coldfront_users]
         )
         for allocation_user in allocation_users:
             if allocation_user not in users:
                 failed_validation = True
                 logger.warn(
-                    f"{allocation_user} exists in the resource {project_id} but not in coldfront"
+                    f"{allocation_user} exists in the resource {project_id} but not in coldfront allocation"
                 )
                 if apply:
                     allocator.remove_role_from_user(allocation_user, project_id)

src/coldfront_plugin_cloud/openshift.py

@@ -253,6 +253,8 @@ def assign_role_on_user(self, username, project_id):
             # Role already exists, ignore
             pass
 
+        super().assign_role_on_user(username, project_id)
+
     def remove_role_from_user(self, username, project_id):
         """Remove a role from a user in a project using direct OpenShift API calls"""
         try:
@@ -275,6 +277,8 @@ def remove_role_from_user(self, username, project_id):
             # Rolebinding doesn't exist, nothing to remove
             pass
 
+        super().remove_role_from_user(username, project_id)
+
     def _create_project(self, project_name, project_id):
         pi_username = self.allocation.project.pi.username