add pulp services

playbooks/deploy.yaml

@@ -27,16 +27,22 @@
     httpd_client_ca_certificate: "{{ certificates_ca_directory }}/certs/ca.crt"
     httpd_server_certificate: "{{ certificates_ca_directory }}/certs/{{ certificates_server }}.crt"
     httpd_server_key: "{{ certificates_ca_directory }}/private/{{ certificates_server }}.key"
+    pulp_db_password: "CHANGEME"
+    content_origin: "https://{{ ansible_fqdn }}"
     postgresql_databases:
       - name: candlepin
         owner: candlepin
       - name: foreman
         owner: foreman
+      - name: pulp
+        owner: pulp
     postgresql_users:
       - name: candlepin
         password: "{{ candlepin_db_password }}"
       - name: foreman
         password: "{{ foreman_db_password }}"
+      - name: pulp
+        password: "{{ pulp_db_password }}"
     postgresql_hba_entries:
       - { type: local, database: all, user: postgres, auth_method: ident }
       - { type: local, database: all, user: all, auth_method: ident }
@@ -47,9 +53,9 @@
   roles:
     - certificates
     - geerlingguy.postgresql
+    - redis
     - candlepin
     - httpd
     - pulp
     - foreman_proxy
-    - redis
     - foreman

roles/httpd/defaults/main.yml

@@ -1,4 +1,4 @@
 httpd_ssl_dir: /etc/pki/httpd
-httpd_pulp_api_backend: http://localhost:8080
-httpd_pulp_content_backend: http://localhost:8080
+httpd_pulp_api_backend: http://localhost:24817
+httpd_pulp_content_backend: http://localhost:24816
 httpd_foreman_backend: http://localhost:3000

roles/pulp/defaults/main.yaml

@@ -1,10 +1,18 @@
 ---
-pulp_image: quay.io/pulp/pulp:stable
-pulp_ports:
-  - "8080:80"
+pulp_image: quay.io/pulp/pulp-minimal:stable
+pulp_api_image: "{{ pulp_image }}"
+pulp_content_image: "{{ pulp_image }}"
+pulp_worker_image: "{{ pulp_image }}"
+
+pulp_api_ports:
+  - "24817:80"
+pulp_content_ports:
+  - "24816:80"
+pulp_worker_count: 2
+
 pulp_volumes:
-  - /var/lib/pulp/settings:/etc/pulp:Z
-  - /var/lib/pulp/pulp_storage:/var/lib/pulp:Z
-  - /var/lib/pulp/pgsql:/var/lib/pgsql:Z
-  - /var/lib/pulp/containers:/var/lib/containers:Z
-pulp_container_name: pulp
+  - /var/lib/pulp:/var/lib/pulp
+
+pulp_api_container_name: pulp-api
+pulp_content_container_name: pulp-content
+pulp_worker_container_name: pulp-worker

roles/pulp/tasks/main.yaml

@@ -1,6 +1,16 @@
-- name: Pull the Pulp container image
+- name: Pull the Pulp API container image
   containers.podman.podman_image:
-    name: "{{ pulp_image }}"
+    name: "{{ pulp_api_image }}"
+    state: present
+
+- name: Pull the Pulp Content container image
+  containers.podman.podman_image:
+    name: "{{ pulp_content_image }}"
+    state: present
+
+- name: Pull the Pulp Worker container image
+  containers.podman.podman_image:
+    name: "{{ pulp_worker_image }}"
     state: present
 
 - name: Create Pulp storage
@@ -10,44 +20,155 @@
     mode: "0755"
   loop: "{{ pulp_volumes }}"
 
+- name: Create Pulp storage subdirs
+  ansible.builtin.file:
+    path: "/var/lib/pulp/{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - tmp
+    - assets
+
 - name: Create settings config secret
   containers.podman.podman_secret:
     state: present
     name: pulp-settings-py
     data: "{{ lookup('ansible.builtin.template', 'settings.py.j2') }}"
 
-- name: Deploy Pulp Container
+- name: Generate database symmetric key
+  ansible.builtin.command: "bash -c 'openssl rand -base64 32 | tr \"+/\" \"-_\" > /var/lib/pulp/database_fields.symmetric.key'"
+  args:
+    creates: /var/lib/pulp/database_fields.symmetric.key
+
+- name: Load database symmetric key
+  ansible.builtin.slurp:
+    src: /var/lib/pulp/database_fields.symmetric.key
+  register: pulp_key
+
+- name: Create database symmetric key secret
+  containers.podman.podman_secret:
+    state: present
+    name: pulp-symmetric-key
+    data: "{{ pulp_key['content'] | b64decode }}"
+
+- name: Wait for PostgreSQL to be ready
+  ansible.builtin.wait_for:
+    host: "localhost"
+    port: 5432
+    timeout: 300
+
+- name: Deploy Pulp API Container
   containers.podman.podman_container:
-    name: "{{ pulp_container_name }}"
-    image: "{{ pulp_image }}"
+    name: "{{ pulp_api_container_name }}"
+    image: "{{ pulp_api_image }}"
     state: quadlet
-    ports: "{{ pulp_ports }}"
+    command: pulp-api
+    network: host
     volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
     secrets:
       - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
     quadlet_options:
       - |
         [Install]
         WantedBy=default.target
+        Wants=postgresql.service
+        [Service]
+        Restart=always
+        RestartSec=3
+
+- name: Deploy Pulp Content Container
+  containers.podman.podman_container:
+    name: "{{ pulp_content_container_name }}"
+    image: "{{ pulp_content_image }}"
+    state: quadlet
+    command: pulp-content
+    network: host
+    volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
+    secrets:
+      - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
+    quadlet_options:
+      - |
+        [Install]
+        WantedBy=default.target
+        Wants=postgresql.service
+        [Service]
+        Restart=always
+        RestartSec=3
+
+- name: Deploy Pulp Worker Container
+  containers.podman.podman_container:
+    name: "{{ pulp_worker_container_name }}"
+    image: "{{ pulp_worker_image }}"
+    state: quadlet
+    command: pulp-worker
+    network: host
+    volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
+    secrets:
+      - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
+    quadlet_options:
+      - |
+        [Install]
+        WantedBy=default.target
+        Wants=postgresql.service
+        [Service]
+        Restart=always
+        RestartSec=3
 
 - name: Run daemon reload to make Quadlet create the service files
   ansible.builtin.systemd:
     daemon_reload: true
 
-- name: Start the Pulp Service
+- name: Migrate the Pulp database
+  containers.podman.podman_container:
+    name: pulpcore-manager-migrate
+    image: "{{ pulp_api_image }}"
+    command: pulpcore-manager migrate --noinput
+    network: host
+    secrets:
+      - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
+
+- name: Start the Pulp API services
   ansible.builtin.systemd:
-    name: pulp
+    name: pulp-api
     enabled: true
-    state: restarted
+    state: started
 
-- name: Wait for Pulp service to be accessible
+- name: Wait for Pulp API service to be accessible
   ansible.builtin.wait_for:
     host: "{{ ansible_hostname }}"
-    port: 8080
+    port: 24817
     timeout: 300
 
+- name: Start the Pulp Content services
+  ansible.builtin.systemd:
+    name: pulp-content
+    enabled: true
+    state: started
+
+- name: Wait for Pulp Content service to be accessible
+  ansible.builtin.wait_for:
+    host: "{{ ansible_hostname }}"
+    port: 24816
+    timeout: 600
+
+- name: Start the Pulp Worker service
+  ansible.builtin.systemd:
+    name: pulp-worker
+    enabled: true
+    state: started
+
 # Only needed until we have cert auth configured
 - name: Set Pulp admin password
   containers.podman.podman_container_exec:
-    name: "{{ pulp_container_name }}"
+    name: "{{ pulp_api_container_name }}"
     command: pulpcore-manager reset-admin-password --password CHANGEME

roles/pulp/templates/settings.py.j2

@@ -1,7 +1,20 @@
-CONTENT_ORIGIN="http://{{ ansible_hostname }}:8080"
+CONTENT_ORIGIN="http://{{ ansible_fqdn }}:24816"
 CACHE_ENABLED=True
 REDIS_HOST="localhost"
 REDIS_PORT=6379
+REDIS_DB=8
+
+DATABASES = {
+	'default': {
+		'ENGINE': 'django.db.backends.postgresql',
+		'NAME': 'pulp',
+		'USER': 'pulp',
+		'PASSWORD': '{{ pulp_db_password }}',
+		'HOST': 'localhost',
+		'PORT': '5432',
+	}
+}
+
 AUTHENTICATION_BACKENDS=['pulpcore.app.authentication.PulpNoCreateRemoteUserBackend']
 REMOTE_USER_ENVIRON_NAME="HTTP_REMOTE_USER"
 REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES=('rest_framework.authentication.SessionAuthentication', 'pulpcore.app.authentication.PulpRemoteUserAuthentication')

tests/pulp_test.py

@@ -1,38 +1,45 @@
 import json
-
 import pytest
 
-
 PULP_HOST = 'localhost'
-PULP_PORT = 8080
-
+PULP_API_PORT = 24817
+PULP_CONTENT_PORT = 24816
 
 @pytest.fixture(scope="module")
 def pulp_status_curl(host):
-    return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/status/")
-
+    return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/status/")
 
 @pytest.fixture(scope="module")
 def pulp_status(pulp_status_curl):
     return json.loads(pulp_status_curl.stdout)
 
+def test_pulp_api_service(host):
+    pulp_api = host.service("pulp-api")
+    assert pulp_api.is_running
+    assert pulp_api.is_enabled
 
-def test_pulp_service(host):
-    pulp = host.service("pulp")
-    assert pulp.is_running
-    assert pulp.is_enabled
+def test_pulp_content_service(host):
+    pulp_content = host.service("pulp-content")
+    assert pulp_content.is_running
+    assert pulp_content.is_enabled
 
+def test_pulp_worker_service(host):
+    pulp_worker = host.service("pulp-worker")
+    assert pulp_worker.is_running
+    assert pulp_worker.is_enabled
 
-def test_pulp_port(host):
-    pulp = host.addr(PULP_HOST)
-    assert pulp.port(PULP_PORT).is_reachable
+def test_pulp_api_port(host):
+    pulp_api = host.addr(PULP_HOST)
+    assert pulp_api.port(PULP_API_PORT).is_reachable
 
+def test_pulp_content_port(host):
+    pulp_content = host.addr(PULP_HOST)
+    assert pulp_content.port(PULP_CONTENT_PORT).is_reachable
 
 def test_pulp_status(pulp_status_curl):
     assert pulp_status_curl.succeeded
     assert pulp_status_curl.stderr == '200'
 
-
 def test_pulp_status_database_connection(pulp_status):
     assert pulp_status['database_connection']['connected']
 
@@ -54,6 +61,9 @@ def test_pulp_status_workers(pulp_status):
 
 @pytest.mark.xfail(reason='password auth is deactivated when we use cert auth')
 def test_pulp_admin_auth(host):
-    cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/users/")
+    cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/users/")
     assert cmd.succeeded
     assert cmd.stderr == '200'
+
+def test_pulp_volumes(host):
+    assert host.file("/var/lib/pulp").is_directory