Pulp with tests

.github/workflows/test.yml

@@ -57,3 +57,10 @@ jobs:
           ansible-playbook playbooks/deploy.yaml
       - name: Run tests
         run: ./run_tests
+      - name: Upload logs
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: logs
+          path: logs.tar.gz
+          retention-days: 5

playbooks/deploy.yaml

@@ -27,16 +27,21 @@
     httpd_client_ca_certificate: "{{ certificates_ca_directory }}/certs/ca.crt"
     httpd_server_certificate: "{{ certificates_ca_directory }}/certs/{{ certificates_server }}.crt"
     httpd_server_key: "{{ certificates_ca_directory }}/private/{{ certificates_server }}.key"
+    pulp_db_password: "CHANGEME"
     postgresql_databases:
       - name: candlepin
         owner: candlepin
       - name: foreman
         owner: foreman
+      - name: pulp
+        owner: pulp
     postgresql_users:
       - name: candlepin
         password: "{{ candlepin_db_password }}"
       - name: foreman
         password: "{{ foreman_db_password }}"
+      - name: pulp
+        password: "{{ pulp_db_password }}"
     postgresql_hba_entries:
       - { type: local, database: all, user: postgres, auth_method: ident }
       - { type: local, database: all, user: all, auth_method: ident }
@@ -47,9 +52,9 @@
   roles:
     - certificates
     - geerlingguy.postgresql
+    - redis
     - candlepin
     - httpd
     - pulp
     - foreman_proxy
-    - redis
     - foreman

requirements.txt

@@ -1,2 +1,3 @@
 pytest-testinfra
 paramiko
+apypie @ git+https://github.com/Apipie/apypie.git@foreman

roles/foreman/defaults/main.yaml

@@ -1,3 +1,3 @@
 ---
 foreman_container_image: "quay.io/evgeni/foreman-rpm"
-foreman_container_tag: "3.12"
+foreman_container_tag: "nightly"

roles/httpd/defaults/main.yml

@@ -1,4 +1,4 @@
 httpd_ssl_dir: /etc/pki/httpd
-httpd_pulp_api_backend: http://localhost:8080
-httpd_pulp_content_backend: http://localhost:8080
+httpd_pulp_api_backend: http://localhost:24817
+httpd_pulp_content_backend: http://localhost:24816
 httpd_foreman_backend: http://localhost:3000

roles/pulp/defaults/main.yaml

@@ -1,10 +1,18 @@
 ---
-pulp_image: quay.io/pulp/pulp:stable
-pulp_ports:
-  - "8080:80"
+pulp_api_image: quay.io/pulp/pulp-minimal:stable
+pulp_content_image: quay.io/pulp/pulp-minimal:stable
+pulp_worker_image: quay.io/pulp/pulp-minimal:stable
+
+pulp_api_ports:
+  - "24817:80"
+pulp_content_ports:
+  - "24816:80"
+pulp_worker_count: 2
+
 pulp_volumes:
-  - /var/lib/pulp/settings:/etc/pulp:Z
-  - /var/lib/pulp/pulp_storage:/var/lib/pulp:Z
-  - /var/lib/pulp/pgsql:/var/lib/pgsql:Z
-  - /var/lib/pulp/containers:/var/lib/containers:Z
-pulp_container_name: pulp
+  - /var/lib/pulp/pulp_storage:/var/lib/pulp
+  - /var/lib/pulp/containers:/var/lib/containers
+
+pulp_api_container_name: pulp-api
+pulp_content_container_name: pulp-content
+pulp_worker_container_name: pulp-worker

roles/pulp/tasks/main.yaml

@@ -1,6 +1,16 @@
-- name: Pull the Pulp container image
+- name: Pull the Pulp API container image
   containers.podman.podman_image:
-    name: "{{ pulp_image }}"
+    name: "{{ pulp_api_image }}"
+    state: present
+
+- name: Pull the Pulp Content container image
+  containers.podman.podman_image:
+    name: "{{ pulp_content_image }}"
+    state: present
+
+- name: Pull the Pulp Worker container image
+  containers.podman.podman_image:
+    name: "{{ pulp_worker_image }}"
     state: present
 
 - name: Create Pulp storage
@@ -10,44 +20,143 @@
     mode: "0755"
   loop: "{{ pulp_volumes }}"
 
+- name: Create Pulp storage subdirs
+  ansible.builtin.file:
+    path: "/var/lib/pulp/pulp_storage/{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - tmp
+    - assets
+
 - name: Create settings config secret
   containers.podman.podman_secret:
     state: present
     name: pulp-settings-py
     data: "{{ lookup('ansible.builtin.template', 'settings.py.j2') }}"
 
-- name: Deploy Pulp Container
+- name: Generate database symmetric key
+  ansible.builtin.command: "bash -c 'openssl rand -base64 32 | tr \"+/\" \"-_\" > /var/lib/pulp/database_fields.symmetric.key'"
+  args:
+    creates: /var/lib/pulp/database_fields.symmetric.key
+
+- name: Load database symmetric key
+  ansible.builtin.slurp:
+    src: /var/lib/pulp/database_fields.symmetric.key
+  register: pulp_key
+
+- name: Create database symmetric key secret
+  containers.podman.podman_secret:
+    state: present
+    name: pulp-symmetric-key
+    data: "{{ pulp_key['content'] | b64decode }}"
+
+- name: Wait for PostgreSQL to be ready
+  ansible.builtin.wait_for:
+    host: "localhost"
+    port: 5432
+    timeout: 300
+
+- name: Deploy Pulp API Container
   containers.podman.podman_container:
-    name: "{{ pulp_container_name }}"
-    image: "{{ pulp_image }}"
+    name: "{{ pulp_api_container_name }}"
+    image: "{{ pulp_api_image }}"
     state: quadlet
-    ports: "{{ pulp_ports }}"
+    command: pulp-api
+    network: host
     volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
     secrets:
       - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
     quadlet_options:
       - |
         [Install]
         WantedBy=default.target
 
+- name: Deploy Pulp Content Container
+  containers.podman.podman_container:
+    name: "{{ pulp_content_container_name }}"
+    image: "{{ pulp_content_image }}"
+    state: quadlet
+    command: pulp-content
+    network: host
+    volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
+    secrets:
+      - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
+    quadlet_options:
+      - |
+        [Install]
+        WantedBy=default.target
+
+- name: Deploy Pulp Worker Container
+  containers.podman.podman_container:
+    name: "{{ pulp_worker_container_name }}"
+    image: "{{ pulp_worker_image }}"
+    state: quadlet
+    command: pulp-worker
+    network: host
+    volumes: "{{ pulp_volumes }}"
+    security_opt:
+      - "label=disable"
+    secrets:
+      - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py'
+      - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key'
+    quadlet_options:
+      - |
+        [Install]
+        WantedBy=default.target
+        [Service]
+        # This provides reconnect support for PostgreSQL and Redis. Without reconnect support, if either
+        # is not available at startup or becomes disconnected, this process will die and not respawn.
+        Restart=always
+        RestartSec=3
+
 - name: Run daemon reload to make Quadlet create the service files
   ansible.builtin.systemd:
     daemon_reload: true
 
-- name: Start the Pulp Service
+- name: Start the Pulp API services
   ansible.builtin.systemd:
-    name: pulp
+    name: pulp-api
     enabled: true
-    state: restarted
+    state: started
+
+- name: Migrate the Pylp database
+  containers.podman.podman_container_exec:
+    name: "{{ pulp_api_container_name }}"
+    command: pulpcore-manager migrate --noinput
 
-- name: Wait for Pulp service to be accessible
+- name: Wait for Pulp API service to be accessible
   ansible.builtin.wait_for:
     host: "{{ ansible_hostname }}"
-    port: 8080
+    port: 24817
     timeout: 300
 
+- name: Start the Pulp Content services
+  ansible.builtin.systemd:
+    name: pulp-content
+    enabled: true
+    state: started
+
+- name: Wait for Pulp Content service to be accessible
+  ansible.builtin.wait_for:
+    host: "{{ ansible_hostname }}"
+    port: 24816
+    timeout: 600
+
+- name: Start the Pulp Worker service
+  ansible.builtin.systemd:
+    name: pulp-worker
+    enabled: true
+    state: started
+
 # Only needed until we have cert auth configured
 - name: Set Pulp admin password
   containers.podman.podman_container_exec:
-    name: "{{ pulp_container_name }}"
+    name: "{{ pulp_api_container_name }}"
     command: pulpcore-manager reset-admin-password --password CHANGEME

roles/pulp/templates/settings.py.j2

@@ -1,7 +1,21 @@
 CONTENT_ORIGIN="http://{{ ansible_hostname }}:8080"
+API_CONTENT_ORIGIN="http://{{ ansible_hostname }}:24817"
+CONTENT_SERVICE_ORIGIN="http://{{ ansible_hostname }}:24816"
 CACHE_ENABLED=True
 REDIS_HOST="localhost"
 REDIS_PORT=6379
+
+DATABASES = {
+	'default': {
+		'ENGINE': 'django.db.backends.postgresql',
+		'NAME': 'pulp',
+		'USER': 'pulp',
+		'PASSWORD': '{{ pulp_db_password }}',
+		'HOST': 'localhost',
+		'PORT': '5432',
+	}
+}
+
 AUTHENTICATION_BACKENDS=['pulpcore.app.authentication.PulpNoCreateRemoteUserBackend']
 REMOTE_USER_ENVIRON_NAME="HTTP_REMOTE_USER"
 REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES=('rest_framework.authentication.SessionAuthentication', 'pulpcore.app.authentication.PulpRemoteUserAuthentication')

run_tests

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 vagrant ssh-config > .vagrant/ssh-config
-py.test --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py
+py.test -vv --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py

tests/conftest.py

@@ -0,0 +1,45 @@
+import uuid
+
+import apypie
+import paramiko
+import pytest
+
+
+@pytest.fixture(scope="module")
+def ssh_config():
+    config = paramiko.SSHConfig.from_path('./.vagrant/ssh-config')
+    return config.lookup('quadlet')
+
+
+@pytest.fixture(scope="module")
+def foremanapi(ssh_config):
+    return apypie.ForemanApi(
+        uri=f'https://{ssh_config['hostname']}',
+        username='admin',
+        password='changeme',
+        verify_ssl=False,
+    )
+
+@pytest.fixture
+def organization(foremanapi):
+    org = foremanapi.create('organizations', {'name': str(uuid.uuid4())})
+    yield org
+    foremanapi.delete('organizations', org)
+
+@pytest.fixture
+def product(organization, foremanapi):
+    prod = foremanapi.create('products', {'name': str(uuid.uuid4()), 'organization_id': organization['id']})
+    yield prod
+    foremanapi.delete('products', prod)
+
+@pytest.fixture
+def yum_repository(product, organization, foremanapi):
+    repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'yum', 'url': 'https://fixtures.pulpproject.org/rpm-no-comps/'})
+    yield repo
+    foremanapi.delete('repositories', repo)
+
+@pytest.fixture
+def file_repository(product, organization, foremanapi):
+    repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'file', 'url': 'https://fixtures.pulpproject.org/file/'})
+    yield repo
+    foremanapi.delete('repositories', repo)

tests/foreman_api_test.py

@@ -0,0 +1,28 @@
+import urllib.parse
+
+import requests
+
+
+def _repo_url(repo, ssh_config):
+    return urllib.parse.urlunparse(urllib.parse.urlparse(repo['full_path'])._replace(netloc=ssh_config['hostname']))
+
+
+def test_foreman_organization(organization):
+    assert organization
+
+def test_foreman_product(product):
+    assert product
+
+def test_foreman_yum_repository(yum_repository, foremanapi, ssh_config):
+    assert yum_repository
+    foremanapi.resource_action('repositories', 'sync', {'id': yum_repository['id']})
+    repo_url = _repo_url(yum_repository, ssh_config)
+    assert requests.get(f'{repo_url}/repodata/repomd.xml', verify=False)
+    assert requests.get(f'{repo_url}/Packages/b/bear-4.1-1.noarch.rpm', verify=False)
+
+
+def test_foreman_file_repository(file_repository, foremanapi, ssh_config):
+    assert file_repository
+    foremanapi.resource_action('repositories', 'sync', {'id': file_repository['id']})
+    repo_url = _repo_url(file_repository, ssh_config)
+    assert requests.get(f'{repo_url}/1.iso', verify=False)