From aa775544b9c19c9dc8e6b12dfe3010df61ce1e46 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Fri, 25 Oct 2024 13:20:32 +0200 Subject: [PATCH 01/20] use the api to create a set of reusable fixtures --- requirements.txt | 1 + tests/conftest.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/foreman_api_test.py | 8 ++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/foreman_api_test.py diff --git a/requirements.txt b/requirements.txt index 36f79075..a5fae92c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pytest-testinfra paramiko +apypie diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..fb43b117 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,39 @@ +import uuid + +import apypie +import paramiko +import pytest + + +@pytest.fixture(scope="module") +def ssh_config(): + config = paramiko.SSHConfig.from_path('./.vagrant/ssh-config') + return config.lookup('quadlet') + + +@pytest.fixture(scope="module") +def foremanapi(ssh_config): + return apypie.ForemanApi( + uri=f'https://{ssh_config['hostname']}', + username='admin', + password='changeme', + verify_ssl=False, + ) + +@pytest.fixture +def organization(foremanapi): + org = foremanapi.create('organizations', {'name': str(uuid.uuid4())}) + yield org + foremanapi.delete('organizations', org) + +@pytest.fixture +def product(organization, foremanapi): + prod = foremanapi.create('products', {'name': str(uuid.uuid4()), 'organization_id': organization['id']}) + yield prod + foremanapi.delete('products', prod) + +@pytest.fixture +def repository(product, organization, foremanapi): + repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'yum'}) + yield repo + foremanapi.delete('repositories', repo) diff --git a/tests/foreman_api_test.py b/tests/foreman_api_test.py new file mode 100644 index 00000000..9b8e449f --- /dev/null +++ b/tests/foreman_api_test.py @@ -0,0 +1,8 @@ +def test_foreman_organization(organization): + assert organization + +def test_foreman_product(product): + assert product + +def test_foreman_repository(repository): + assert repository From b55b21d5c4c85e1034c2b160a7df87f31da721ce Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 29 Oct 2024 07:10:34 +0100 Subject: [PATCH 02/20] use nightly with katello fixes --- roles/foreman/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/foreman/defaults/main.yaml b/roles/foreman/defaults/main.yaml index 748a063e..32a31f61 100644 --- a/roles/foreman/defaults/main.yaml +++ b/roles/foreman/defaults/main.yaml @@ -1,3 +1,3 @@ --- foreman_container_image: "quay.io/evgeni/foreman-rpm" -foreman_container_tag: "3.12" +foreman_container_tag: "nightly" From 365c4ceed735ffba0b9311dfeb0c0b12a848a428 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 29 Oct 2024 08:15:51 +0100 Subject: [PATCH 03/20] apypie from git --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5fae92c..4b1c23e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pytest-testinfra paramiko -apypie +apypie @ git+https://github.com/Apipie/apypie.git@foreman From c93f0d17e859988e407d652210f329d3446d8eb5 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 29 Oct 2024 08:54:13 +0100 Subject: [PATCH 04/20] be verbose --- run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests b/run_tests index 130df808..5e9d766e 100755 --- a/run_tests +++ b/run_tests @@ -1,4 +1,4 @@ #!/bin/bash vagrant ssh-config > .vagrant/ssh-config -py.test --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py +py.test -vv --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py From 2a77e2e7fe93953720e2df79306c50f7809103b2 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Wed, 30 Oct 2024 09:40:06 +0100 Subject: [PATCH 05/20] yum/file --- tests/conftest.py | 10 ++++++++-- tests/foreman_api_test.py | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fb43b117..b99bf548 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,13 @@ def product(organization, foremanapi): foremanapi.delete('products', prod) @pytest.fixture -def repository(product, organization, foremanapi): - repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'yum'}) +def yum_repository(product, organization, foremanapi): + repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'yum', 'url': 'https://fixtures.pulpproject.org/rpm-no-comps/'}) + yield repo + foremanapi.delete('repositories', repo) + +@pytest.fixture +def file_repository(product, organization, foremanapi): + repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'file', 'url': 'https://fixtures.pulpproject.org/file/'}) yield repo foremanapi.delete('repositories', repo) diff --git a/tests/foreman_api_test.py b/tests/foreman_api_test.py index 9b8e449f..4472084f 100644 --- a/tests/foreman_api_test.py +++ b/tests/foreman_api_test.py @@ -1,8 +1,28 @@ +import urllib.parse + +import requests + + +def _repo_url(repo, ssh_config): + return urllib.parse.urlunparse(urllib.parse.urlparse(repo['full_path'])._replace(netloc=ssh_config['hostname'])) + + def test_foreman_organization(organization): assert organization def test_foreman_product(product): assert product -def test_foreman_repository(repository): - assert repository +def test_foreman_yum_repository(yum_repository, foremanapi, ssh_config): + assert yum_repository + foremanapi.resource_action('repositories', 'sync', {'id': yum_repository['id']}) + repo_url = _repo_url(yum_repository, ssh_config) + assert requests.get(f'{repo_url}/repodata/repomd.xml', verify=False) + assert requests.get(f'{repo_url}/Packages/b/bear-4.1-1.noarch.rpm', verify=False) + + +def test_foreman_file_repository(file_repository, foremanapi, ssh_config): + assert file_repository + foremanapi.resource_action('repositories', 'sync', {'id': file_repository['id']}) + repo_url = _repo_url(file_repository, ssh_config) + assert requests.get(f'{repo_url}/1.iso', verify=False) From 9d49f32944ece3d46e8df69696f605a0441db1dc Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Wed, 30 Oct 2024 13:32:43 +0100 Subject: [PATCH 06/20] archive some logs --- .github/workflows/test.yml | 7 +++++++ tests/zzz_test.py | 8 ++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/zzz_test.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d62026d..5a45a4ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,3 +57,10 @@ jobs: ansible-playbook playbooks/deploy.yaml - name: Run tests run: ./run_tests + - name: Upload logs + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: logs + path: logs.tar.gz + retention-days: 5 diff --git a/tests/zzz_test.py b/tests/zzz_test.py new file mode 100644 index 00000000..e55c3316 --- /dev/null +++ b/tests/zzz_test.py @@ -0,0 +1,8 @@ +def test_collect_report(host): + host.run('mkdir -p logs') + for container, filename in [('foreman', '/var/log/foreman/production.log')]: + localfile = filename.replace('/', '_') + host.run(f'podman cp {container}:{filename} logs/{container}-{localfile}') + host.run('tar caf logs.tar.gz logs/') + with open('logs.tar.gz', 'wb') as logstar: + logstar.write(host.file('logs.tar.gz').content) From e840f5a70105bcb744a7b10ae0125e73786acc55 Mon Sep 17 00:00:00 2001 From: akumari Date: Tue, 15 Oct 2024 15:39:45 +0530 Subject: [PATCH 07/20] add pulp services --- playbooks/deploy.yaml | 7 +- roles/pulp/defaults/main.yaml | 18 +++-- roles/pulp/tasks/main.yaml | 102 ++++++++++++++++++++++++---- roles/pulp/templates/settings.py.j2 | 14 ++++ tests/pulp_test.py | 27 +++++++- 5 files changed, 148 insertions(+), 20 deletions(-) diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index 278021af..6653d355 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -27,16 +27,21 @@ httpd_client_ca_certificate: "{{ certificates_ca_directory }}/certs/ca.crt" httpd_server_certificate: "{{ certificates_ca_directory }}/certs/{{ certificates_server }}.crt" httpd_server_key: "{{ certificates_ca_directory }}/private/{{ certificates_server }}.key" + pulp_db_password: "CHANGEME" postgresql_databases: - name: candlepin owner: candlepin - name: foreman owner: foreman + - name: pulp + owner: pulp postgresql_users: - name: candlepin password: "{{ candlepin_db_password }}" - name: foreman password: "{{ foreman_db_password }}" + - name: pulp + password: "{{ pulp_db_password }}" postgresql_hba_entries: - { type: local, database: all, user: postgres, auth_method: ident } - { type: local, database: all, user: all, auth_method: ident } @@ -47,9 +52,9 @@ roles: - certificates - geerlingguy.postgresql + - redis - candlepin - httpd - pulp - foreman_proxy - - redis - foreman diff --git a/roles/pulp/defaults/main.yaml b/roles/pulp/defaults/main.yaml index fee3d5f8..2faa9e61 100644 --- a/roles/pulp/defaults/main.yaml +++ b/roles/pulp/defaults/main.yaml @@ -1,10 +1,20 @@ --- -pulp_image: quay.io/pulp/pulp:stable -pulp_ports: - - "8080:80" +pulp_api_image: quay.io/pulp/pulp-minimal:stable +pulp_content_image: quay.io/pulp/pulp-minimal:stable +pulp_worker_image: quay.io/pulp/pulp-minimal:stable + +pulp_api_ports: + - "24817:80" +pulp_content_ports: + - "24816:80" +pulp_worker_count: 2 + pulp_volumes: - /var/lib/pulp/settings:/etc/pulp:Z - /var/lib/pulp/pulp_storage:/var/lib/pulp:Z - /var/lib/pulp/pgsql:/var/lib/pgsql:Z - /var/lib/pulp/containers:/var/lib/containers:Z -pulp_container_name: pulp + +pulp_api_container_name: pulp-api +pulp_content_container_name: pulp-content +pulp_worker_container_name: pulp-worker diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index f5f8bd8f..bbd6eec7 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -1,6 +1,16 @@ -- name: Pull the Pulp container image +- name: Pull the Pulp API container image containers.podman.podman_image: - name: "{{ pulp_image }}" + name: "{{ pulp_api_image }}" + state: present + +- name: Pull the Pulp Content container image + containers.podman.podman_image: + name: "{{ pulp_content_image }}" + state: present + +- name: Pull the Pulp Worker container image + containers.podman.podman_image: + name: "{{ pulp_worker_image }}" state: present - name: Create Pulp storage @@ -16,15 +26,71 @@ name: pulp-settings-py data: "{{ lookup('ansible.builtin.template', 'settings.py.j2') }}" -- name: Deploy Pulp Container +- name: Generate database symmetric key + ansible.builtin.command: "bash -c 'openssl rand -base64 32 | tr \"+/\" \"-_\" > /var/lib/pulp/database_fields.symmetric.key'" + args: + creates: /var/lib/pulp/database_fields.symmetric.key + +- name: Create database symmetric key secret + containers.podman.podman_secret: + state: present + name: pulp-symmetric-key + data: "{{ lookup('file', '/var/lib/pulp/database_fields.symmetric.key') }}" + +- name: Wait for PostgreSQL to be ready + ansible.builtin.wait_for: + host: "localhost" + port: 5432 + timeout: 300 + +- name: Deploy Pulp API Container containers.podman.podman_container: - name: "{{ pulp_container_name }}" - image: "{{ pulp_image }}" + name: "{{ pulp_api_container_name }}" + image: "{{ pulp_api_image }}" state: quadlet - ports: "{{ pulp_ports }}" + command: pulp-api + ports: "{{ pulp_api_ports }}" volumes: "{{ pulp_volumes }}" secrets: - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' + quadlet_options: + - | + [Install] + WantedBy=default.target + +- name: Deploy Pulp Content Container + containers.podman.podman_container: + name: "{{ pulp_content_container_name }}" + image: "{{ pulp_content_image }}" + state: quadlet + command: pulp-content + ports: "{{ pulp_content_ports }}" + volumes: "{{ pulp_volumes }}" + secrets: + - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' + quadlet_options: + - | + [Install] + WantedBy=default.target + +- name: Wait for Pulp API service to be accessible + ansible.builtin.wait_for: + host: "{{ ansible_hostname }}" + port: 24817 + timeout: 300 + +- name: Deploy Pulp Worker Container + containers.podman.podman_container: + name: "{{ pulp_worker_container_name }}" + image: "{{ pulp_worker_image }}" + state: quadlet + command: pulp-worker + volumes: "{{ pulp_volumes }}" + secrets: + - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' quadlet_options: - | [Install] @@ -34,17 +100,29 @@ ansible.builtin.systemd: daemon_reload: true -- name: Start the Pulp Service +- name: Start the Pulp API services ansible.builtin.systemd: - name: pulp + name: pulp-api enabled: true - state: restarted + state: started -- name: Wait for Pulp service to be accessible +- name: Start the Pulp Content services + ansible.builtin.systemd: + name: pulp-content + enabled: true + state: started + +- name: Wait for Pulp Content service to be accessible ansible.builtin.wait_for: host: "{{ ansible_hostname }}" - port: 8080 - timeout: 300 + port: 24816 + timeout: 600 + +- name: Start the Pulp Worker service + ansible.builtin.systemd: + name: pulp-worker + enabled: true + state: started # Only needed until we have cert auth configured - name: Set Pulp admin password diff --git a/roles/pulp/templates/settings.py.j2 b/roles/pulp/templates/settings.py.j2 index 16a2a0a6..97982fb9 100644 --- a/roles/pulp/templates/settings.py.j2 +++ b/roles/pulp/templates/settings.py.j2 @@ -1,7 +1,21 @@ CONTENT_ORIGIN="http://{{ ansible_hostname }}:8080" +API_CONTENT_ORIGIN="http://{{ ansible_hostname }}:24817" +CONTENT_SERVICE_ORIGIN="http://{{ ansible_hostname }}:24816" CACHE_ENABLED=True REDIS_HOST="localhost" REDIS_PORT=6379 + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'pulp', + 'USER': 'pulp', + 'PASSWORD': '{{ pulp_db_password }}', + 'HOST': 'localhost', + 'PORT': '5432', + } +} + AUTHENTICATION_BACKENDS=['pulpcore.app.authentication.PulpNoCreateRemoteUserBackend'] REMOTE_USER_ENVIRON_NAME="HTTP_REMOTE_USER" REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES=('rest_framework.authentication.SessionAuthentication', 'pulpcore.app.authentication.PulpRemoteUserAuthentication') diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 1ee4e9cf..9fe77299 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -1,11 +1,11 @@ import json - import pytest PULP_HOST = 'localhost' PULP_PORT = 8080 - +PULP_API_PORT = 24817 +PULP_CONTENT_PORT = 24816 @pytest.fixture(scope="module") def pulp_status_curl(host): @@ -22,17 +22,38 @@ def test_pulp_service(host): assert pulp.is_running assert pulp.is_enabled +def test_pulp_api_service(host): + pulp_api = host.service("pulp-api") + assert pulp_api.is_running + assert pulp_api.is_enabled + +def test_pulp_content_service(host): + pulp_content = host.service("pulp-content") + assert pulp_content.is_running + assert pulp_content.is_enabled + +def test_pulp_worker_services(host): + for i in range(1, 3): + pulp_worker = host.service(f"pulp-worker@{i}") + assert pulp_worker.is_running + assert pulp_worker.is_enabled def test_pulp_port(host): pulp = host.addr(PULP_HOST) assert pulp.port(PULP_PORT).is_reachable +def test_pulp_api_port(host): + pulp_api = host.addr(PULP_HOST) + assert pulp_api.port(PULP_API_PORT).is_reachable + +def test_pulp_content_port(host): + pulp_content = host.addr(PULP_HOST) + assert pulp_content.port(PULP_CONTENT_PORT).is_reachable def test_pulp_status(pulp_status_curl): assert pulp_status_curl.succeeded assert pulp_status_curl.stderr == '200' - def test_pulp_status_database_connection(pulp_status): assert pulp_status['database_connection']['connected'] From 44a5a953bfd200e2c64f855878eed92bc27bdf4f Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:35:24 +0100 Subject: [PATCH 08/20] stop managing etc pulp and pgsql, disable selinux labeling --- roles/pulp/defaults/main.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/pulp/defaults/main.yaml b/roles/pulp/defaults/main.yaml index 2faa9e61..2ded723e 100644 --- a/roles/pulp/defaults/main.yaml +++ b/roles/pulp/defaults/main.yaml @@ -10,10 +10,8 @@ pulp_content_ports: pulp_worker_count: 2 pulp_volumes: - - /var/lib/pulp/settings:/etc/pulp:Z - - /var/lib/pulp/pulp_storage:/var/lib/pulp:Z - - /var/lib/pulp/pgsql:/var/lib/pgsql:Z - - /var/lib/pulp/containers:/var/lib/containers:Z + - /var/lib/pulp/pulp_storage:/var/lib/pulp + - /var/lib/pulp/containers:/var/lib/containers pulp_api_container_name: pulp-api pulp_content_container_name: pulp-content From e88008424d58ddf2add2bf8b98ae023b654c62f7 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:36:05 +0100 Subject: [PATCH 09/20] run podman with label=disable to avoid permission errors the folders are shared between the containers, and mounting them with :z or :Z breaks stuff (even if :Z *should* work). label=disable avoids this issues. --- roles/pulp/tasks/main.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index bbd6eec7..dcde5309 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -51,6 +51,8 @@ command: pulp-api ports: "{{ pulp_api_ports }}" volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" secrets: - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' @@ -67,6 +69,8 @@ command: pulp-content ports: "{{ pulp_content_ports }}" volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" secrets: - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' @@ -88,6 +92,8 @@ state: quadlet command: pulp-worker volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" secrets: - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' From 42707e38bb6c41db396879610a713dad61900d2e Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:37:13 +0100 Subject: [PATCH 10/20] manually create tmp and asset dirs for pulp --- roles/pulp/tasks/main.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index dcde5309..b4248241 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -20,6 +20,15 @@ mode: "0755" loop: "{{ pulp_volumes }}" +- name: Create Pulp storage subdirs + ansible.builtin.file: + path: "/var/lib/pulp/pulp_storage/{{ item }}" + state: directory + mode: "0755" + loop: + - tmp + - assets + - name: Create settings config secret containers.podman.podman_secret: state: present From f2fa1e3738702cf809bd9064f9199bfd9a33c7aa Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:37:31 +0100 Subject: [PATCH 11/20] get the encryption key from the vm --- roles/pulp/tasks/main.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index b4248241..b3e69c62 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -40,11 +40,16 @@ args: creates: /var/lib/pulp/database_fields.symmetric.key +- name: Load database symmetric key + ansible.builtin.slurp: + src: /var/lib/pulp/database_fields.symmetric.key + register: pulp_key + - name: Create database symmetric key secret containers.podman.podman_secret: state: present name: pulp-symmetric-key - data: "{{ lookup('file', '/var/lib/pulp/database_fields.symmetric.key') }}" + data: "{{ pulp_key['content'] | b64decode }}" - name: Wait for PostgreSQL to be ready ansible.builtin.wait_for: From e6e0116a2327892235f9f104dae7c2a13db550a2 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:38:22 +0100 Subject: [PATCH 12/20] use host networking so that the containers can reach psql on localhost --- roles/pulp/tasks/main.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index b3e69c62..ee94ef69 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -63,7 +63,7 @@ image: "{{ pulp_api_image }}" state: quadlet command: pulp-api - ports: "{{ pulp_api_ports }}" + network: host volumes: "{{ pulp_volumes }}" security_opt: - "label=disable" @@ -81,7 +81,7 @@ image: "{{ pulp_content_image }}" state: quadlet command: pulp-content - ports: "{{ pulp_content_ports }}" + network: host volumes: "{{ pulp_volumes }}" security_opt: - "label=disable" @@ -105,6 +105,7 @@ image: "{{ pulp_worker_image }}" state: quadlet command: pulp-worker + network: host volumes: "{{ pulp_volumes }}" security_opt: - "label=disable" From e4f8323a9b8deac98fc130a8b2d0af9f23fa4520 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:39:03 +0100 Subject: [PATCH 13/20] manually migrate the database --- roles/pulp/tasks/main.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index ee94ef69..6d74a36c 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -127,6 +127,11 @@ enabled: true state: started +- name: Migrate the Pylp database + containers.podman.podman_container_exec: + name: "{{ pulp_api_container_name }}" + command: pulpcore-manager migrate --noinput + - name: Start the Pulp Content services ansible.builtin.systemd: name: pulp-content From 0b75f52e6a4cf285a6f3f7a5c008549379c15cf7 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 15:39:16 +0100 Subject: [PATCH 14/20] use the right container name when reseting the password --- roles/pulp/tasks/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index 6d74a36c..20c5f0b4 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -153,5 +153,5 @@ # Only needed until we have cert auth configured - name: Set Pulp admin password containers.podman.podman_container_exec: - name: "{{ pulp_container_name }}" + name: "{{ pulp_api_container_name }}" command: pulpcore-manager reset-admin-password --password CHANGEME From 7d01652072494041f84362b817140002cf56788d Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 17:04:46 +0100 Subject: [PATCH 15/20] wait for api later --- roles/pulp/tasks/main.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index 20c5f0b4..c25ae3d1 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -93,12 +93,6 @@ [Install] WantedBy=default.target -- name: Wait for Pulp API service to be accessible - ansible.builtin.wait_for: - host: "{{ ansible_hostname }}" - port: 24817 - timeout: 300 - - name: Deploy Pulp Worker Container containers.podman.podman_container: name: "{{ pulp_worker_container_name }}" @@ -132,6 +126,12 @@ name: "{{ pulp_api_container_name }}" command: pulpcore-manager migrate --noinput +- name: Wait for Pulp API service to be accessible + ansible.builtin.wait_for: + host: "{{ ansible_hostname }}" + port: 24817 + timeout: 300 + - name: Start the Pulp Content services ansible.builtin.systemd: name: pulp-content From 770b5575c3bcab81e71e1f8384ab62d49d0d6d4a Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Mon, 4 Nov 2024 17:30:52 +0100 Subject: [PATCH 16/20] use right ports for pulp in http config --- roles/httpd/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/httpd/defaults/main.yml b/roles/httpd/defaults/main.yml index d3dd7742..93399472 100644 --- a/roles/httpd/defaults/main.yml +++ b/roles/httpd/defaults/main.yml @@ -1,4 +1,4 @@ httpd_ssl_dir: /etc/pki/httpd -httpd_pulp_api_backend: http://localhost:8080 -httpd_pulp_content_backend: http://localhost:8080 +httpd_pulp_api_backend: http://localhost:24817 +httpd_pulp_content_backend: http://localhost:24816 httpd_foreman_backend: http://localhost:3000 From 836d8ec165a6844c2c9472d0c99117c109fdc23c Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 5 Nov 2024 09:21:08 +0100 Subject: [PATCH 17/20] don't use PULP_PORT in tests --- tests/pulp_test.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 9fe77299..6e829a87 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -3,13 +3,12 @@ PULP_HOST = 'localhost' -PULP_PORT = 8080 PULP_API_PORT = 24817 PULP_CONTENT_PORT = 24816 @pytest.fixture(scope="module") def pulp_status_curl(host): - return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/status/") + return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/status/") @pytest.fixture(scope="module") @@ -38,10 +37,6 @@ def test_pulp_worker_services(host): assert pulp_worker.is_running assert pulp_worker.is_enabled -def test_pulp_port(host): - pulp = host.addr(PULP_HOST) - assert pulp.port(PULP_PORT).is_reachable - def test_pulp_api_port(host): pulp_api = host.addr(PULP_HOST) assert pulp_api.port(PULP_API_PORT).is_reachable @@ -75,6 +70,6 @@ def test_pulp_status_workers(pulp_status): @pytest.mark.xfail(reason='password auth is deactivated when we use cert auth') def test_pulp_admin_auth(host): - cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/users/") + cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/users/") assert cmd.succeeded assert cmd.stderr == '200' From 1ebaacc2edfd83e84480d9aab58b2aff2bd313ed Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 5 Nov 2024 09:21:43 +0100 Subject: [PATCH 18/20] don't test pulp service, it's gone --- tests/pulp_test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 6e829a87..2e0525e3 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -16,11 +16,6 @@ def pulp_status(pulp_status_curl): return json.loads(pulp_status_curl.stdout) -def test_pulp_service(host): - pulp = host.service("pulp") - assert pulp.is_running - assert pulp.is_enabled - def test_pulp_api_service(host): pulp_api = host.service("pulp-api") assert pulp_api.is_running From 66dcbd7f33b077553f71811733913431bec47f92 Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 5 Nov 2024 10:48:39 +0100 Subject: [PATCH 19/20] always restart pulp workers pulp workers die and don't respawn when the DB connection drops --- roles/pulp/tasks/main.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index c25ae3d1..2ff1525c 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -110,6 +110,11 @@ - | [Install] WantedBy=default.target + [Service] + # This provides reconnect support for PostgreSQL and Redis. Without reconnect support, if either + # is not available at startup or becomes disconnected, this process will die and not respawn. + Restart=always + RestartSec=3 - name: Run daemon reload to make Quadlet create the service files ansible.builtin.systemd: From 80ee1de93cfe02942983dbcce8e74b2c65b1033d Mon Sep 17 00:00:00 2001 From: Evgeni Golov Date: Tue, 5 Nov 2024 11:14:36 +0100 Subject: [PATCH 20/20] only one worker for now --- tests/pulp_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 2e0525e3..056ccf99 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -27,10 +27,9 @@ def test_pulp_content_service(host): assert pulp_content.is_enabled def test_pulp_worker_services(host): - for i in range(1, 3): - pulp_worker = host.service(f"pulp-worker@{i}") - assert pulp_worker.is_running - assert pulp_worker.is_enabled + pulp_worker = host.service("pulp-worker") + assert pulp_worker.is_running + assert pulp_worker.is_enabled def test_pulp_api_port(host): pulp_api = host.addr(PULP_HOST)