diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d62026d..5a45a4ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,3 +57,10 @@ jobs: ansible-playbook playbooks/deploy.yaml - name: Run tests run: ./run_tests + - name: Upload logs + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: logs + path: logs.tar.gz + retention-days: 5 diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index 278021af..6653d355 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -27,16 +27,21 @@ httpd_client_ca_certificate: "{{ certificates_ca_directory }}/certs/ca.crt" httpd_server_certificate: "{{ certificates_ca_directory }}/certs/{{ certificates_server }}.crt" httpd_server_key: "{{ certificates_ca_directory }}/private/{{ certificates_server }}.key" + pulp_db_password: "CHANGEME" postgresql_databases: - name: candlepin owner: candlepin - name: foreman owner: foreman + - name: pulp + owner: pulp postgresql_users: - name: candlepin password: "{{ candlepin_db_password }}" - name: foreman password: "{{ foreman_db_password }}" + - name: pulp + password: "{{ pulp_db_password }}" postgresql_hba_entries: - { type: local, database: all, user: postgres, auth_method: ident } - { type: local, database: all, user: all, auth_method: ident } @@ -47,9 +52,9 @@ roles: - certificates - geerlingguy.postgresql + - redis - candlepin - httpd - pulp - foreman_proxy - - redis - foreman diff --git a/requirements.txt b/requirements.txt index 36f79075..4b1c23e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pytest-testinfra paramiko +apypie @ git+https://github.com/Apipie/apypie.git@foreman diff --git a/roles/foreman/defaults/main.yaml b/roles/foreman/defaults/main.yaml index 748a063e..32a31f61 100644 --- a/roles/foreman/defaults/main.yaml +++ b/roles/foreman/defaults/main.yaml @@ -1,3 +1,3 @@ --- foreman_container_image: "quay.io/evgeni/foreman-rpm" -foreman_container_tag: "3.12" +foreman_container_tag: "nightly" diff --git a/roles/httpd/defaults/main.yml b/roles/httpd/defaults/main.yml index d3dd7742..93399472 100644 --- a/roles/httpd/defaults/main.yml +++ b/roles/httpd/defaults/main.yml @@ -1,4 +1,4 @@ httpd_ssl_dir: /etc/pki/httpd -httpd_pulp_api_backend: http://localhost:8080 -httpd_pulp_content_backend: http://localhost:8080 +httpd_pulp_api_backend: http://localhost:24817 +httpd_pulp_content_backend: http://localhost:24816 httpd_foreman_backend: http://localhost:3000 diff --git a/roles/pulp/defaults/main.yaml b/roles/pulp/defaults/main.yaml index fee3d5f8..2ded723e 100644 --- a/roles/pulp/defaults/main.yaml +++ b/roles/pulp/defaults/main.yaml @@ -1,10 +1,18 @@ --- -pulp_image: quay.io/pulp/pulp:stable -pulp_ports: - - "8080:80" +pulp_api_image: quay.io/pulp/pulp-minimal:stable +pulp_content_image: quay.io/pulp/pulp-minimal:stable +pulp_worker_image: quay.io/pulp/pulp-minimal:stable + +pulp_api_ports: + - "24817:80" +pulp_content_ports: + - "24816:80" +pulp_worker_count: 2 + pulp_volumes: - - /var/lib/pulp/settings:/etc/pulp:Z - - /var/lib/pulp/pulp_storage:/var/lib/pulp:Z - - /var/lib/pulp/pgsql:/var/lib/pgsql:Z - - /var/lib/pulp/containers:/var/lib/containers:Z -pulp_container_name: pulp + - /var/lib/pulp/pulp_storage:/var/lib/pulp + - /var/lib/pulp/containers:/var/lib/containers + +pulp_api_container_name: pulp-api +pulp_content_container_name: pulp-content +pulp_worker_container_name: pulp-worker diff --git a/roles/pulp/tasks/main.yaml b/roles/pulp/tasks/main.yaml index f5f8bd8f..2ff1525c 100644 --- a/roles/pulp/tasks/main.yaml +++ b/roles/pulp/tasks/main.yaml @@ -1,6 +1,16 @@ -- name: Pull the Pulp container image +- name: Pull the Pulp API container image containers.podman.podman_image: - name: "{{ pulp_image }}" + name: "{{ pulp_api_image }}" + state: present + +- name: Pull the Pulp Content container image + containers.podman.podman_image: + name: "{{ pulp_content_image }}" + state: present + +- name: Pull the Pulp Worker container image + containers.podman.podman_image: + name: "{{ pulp_worker_image }}" state: present - name: Create Pulp storage @@ -10,44 +20,143 @@ mode: "0755" loop: "{{ pulp_volumes }}" +- name: Create Pulp storage subdirs + ansible.builtin.file: + path: "/var/lib/pulp/pulp_storage/{{ item }}" + state: directory + mode: "0755" + loop: + - tmp + - assets + - name: Create settings config secret containers.podman.podman_secret: state: present name: pulp-settings-py data: "{{ lookup('ansible.builtin.template', 'settings.py.j2') }}" -- name: Deploy Pulp Container +- name: Generate database symmetric key + ansible.builtin.command: "bash -c 'openssl rand -base64 32 | tr \"+/\" \"-_\" > /var/lib/pulp/database_fields.symmetric.key'" + args: + creates: /var/lib/pulp/database_fields.symmetric.key + +- name: Load database symmetric key + ansible.builtin.slurp: + src: /var/lib/pulp/database_fields.symmetric.key + register: pulp_key + +- name: Create database symmetric key secret + containers.podman.podman_secret: + state: present + name: pulp-symmetric-key + data: "{{ pulp_key['content'] | b64decode }}" + +- name: Wait for PostgreSQL to be ready + ansible.builtin.wait_for: + host: "localhost" + port: 5432 + timeout: 300 + +- name: Deploy Pulp API Container containers.podman.podman_container: - name: "{{ pulp_container_name }}" - image: "{{ pulp_image }}" + name: "{{ pulp_api_container_name }}" + image: "{{ pulp_api_image }}" state: quadlet - ports: "{{ pulp_ports }}" + command: pulp-api + network: host volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" secrets: - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' quadlet_options: - | [Install] WantedBy=default.target +- name: Deploy Pulp Content Container + containers.podman.podman_container: + name: "{{ pulp_content_container_name }}" + image: "{{ pulp_content_image }}" + state: quadlet + command: pulp-content + network: host + volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" + secrets: + - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' + quadlet_options: + - | + [Install] + WantedBy=default.target + +- name: Deploy Pulp Worker Container + containers.podman.podman_container: + name: "{{ pulp_worker_container_name }}" + image: "{{ pulp_worker_image }}" + state: quadlet + command: pulp-worker + network: host + volumes: "{{ pulp_volumes }}" + security_opt: + - "label=disable" + secrets: + - 'pulp-settings-py,type=mount,target=/etc/pulp/settings.py' + - 'pulp-symmetric-key,type=mount,target=/etc/pulp/certs/database_fields.symmetric.key' + quadlet_options: + - | + [Install] + WantedBy=default.target + [Service] + # This provides reconnect support for PostgreSQL and Redis. Without reconnect support, if either + # is not available at startup or becomes disconnected, this process will die and not respawn. + Restart=always + RestartSec=3 + - name: Run daemon reload to make Quadlet create the service files ansible.builtin.systemd: daemon_reload: true -- name: Start the Pulp Service +- name: Start the Pulp API services ansible.builtin.systemd: - name: pulp + name: pulp-api enabled: true - state: restarted + state: started + +- name: Migrate the Pylp database + containers.podman.podman_container_exec: + name: "{{ pulp_api_container_name }}" + command: pulpcore-manager migrate --noinput -- name: Wait for Pulp service to be accessible +- name: Wait for Pulp API service to be accessible ansible.builtin.wait_for: host: "{{ ansible_hostname }}" - port: 8080 + port: 24817 timeout: 300 +- name: Start the Pulp Content services + ansible.builtin.systemd: + name: pulp-content + enabled: true + state: started + +- name: Wait for Pulp Content service to be accessible + ansible.builtin.wait_for: + host: "{{ ansible_hostname }}" + port: 24816 + timeout: 600 + +- name: Start the Pulp Worker service + ansible.builtin.systemd: + name: pulp-worker + enabled: true + state: started + # Only needed until we have cert auth configured - name: Set Pulp admin password containers.podman.podman_container_exec: - name: "{{ pulp_container_name }}" + name: "{{ pulp_api_container_name }}" command: pulpcore-manager reset-admin-password --password CHANGEME diff --git a/roles/pulp/templates/settings.py.j2 b/roles/pulp/templates/settings.py.j2 index 16a2a0a6..97982fb9 100644 --- a/roles/pulp/templates/settings.py.j2 +++ b/roles/pulp/templates/settings.py.j2 @@ -1,7 +1,21 @@ CONTENT_ORIGIN="http://{{ ansible_hostname }}:8080" +API_CONTENT_ORIGIN="http://{{ ansible_hostname }}:24817" +CONTENT_SERVICE_ORIGIN="http://{{ ansible_hostname }}:24816" CACHE_ENABLED=True REDIS_HOST="localhost" REDIS_PORT=6379 + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'pulp', + 'USER': 'pulp', + 'PASSWORD': '{{ pulp_db_password }}', + 'HOST': 'localhost', + 'PORT': '5432', + } +} + AUTHENTICATION_BACKENDS=['pulpcore.app.authentication.PulpNoCreateRemoteUserBackend'] REMOTE_USER_ENVIRON_NAME="HTTP_REMOTE_USER" REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES=('rest_framework.authentication.SessionAuthentication', 'pulpcore.app.authentication.PulpRemoteUserAuthentication') diff --git a/run_tests b/run_tests index 130df808..5e9d766e 100755 --- a/run_tests +++ b/run_tests @@ -1,4 +1,4 @@ #!/bin/bash vagrant ssh-config > .vagrant/ssh-config -py.test --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py +py.test -vv --hosts=quadlet --sudo --ssh-config=.vagrant/ssh-config tests/*_test.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..b99bf548 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +import uuid + +import apypie +import paramiko +import pytest + + +@pytest.fixture(scope="module") +def ssh_config(): + config = paramiko.SSHConfig.from_path('./.vagrant/ssh-config') + return config.lookup('quadlet') + + +@pytest.fixture(scope="module") +def foremanapi(ssh_config): + return apypie.ForemanApi( + uri=f'https://{ssh_config['hostname']}', + username='admin', + password='changeme', + verify_ssl=False, + ) + +@pytest.fixture +def organization(foremanapi): + org = foremanapi.create('organizations', {'name': str(uuid.uuid4())}) + yield org + foremanapi.delete('organizations', org) + +@pytest.fixture +def product(organization, foremanapi): + prod = foremanapi.create('products', {'name': str(uuid.uuid4()), 'organization_id': organization['id']}) + yield prod + foremanapi.delete('products', prod) + +@pytest.fixture +def yum_repository(product, organization, foremanapi): + repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'yum', 'url': 'https://fixtures.pulpproject.org/rpm-no-comps/'}) + yield repo + foremanapi.delete('repositories', repo) + +@pytest.fixture +def file_repository(product, organization, foremanapi): + repo = foremanapi.create('repositories', {'name': str(uuid.uuid4()), 'product_id': product['id'], 'content_type': 'file', 'url': 'https://fixtures.pulpproject.org/file/'}) + yield repo + foremanapi.delete('repositories', repo) diff --git a/tests/foreman_api_test.py b/tests/foreman_api_test.py new file mode 100644 index 00000000..4472084f --- /dev/null +++ b/tests/foreman_api_test.py @@ -0,0 +1,28 @@ +import urllib.parse + +import requests + + +def _repo_url(repo, ssh_config): + return urllib.parse.urlunparse(urllib.parse.urlparse(repo['full_path'])._replace(netloc=ssh_config['hostname'])) + + +def test_foreman_organization(organization): + assert organization + +def test_foreman_product(product): + assert product + +def test_foreman_yum_repository(yum_repository, foremanapi, ssh_config): + assert yum_repository + foremanapi.resource_action('repositories', 'sync', {'id': yum_repository['id']}) + repo_url = _repo_url(yum_repository, ssh_config) + assert requests.get(f'{repo_url}/repodata/repomd.xml', verify=False) + assert requests.get(f'{repo_url}/Packages/b/bear-4.1-1.noarch.rpm', verify=False) + + +def test_foreman_file_repository(file_repository, foremanapi, ssh_config): + assert file_repository + foremanapi.resource_action('repositories', 'sync', {'id': file_repository['id']}) + repo_url = _repo_url(file_repository, ssh_config) + assert requests.get(f'{repo_url}/1.iso', verify=False) diff --git a/tests/pulp_test.py b/tests/pulp_test.py index 1ee4e9cf..056ccf99 100644 --- a/tests/pulp_test.py +++ b/tests/pulp_test.py @@ -1,15 +1,14 @@ import json - import pytest PULP_HOST = 'localhost' -PULP_PORT = 8080 - +PULP_API_PORT = 24817 +PULP_CONTENT_PORT = 24816 @pytest.fixture(scope="module") def pulp_status_curl(host): - return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/status/") + return host.run(f"curl -k -s -w '%{{stderr}}%{{http_code}}' http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/status/") @pytest.fixture(scope="module") @@ -17,22 +16,33 @@ def pulp_status(pulp_status_curl): return json.loads(pulp_status_curl.stdout) -def test_pulp_service(host): - pulp = host.service("pulp") - assert pulp.is_running - assert pulp.is_enabled +def test_pulp_api_service(host): + pulp_api = host.service("pulp-api") + assert pulp_api.is_running + assert pulp_api.is_enabled +def test_pulp_content_service(host): + pulp_content = host.service("pulp-content") + assert pulp_content.is_running + assert pulp_content.is_enabled -def test_pulp_port(host): - pulp = host.addr(PULP_HOST) - assert pulp.port(PULP_PORT).is_reachable +def test_pulp_worker_services(host): + pulp_worker = host.service("pulp-worker") + assert pulp_worker.is_running + assert pulp_worker.is_enabled +def test_pulp_api_port(host): + pulp_api = host.addr(PULP_HOST) + assert pulp_api.port(PULP_API_PORT).is_reachable + +def test_pulp_content_port(host): + pulp_content = host.addr(PULP_HOST) + assert pulp_content.port(PULP_CONTENT_PORT).is_reachable def test_pulp_status(pulp_status_curl): assert pulp_status_curl.succeeded assert pulp_status_curl.stderr == '200' - def test_pulp_status_database_connection(pulp_status): assert pulp_status['database_connection']['connected'] @@ -54,6 +64,6 @@ def test_pulp_status_workers(pulp_status): @pytest.mark.xfail(reason='password auth is deactivated when we use cert auth') def test_pulp_admin_auth(host): - cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_PORT}/pulp/api/v3/users/") + cmd = host.run(f"curl --silent --write-out '%{{stderr}}%{{http_code}}' --user admin:CHANGEME http://{PULP_HOST}:{PULP_API_PORT}/pulp/api/v3/users/") assert cmd.succeeded assert cmd.stderr == '200' diff --git a/tests/zzz_test.py b/tests/zzz_test.py new file mode 100644 index 00000000..e55c3316 --- /dev/null +++ b/tests/zzz_test.py @@ -0,0 +1,8 @@ +def test_collect_report(host): + host.run('mkdir -p logs') + for container, filename in [('foreman', '/var/log/foreman/production.log')]: + localfile = filename.replace('/', '_') + host.run(f'podman cp {container}:{filename} logs/{container}-{localfile}') + host.run('tar caf logs.tar.gz logs/') + with open('logs.tar.gz', 'wb') as logstar: + logstar.write(host.file('logs.tar.gz').content)