Merge branch 'master' into fix/bump-wire-server-5.25-pg-secrets

Author: sghosh23

.github/workflows/changelog-verify.yml

@@ -25,15 +25,21 @@ jobs:
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then
             BASE_SHA=${{ github.event.pull_request.base.sha }}
             HEAD_SHA=${{ github.event.pull_request.head.sha }}
+
+          # Use 3-dot syntax so we only see changes introduced by this PR
+            CHANGED_FILES=$(git diff --name-only "$BASE_SHA...$HEAD_SHA" -- changelog.d/ | grep -vE "^$" || true)
           else
             # For push events, compare with the previous commit
             HEAD_SHA=${{ github.sha }}
             BASE_SHA=$(git rev-parse HEAD~1)
+            CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- changelog.d/ | grep -vE "^$" || true)
           fi
           
           echo "BASE_SHA: $BASE_SHA"
           echo "HEAD_SHA: $HEAD_SHA"
-          
+          echo "CHANGED_FILES:"
+          echo "$CHANGED_FILES"
+
           # Check if commit is by zebot with wire-build update message
           COMMIT_AUTHOR=$(git log --format="%an" -1 $HEAD_SHA)
           COMMIT_MESSAGE=$(git log --format="%s" -1 $HEAD_SHA)
@@ -44,11 +50,10 @@ jobs:
             echo "Message: $COMMIT_MESSAGE"
             exit 0
           fi
-          
-          CHANGED_FILES=$(git diff --name-only $BASE_SHA $HEAD_SHA -- changelog.d/ | grep -vE "^$")
 
           if [ -z "$CHANGED_FILES" ]; then
-            echo "No files changed in changelog.d/"
+            echo "No files changed in changelog.d/ for this ${GITHUB_EVENT_NAME:-event}."
+            echo "Every PR must add or modify at least one changelog.d/ entry."
             exit 1
           fi
 

.github/workflows/offline.yml

@@ -2,13 +2,14 @@
 #
 # This workflow builds offline deployment artifacts for different profiles:
 #   - default: Production deployment (includes external charts, ansible, terraform)
-#   - demo: Demo/WIAB deployment (includes databases-ephemeral)
+#   - build-wiab-staging: Wire-in-a-box (wiab-stag) a production like deployment (includes external charts, ansible, terraform)
+#   - wiab-dev: Wire-in-a-box dev deployment (includes databases-ephemeral)
 #   - min: Minimal deployment
 #
 # Build Optimization via PR Labels:
 #   - No label: No builds run (must add label to trigger builds)
 #   - 'build-default': Builds only default profile
-#   - 'build-demo': Builds only demo profile
+#   - 'build-dev': Builds only demo profile
 #   - 'build-wiab-staging' - Builds only wiab-staging profile
 #   - 'build-min': Builds only min profile
 #   - 'build-all': Explicitly builds all profiles (useful for workflow changes)
@@ -33,6 +34,7 @@ jobs:
   build-default:
     name: Build default profile
     if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
       contains(github.event.pull_request.labels.*.name, 'build-all') ||
       contains(github.event.pull_request.labels.*.name, 'build-default') ||
       contains(github.event.pull_request.labels.*.name, 'build-wiab-staging')
@@ -77,6 +79,7 @@ jobs:
     name: Verify default profile
     needs: build-default
     if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
       contains(github.event.pull_request.labels.*.name, 'build-all') ||
       contains(github.event.pull_request.labels.*.name, 'build-default')
     runs-on:
@@ -116,6 +119,7 @@ jobs:
     name: Verify wiab staging profile
     needs: build-default
     if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
       contains(github.event.pull_request.labels.*.name, 'build-all') ||
       contains(github.event.pull_request.labels.*.name, 'build-wiab-staging')
     runs-on:
@@ -176,11 +180,12 @@ jobs:
           DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
 
   # Build demo profile
-  build-demo:
+  build-dev:
     name: Build demo profile
     if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
       contains(github.event.pull_request.labels.*.name, 'build-all') ||
-      contains(github.event.pull_request.labels.*.name, 'build-demo')
+      contains(github.event.pull_request.labels.*.name, 'build-dev')
     runs-on:
       group: wire-server-deploy
     steps:
@@ -239,6 +244,7 @@ jobs:
   build-min:
     name: Build min profile
     if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
       contains(github.event.pull_request.labels.*.name, 'build-all') ||
       contains(github.event.pull_request.labels.*.name, 'build-min')
     runs-on:

CHANGELOG.md

@@ -12,6 +12,66 @@
 
 -->
 
+# Relase 5.23 
+
+## release-notes
+
+* Changed: wire-server updated to version 5.23.0 for prod, wiab-staging and wiab-dev/demo
+* Changed: cargohold service will use the scoped `cargohold` user with least privilege, with access limited to its `assets` bucket only (#814)
+* Changed: Enable Ansible-based RabbitMQ deployment and fix RabbitMQ host configuration for wire-server (#861)
+
+### Data stores (PostgreSQL, Cassandra)
+
+* Added: enable support for PostgreSQL deployment via Ansible (#797)
+* Added: PostgreSQL high availability cluster with repmgr (#807)
+* Changed: PostgreSQL password management is now centralized in Kubernetes Secrets (repmgr and wire-server credentials), eliminating hardcoded passwords from inventory (#819)
+* Changed: update Cassandra from 3.11.16 to 3.11.19 (#831)
+
+### Features / configuration
+* Added: config for MLS deployment into example files (#824)
+
+## wire-builds
+
+* Changed: pre_clean_values_0.sh to clean unnecessary files
+  * Removed: `patch-chart-images.sh` as it is not required anymore
+  * Fixed: default|demo|min-build definitions to have more precise values and chart definitions (#825)
+* Changed: Standardized all scripts to use `yq-go` (v4+) for YAML processing, replacing deprecated `python-yq`. Updated syntax in offline deployment scripts (`cd.sh`, `cd-with-retry.sh`), build scripts (`build_adminhost_containers.sh`), demo deployment (`offline_deploy_k8s.sh`), secret sync utilities, and chart image extraction to ensure reliable YAML manipulation and fix CI build errors (#820)
+
+## deploy-builds
+
+### WIAB demo / staging (high‑level)
+
+* Fixed: coturn and PostgreSQL secrets for demo-wiab
+  * Added: `kube-prometheus-stack` values and enabled monitoring support from wire-server for demo-wiab
+  * Added: values for wire-utility in demo-wiab (#826)
+* Added: enable `cd-demo.sh` to verify demo-wiab builds (#826)
+* Changed: add Ansible playbook for wiab-staging VM provisioning
+  * Added: Terraform resources for wiab-staging
+  * Added: `cd_staging` script to verify the default build bundle
+  * Changed: restructured `offline.yml` flow – introduced wiab-staging build and split bundle processing with default-build (#861)
+
+### Offline / CI / deployment pipeline
+
+* Added: `bin/helm-operations.sh` to replace `offline-helm` and more closely follow production instructions
+  * Changed: `bin/offline-secrets.sh` to support `helm-operations.sh` and add support for coturn secret (#858)
+* Changed: Optimize Wire offline deployment pipeline with parallel job execution and S3 direct downloads
+  * Added: retry logic with progressive server type fallbacks for Hetzner Cloud resource availability issues (#815)
+* Changed: offline workflow to require explicit labels for PR builds (`build-default`, `build-demo`, `build-min`, `build-all`); PRs without labels no longer trigger builds (#836)
+* Changed: remove hardcoded PostgreSQL passwords from `demo-secrets.example.yaml` and automatically inject passwords from `databases-ephemeral` chart during deployment (#817)
+
+## docs
+
+* Added: documentation on how to set up DKIM for SMTP in wire-server (#793)
+* Added: enable cert-manager Helm chart deployment with example values files (#805)
+* Added: wiab-staging documentation to wire-server-deploy and fixed coturn port ranges (#861)
+* Added: Enable changelog management in wire-server-deploy (#764)
+
+## bug-fixes
+* Fixed: Optimize the `offline-env` load and add pipe/redirect functionality with `d` (#812)
+* Fixed:  add localhost authentication for `postgres_exporter`, upgrade to v0.18.1, and enable `stat_checkpointer` collector for PostgreSQL 17 checkpoint metrics (#832)
+* Fixed: changelog-verify.yml workflow to allow Zebot pushes to master (#806)
+* Changed: offline-vm-setup.sh script now uses an Ubuntu cloud image and local seed ISO (#861) 
+* Fixed: Update kubernetes_logging.yml to use the standard kubelet log path instead of Docker-specific paths. (#864)
 
 # 2021-08-27
 

ansible/inventory/demo/host.yml

@@ -18,7 +18,7 @@ wiab:
     wire_ip: ""
 
     # artifact_hash
-    artifact_hash: "8e5087a0d9c58a9bd34c6c02f87514abe8b3ce0e"
+    artifact_hash: "94523acf6df5a177fd7fc1a7fdc004ce5335233b"
 
     # docker vars
     docker_ce_version: "5:28.1.1-1~ubuntu.24.04~noble"

ansible/inventory/demo/wiab-staging.yml

@@ -6,4 +6,4 @@ wiab-staging:
       ansible_user: 'demo'
       ansible_ssh_private_key_file: "~/.ssh/id_ed25519"
   vars:
-    artifact_hash: e5e1919173f6fe2b56034a57e2dc59e8343e8b8e
+    artifact_hash: f1f624256bdab0f9f76158c7f45e0618ee641237

ansible/kubernetes_logging.yml

@@ -11,9 +11,9 @@
         # options, ensure to also keep the documentation up-to-date, see the
         # documentation introduced in
         # https://github.com/wireapp/wire-docs/pull/79
-        - name: podlogs
-          path: "/var/lib/docker/containers/*/*.log"
-          options:
+         - name: podlogs
+           path: "/var/log/pods/*/*/*.log"
+           options:
             - daily
             - missingok
             - rotate 2

ansible/wiab-demo/deploy_wiab.yml

@@ -36,16 +36,62 @@
   import_playbook: ./install_pkgs.yml
   tags: install_pkgs
 
+- name: Check and configure Ansible Python interpreter for Kubernetes operations
+  tags: always
+  hosts: deploy_node
+  tasks:
+  - name: Detect available Python interpreters and Kubernetes module
+    block:
+    - name: Check if kubernetes module is available in system Python
+      shell: "python3 -c 'import kubernetes; print(kubernetes.__version__)' 2>/dev/null"
+      register: system_k8s_check
+      changed_when: false
+      failed_when: false
+
+    - name: Check virtual environment only if system Python doesn't have kubernetes
+      block:
+      - name: Check if kubernetes module is available in virtual environment
+        shell: "/opt/ansible-venv/bin/python -c 'import kubernetes; print(kubernetes.__version__)' 2>/dev/null"
+        register: venv_k8s_check
+        changed_when: false
+        failed_when: false
+
+      - name: Configure to use virtual environment
+        block:
+        - name: Set ansible_python_interpreter to use virtual environment
+          set_fact:
+            ansible_python_interpreter: /opt/ansible-venv/bin/python
+            ansible_venv_path: /opt/ansible-venv
+
+        when: venv_k8s_check.rc == 0
+
+      - name: Kubernetes module not found - run install_pkgs playbook
+        fail:
+          msg: |
+            ❌ Kubernetes Python module not found!
+            
+            System Python (/usr/bin/python3):
+              Status: NOT available
+              
+            Virtual Environment (/opt/ansible-venv/bin/python):
+              Status: NOT available
+              
+            To install kubernetes module, run:
+            ansible-playbook -i inventory.yml deploy_wiab.yml --tags install_pkgs
+        when: venv_k8s_check.rc != 0
+
+      when: system_k8s_check.rc != 0
+
 - name: Manage SSH keys (dependency for minikube, asset_host, seed_containers)
   import_playbook: ./setup_ssh.yml
   tags: always
   when: >
-    (('minikube' not in ansible_skip_tags or
+    ('minikube' not in ansible_skip_tags or
     'asset_host' not in ansible_skip_tags or
     'seed_containers' not in ansible_skip_tags)
-    and (ansible_skip_tags | length > 0))
-    or
-    ('minikube' in ansible_run_tags or
+    and
+    ('all' in ansible_run_tags or
+    'minikube' in ansible_run_tags or
     'asset_host' in ansible_run_tags or
     'seed_containers' in ansible_run_tags)
 
@@ -104,15 +150,16 @@
 
     tags: always
     when: >
-      (('minikube' not in ansible_skip_tags or
+      ('minikube' not in ansible_skip_tags or
       'asset_host' not in ansible_skip_tags or
-      'seed_containers' not in ansible_skip_tags)
-      and (ansible_skip_tags | length > 0))
-      or
-      ('minikube' in ansible_run_tags or
+      'seed_containers' not in ansible_skip_tags or
+      'helm_install' not in ansible_skip_tags)
+      and
+      ('all' in ansible_run_tags or
+      'minikube' in ansible_run_tags or
       'asset_host' in ansible_run_tags or
-      'seed_containers' in ansible_run_tags)
-      or use_cert_manager
+      'seed_containers' in ansible_run_tags or
+      'helm_install' in ansible_run_tags)
 
 - name: Configure Iptables rules
   import_playbook: ./iptables_rules.yml
@@ -126,6 +173,7 @@
   hosts: deploy_node
   become: yes
   become_user: "{{ ansible_user }}"
+  tags: always
   tasks:
   - name: Create a block for Minikube node tasks
     block:
@@ -209,13 +257,12 @@
       delegate_facts: true
       with_items: "{{ groups['k8s-cluster'] }}"
 
-    tags: always
     when: >
-      (('asset_host' not in ansible_skip_tags or
+      ('asset_host' not in ansible_skip_tags or
       'seed_containers' not in ansible_skip_tags)
-      and (ansible_skip_tags | length > 0))
-      or
-      ('asset_host' in ansible_run_tags or
+      and
+      ('all' in ansible_run_tags or
+      'asset_host' in ansible_run_tags or
       'seed_containers' in ansible_run_tags)
 
 - name: Setup Asset Host
@@ -246,11 +293,6 @@
   import_playbook: ./helm_install.yml
   tags: helm_install
 
-- name: Veirfy Cert Manager hairpin Networking
-  import_playbook: ./hairpin_networking.yml
-  tags: always
-  when: use_cert_manager
-
 # since, the temp_dir are created in a different set of tasks, these directories need to be searched 
 - name: Clean up temporary directories
   hosts: localhost

ansible/wiab-demo/helm_install.yml

@@ -271,3 +271,7 @@
         - "For more information:"
         - "https://github.com/wireapp/wire-server/tree/develop/charts/nginx-ingress-services"
     when: not use_cert_manager
+
+- name: Verify Cert Manager hairpin Networking
+  import_playbook: ./hairpin_networking.yml
+  when: use_cert_manager

ansible/wiab-demo/install_pkgs.yml

@@ -65,17 +65,30 @@
 
   - name: Ensure required Python libraries are installed for Kubernetes operations
     block:
-    - name: Install kubernetes Python library via pip
+    - name: Create Python virtual environment for Ansible
+      command: python3 -m venv /opt/ansible-venv
+      args:
+        creates: /opt/ansible-venv/bin/python
+      become: yes
+
+    - name: Install kubernetes Python library in virtual environment
       pip:
         name: 
           - kubernetes>=18.0.0
           - pyyaml>=5.4.1
-        executable: /usr/bin/pip3
+        executable: /opt/ansible-venv/bin/pip
         state: present
-        extra_args: "--break-system-packages"
       become: yes
       register: pip_install_result
 
+    - name: Create symbolic link for ansible Python interpreter
+      file:
+        src: /opt/ansible-venv/bin/python
+        dest: /usr/local/bin/ansible-python
+        state: link
+        force: yes
+      become: yes
+
     - name: Check if Docker CE is installed with correct version
       shell: apt policy docker-ce | grep "Installed:" | awk '{print $2}'
       register: installed_docker_version