Build DOCA-OFED kernel modules for Rocky 10

DOCA OFED support has now been added for Rocky 10:

* Build workflow updated to include Rocky 10 (x86_64 and aarch64
builds)
* Rocky 9.6 to 9.7 OFED upgrade playbook hardcoded to use 9.7 kernel
* Installation playbook now uses host distribution version
* Retries added to pulp pushing playbook
* Pulp pushing playbook can now handle multiple OFED repositories
(for Rocky 9 and 10) being defined
* Updates to Pulp, repository and OFED tag definitions, to handle
both Rocky 9 and 10

Author: owenjones

.github/workflows/package-build-ofed.yml

@@ -18,7 +18,7 @@
     # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
     # but not 5.14.0-611.20.1.
     doca_kernel_arch: "{{ 'x86.64' if ansible_facts.architecture == 'x86_64' else ansible_facts.architecture }}"
-    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}.{{ doca_kernel_arch }}"
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix['9.7'] }}.{{ doca_kernel_arch }}"
   tasks:
     - name: Assert that hosts are running Rocky Linux 9.6
       ansible.builtin.assert:

etc/kayobe/ansible/maintenance/rocky-97-ofed-upgrade.yml

@@ -12,7 +12,7 @@
       changed_when: false
       check_mode: true
       register: result
-      failed_when: result.found
+      failed_when: result.found > 0
 
     - name: Install package dependencies
       ansible.builtin.dnf:

etc/kayobe/ansible/tools/build-ofed-rocky.yml

@@ -7,7 +7,7 @@
     # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
     # but not 5.14.0-611.20.1
     doca_kernel_arch: "{{ 'x86.64' if ansible_facts.architecture == 'x86_64' else ansible_facts.architecture }}"
-    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}.{{ doca_kernel_arch }}"
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[ansible_facts.distribution_version | string] }}.{{ doca_kernel_arch }}"
   tasks:
     - name: Install kernel repo
       ansible.builtin.dnf:

etc/kayobe/ansible/tools/install-doca.yml

@@ -7,13 +7,13 @@
   tasks:
     - name: Get OFED module repo
       ansible.builtin.set_fact:
-        doca_modules_repos: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | selectattr('base_path', 'search', '/' ~ kolla_base_arch ~ '/') | list }}"
+        doca_modules_repos: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | selectattr('base_path', 'search', '/rhel' ~ os_release) | selectattr('base_path', 'search', '/' ~ os_arch ~ '/') | list }}"
 
     - name: Assert OFED module repo is unique
       ansible.builtin.assert:
         that:
           - doca_modules_repos | length == 1
-        fail_msg: "Expected exactly one OFED module repo for {{ kolla_base_arch }}"
+        fail_msg: "Expected exactly one OFED module repo for {{ os_arch }}"
 
     - name: Get OFED module repo variables
       ansible.builtin.set_fact:
@@ -46,7 +46,10 @@
         password: "{{ stackhpc_release_pulp_password }}"
         name: "{{ doca_modules_repo_name }}"
         state: present
-      retries: "{{ pulp_timeout_retries | default(3) }}"
+      register: repository
+      until: repository is success
+      retries: "{{ pulp_timeout_retries | default(3) | int }}"
+      delay: "{{ pulp_delay | default(5) | int }}"
 
     - name: Upload OFED RPMs to Pulp
       pulp.squeezer.rpm_package:
@@ -59,6 +62,9 @@
         state: present
       with_items: "{{ doca_kernel_repo.stdout_lines }}"
       register: package
+      until: package is success
+      retries: "{{ pulp_timeout_retries | default(3) | int }}"
+      delay: "{{ pulp_delay | default(5) | int }}"
 
     - name: Create Pulp publication for OFED
       pulp.squeezer.rpm_publication:

etc/kayobe/ansible/tools/push-ofed.yml

@@ -55,7 +55,7 @@ stackhpc_dnf_repos:
 dnf_custom_repos_doca_rocky_9:
   doca:
     baseurl: "{{ stackhpc_repo_rhel9_doca_url }}"
-    description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
+    description: "DOCA Online Repo {{ stackhpc_pulp_rocky_9_doca_version }} - RHEL $releasever"
     enabled: "{{ dnf_enable_doca | bool | default(false) }}"
     priority: -1
     file: doca
@@ -64,7 +64,7 @@ dnf_custom_repos_doca_rocky_9:
     password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   doca-modules:
     baseurl: "{{ stackhpc_repo_rhel9_doca_modules_url }}"
-    description: "OFED Kernel module repository for DOCA {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
+    description: "OFED Kernel module repository for DOCA {{ stackhpc_pulp_rocky_9_doca_version }} - RHEL $releasever"
     enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}"
     priority: -1
     file: doca
@@ -75,23 +75,22 @@ dnf_custom_repos_doca_rocky_9:
 dnf_custom_repos_doca_rocky_10:
   doca:
     baseurl: "{{ stackhpc_repo_rhel10_doca_url }}"
-    description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
+    description: "DOCA Online Repo {{ stackhpc_pulp_rocky_10_doca_version }} - RHEL $releasever"
     enabled: "{{ dnf_enable_doca | bool | default(false) }}"
     priority: -1
     file: doca
     gpgcheck: no
     username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
     password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
-    # TODO(owenjones): update this once repo for Rocky 10 has been created in Pulp
-    # doca-modules:
-    #   baseurl: "{{ stackhpc_repo_rhel9_doca_modules_url }}"
-    #   description: "OFED Kernel module repository for DOCA {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
-    #   enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}"
-    #   priority: -1
-    #   file: doca
-    #   gpgcheck: no
-    #   username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
-    #   password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+  doca-modules:
+    baseurl: "{{ stackhpc_repo_rhel10_doca_modules_url }}"
+    description: "OFED Kernel module repository for DOCA {{ stackhpc_pulp_rocky_10_doca_version }} - RHEL $releasever"
+    enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}"
+    priority: -1
+    file: doca
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Custom repositories shared between all RHEL 9/10 derivatives.
 dnf_custom_repos_el9:

etc/kayobe/dnf.yml

@@ -47,6 +47,7 @@ stackhpc_repo_rhel_10_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rh
 stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
 stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
 stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel10_doca_modules_version: "{{ stackhpc_pulp_repo_rhel10_doca_modules_version }}"
 stackhpc_repo_rhel10_doca_version: "{{ stackhpc_pulp_repo_rhel10_doca_version }}"
 stackhpc_repo_rhel9_doca_modules_version: "{{ stackhpc_pulp_repo_rhel9_doca_modules_version }}"
 stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"

etc/kayobe/environments/aio/release-train.yml

@@ -4,14 +4,21 @@
 ###############################################################################
 # DOCA host version
 stackhpc_pulp_doca_version_matrix:
-    "6": 2.9.3
-    "7": 3.2.2
-stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] | default('2.9.1') }}"
+    "9.6": '2.9.3'
+    "9.7": '3.2.2'
+    "10.2": '3.2.2'
+
+doca_rocky_9_release: 9.{{ stackhpc_pulp_repo_rocky_9_minor_version | string }}
+doca_rocky_10_release: 10.{{stackhpc_pulp_repo_rocky_10_minor_version | string }}
+
+stackhpc_pulp_rocky_9_doca_version: "{{ stackhpc_pulp_doca_version_matrix[doca_rocky_9_release] | default('2.9.3') }}"
+stackhpc_pulp_rocky_10_doca_version: "{{ stackhpc_pulp_doca_version_matrix[doca_rocky_10_release] | default('3.2.2') }}"
 
 # Available and tested versions of the pre-compiled doca-ofed kernel modules
 stackhpc_doca_kernel_version_matrix:
-    "6": 5.14.0.570.21.1.el9.6
-    "7": 5.14.0.611.55.1.el9.7
+    "9.6": 5.14.0.570.21.1.el9.6
+    "9.7": 5.14.0.611.55.1.el9.7
+    "10.2": 6.12.0.211.16.1.el10.2.0.1
 
 ###############################################################################
 # Pulp configuration for DOCA OFED
@@ -20,19 +27,29 @@ stackhpc_doca_kernel_version_matrix:
 stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}"
 
 # DOCA Snapshot lookup vars
-doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}9{% else %}9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}"
+doca_rocky_9_minor_dot: "{% if stackhpc_pulp_rocky_9_doca_version is version('3.2.0', '>=') %}9{% else %}{{ doca_rocky_9_release }}{% endif %}"
 # eg stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version or stackhpc_pulp_repo_doca_3_2_2_rhel9_version
-doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version"
-doca_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_aarch64_version"
-doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version"
-doca_modules_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_aarch64_version"
+doca_rocky_9_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_9_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version"
+doca_rocky_9_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_9_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_aarch64_version"
+doca_rocky_9_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_9_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version"
+doca_rocky_9_modules_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_9_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_aarch64_version"
+
+doca_rocky_10_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_10_doca_version | replace('.', '_') }}_rhel10_x86_64_version"
+doca_rocky_10_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_10_doca_version | replace('.', '_') }}_rhel10_aarch64_version"
+doca_rocky_10_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_10_doca_version | replace('.', '_') }}_rhel10_{{ stackhpc_pulp_repo_rocky_10_minor_version }}_modules_version"
+doca_rocky_10_modules_aarch64_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_rocky_10_doca_version | replace('.', '_') }}_rhel10_{{ stackhpc_pulp_repo_rocky_10_minor_version }}_modules_aarch64_version"
 
 # DOCA Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml
-stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', doca_version_lookup_var) }}"
-stackhpc_pulp_repo_rhel9_doca_aarch64_version: "{{ lookup('vars', doca_aarch64_version_lookup_var, default=stackhpc_pulp_repo_rhel9_doca_version) }}"
-stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', doca_modules_version_lookup_var) }}"
-stackhpc_pulp_repo_rhel9_doca_modules_aarch64_version: "{{ lookup('vars', doca_modules_aarch64_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', doca_rocky_9_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel9_doca_aarch64_version: "{{ lookup('vars', doca_rocky_9_aarch64_version_lookup_var, default=stackhpc_pulp_repo_rhel9_doca_version) }}"
+stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', doca_rocky_9_modules_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel9_doca_modules_aarch64_version: "{{ lookup('vars', doca_rocky_9_modules_aarch64_version_lookup_var) }}"
+
+stackhpc_pulp_repo_rhel10_doca_version: "{{ lookup('vars', doca_rocky_10_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel10_doca_aarch64_version: "{{ lookup('vars', doca_rocky_10_aarch64_version_lookup_var, default=stackhpc_pulp_repo_rhel10_doca_version) }}"
+stackhpc_pulp_repo_rhel10_doca_modules_version: "{{ lookup('vars', doca_rocky_10_modules_version_lookup_var) }}"
+stackhpc_pulp_repo_rhel10_doca_modules_aarch64_version: "{{ lookup('vars', doca_rocky_10_modules_aarch64_version_lookup_var) }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/ofed.yml

@@ -28,10 +28,12 @@ stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314
 stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301
 stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841
 stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021
+stackhpc_pulp_repo_doca_3_2_2_rhel10_2_modules_aarch64_version: 20260615T122946
+stackhpc_pulp_repo_doca_3_2_2_rhel10_2_modules_version: 20260615T122946
 stackhpc_pulp_repo_doca_3_2_2_rhel10_aarch64_version: 20260326T091359
 stackhpc_pulp_repo_doca_3_2_2_rhel10_x86_64_version: 20260326T091359
-stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_aarch64_version: 20260611T055845
-stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_version: 20260611T055845
+stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_aarch64_version: 20260615T122946
+stackhpc_pulp_repo_doca_3_2_2_rhel9_7_modules_version: 20260615T122946
 stackhpc_pulp_repo_doca_3_2_2_rhel9_aarch64_version: 20260326T091359
 stackhpc_pulp_repo_doca_3_2_2_rhel9_version: 20260326T091359
 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20260506T205129