Merge branch 'master' into efoley-zuul-ocp-4.16

Author: elfiesmelfie

.zuul.yaml

@@ -132,6 +132,15 @@
     vars:
       scenario: "local_build-index_deploy"
 
+- job:
+    name: stf-crc-nightly_bundles-index_deploy
+    parent: stf-base
+    abstract: true
+    description: |
+      Create an index image using nightly bundles and deploy STF from it
+    vars:
+      scenario: "nightly_bundles-index_deploy"
+
 - job:
     name: stf-crc-ocp_412-nightly_bundles
     parent: stf-crc-nightly_bundles
@@ -188,14 +197,27 @@
       Build STF locally and deploy from index on OCP 4.14
     nodeset: stf-crc_extracted-ocp414
 
-    # TODO: add the other job variations for 4.16
 - job:
     name: stf-crc-ocp_416-local_build-index_deploy
     parent: stf-crc-local_build-index_deploy
     description: |
       Build STF locally and deploy from index on OCP 4.16
     nodeset: stf-crc_extracted-ocp416
 
+- job:
+    name: stf-crc-ocp_414-nightly_bundles-index_deploy
+    parent: stf-crc-nightly_bundles-index_deploy
+    description: |
+      Create an index image using nightly bundles and deploy STF from it on OCP 4.14
+    nodeset: stf-crc_extracted-ocp414
+
+- job:
+    name: stf-crc-ocp_416-nightly_bundles-index_deploy
+    parent: stf-crc-nightly_bundles-index_deploy
+    description: |
+      Create an index image using nightly bundles and deploy STF from it on OCP 4.14
+    nodeset: stf-crc_extracted-ocp416
+
 - project-template:
     name: stf-crc-jobs
     description: |
@@ -208,6 +230,8 @@
         #- stf-crc-ocp_412-local_build-index_deploy
         #- stf-crc-ocp_414-local_build-index_deploy
         - stf-crc-ocp_416-local_build-index_deploy
+        # - stf-crc-ocp_414-nightly_bundles-index_deploy
+        - stf-crc-ocp_416-nightly_bundles-index_deploy
         - stf-crc-ocp_414-nightly_bundles
         - stf-crc-ocp_416-nightly_bundles
 
@@ -218,4 +242,4 @@
     periodic:
       jobs:
         - stf-crc-ocp_412-nightly_bundles
-        - stf-crc-ocp_414-nightly_bundles
+        - stf-crc-ocp_414-nightly_bundles

README.md

@@ -40,6 +40,7 @@ loaded for you.
 * smartgatewayCollectdEventsManifest
 * smartgatewayCeilometerEventsManifest
 * servicemonitorManifest
+* scrapeconfigManifest
 
 ## Development
 

build/stf-run-ci/README.md

@@ -20,7 +20,7 @@ choose to override:
 | `__deploy_stf`                                                       | {true,false}                                                | true                                                                          | Whether to deploy an instance of STF                                                                                                                                                                                    |
 | `__local_build_enabled`                                              | {true,false}                                                | true                                                                          | Whether to deploy STF from local built artifacts. Also see `working_branch`, `sg_branch`, `sgo_branch`                                                                                                                  |
 | `__deploy_from_bundles_enabled`                                      | {true,false}                                                | false                                                                         | Whether to deploy STF from OLM bundles (TODO: compat with `__local_build_enabled`)                                                                                                                                      |
-| `__deploy_from_index_enabled`                                        | {true,false}                                                | false                                                                         | Whether to deploy STF from locally built bundles and index image.                                                                                                                                                       |
+| `__deploy_from_index_enabled`                                        | {true,false}                                                | false                                                                         | Whether to deploy STF from locally built bundles/OLM bundles and index image.                                                                                                                                                       |
 | `__disconnected_deploy`                                              | {true,false}                                                | false                                                                         | Whether to deploy on a disconnected cluster                                                                                                                                                                               |
 | `__service_telemetry_bundle_image_path`                              | <image_path>                                                | `quay.io/infrawatch-operators/service-telemetry-operator-bundle:nightly-head` | Image path to Service Telemetry Operator bundle                                                                                                                                                                         |
 | `__smart_gateway_bundle_image_path`                                  | <image_path>                                                | `quay.io/infrawatch-operators/smart-gateway-operator-bundle:nightly-head`     | Image path to Smart Gateway Operator bundle                                                                                                                                                                             |
@@ -83,9 +83,10 @@ choose to override:
 You can deploy Service Telemetry Framework using this role in a few
 configuration methods:
 
-* local build artifacts from Git repository cloned locally
-* local build artifacts, local bundle artifacts, and Subscription via OLM using locally built index image
-* standard deployment using Subscription and OLM
+* local build artifacts from Git repository cloned locally (local build)
+* local build artifacts, local bundle artifacts, and Subscription via OLM using locally built index image (local build + deploy from index)
+* externally build bundle artifacts and Subscription via OLM using locally built index image (deploy from bundles + deploy from index)
+* standard deployment using Subscription and OLM (deploy from bundles)
 * supporting components but no instance of Service Telemetry Operator
 
 ## Basic deployment
@@ -134,6 +135,27 @@ You can perform a deployment using OLM and a Subscription from locally built art
 ansible-playbook -e __local_build_enabled=true -e __deploy_from_index_enabled=true run-ci.yaml
 ```
 
+## Deployment with pre-build bundles and index
+
+Instead of relying on the operator-sdk to deploy from selected bundles using the "operator-sdk run bundle" utility,
+you can perform a deployment using OLM and a Subscription to a locally created index image like this:
+
+```sh
+ansible-playbook -e __local_build_enabled=false -e __deploy_from_bundles_enabled=true \
+  -e __deploy_from_index_enabled=true \
+  -e __service_telemetry_bundle_image_path=<registry>/<namespace>/stf-service-telemetry-operator-bundle:<tag> \
+  -e __smart_gateway_bundle_image_path=<registry>/<namespace>/stf-smart-gateway-operator-bundle:<tag> \
+  -e pull_secret_registry=<registry> \
+  -e pull_secret_user=<username> \
+  -e pull_secret_pass=<password>
+  run-ci.yaml
+```
+
+Since you will fetch the selected images from a bundle registry, it is required that you have all the required
+access credentials for the desired registry correctly configured. Check the "Deployment with pre-build bundles"
+docs above to get more information about this.
+
+
 # License
 
 Apache v2.0

build/stf-run-ci/tasks/create_catalog.yml

@@ -7,11 +7,65 @@
 
 # Updating to use stdout_lines[-1] so that any additional info that gets added to generate_bundles (e.g. for debug) doesn't break this task
 # Adding from_json so that the JSON output is parsed into a dictionary
-- name: Create info variables from bundle generation output
+- name: Create info variables from bundle generation output (local build)
+  when: __local_build_enabled | bool and not __deploy_from_bundles_enabled | bool
   ansible.builtin.set_fact:
     sto_bundle_info: "{{ generate_bundle_sto.stdout_lines[-1] | from_json }}"
     sgo_bundle_info: "{{ generate_bundle_sgo.stdout_lines[-1] | from_json }}"
 
+- name: Create info variables from provided pre-built bundles (deploy from bundles)
+  when: __deploy_from_bundles_enabled | bool and not __local_build_enabled | bool
+  block:
+    - name: Get STO operator bundle info
+      ansible.builtin.command: oc image info {{ __service_telemetry_bundle_image_path }}
+      register: sto_prebuilt_image_info
+
+    - name: Get SGO operator bundle info
+      ansible.builtin.command: oc image info {{ __smart_gateway_bundle_image_path }}
+      register: sgo_prebuilt_image_info
+
+    - name: Get STO and SGO bundle versions
+      ansible.builtin.set_fact:
+        sto_prebuilt_bundle_version: "{{ sto_prebuilt_image_info.stdout_lines[-1] | split('=') | last }}"
+        sgo_prebuilt_bundle_version: "{{ sgo_prebuilt_image_info.stdout_lines[-1] | split('=') | last }}"
+
+    - name: Set info variables from provided pre-built bundles
+      ansible.builtin.set_fact:
+        sto_bundle_info:
+          'bundle_default_channel': "{{ stf_channel }}"
+          'bundle_channels': "{{ stf_channel }}"
+          'operator_bundle_version': "{{ sto_prebuilt_bundle_version }}"
+        sgo_bundle_info:
+          'bundle_default_channel': "{{ stf_channel }}"
+          'bundle_channels': "{{ stf_channel }}"
+          'operator_bundle_version': "{{ sgo_prebuilt_bundle_version }}"
+
+- name: Show STO and SGO bundle info that will used in the index image
+  ansible.builtin.debug:
+    msg:
+    - "{{ sto_bundle_info }}"
+    - "{{ sgo_bundle_info }}"
+
+- name: Create ImageStream for STO and SGO (deploying from bundles)
+  when: __deploy_from_bundles_enabled | bool
+  block:
+    - name: Set correct STO and SGO bundle paths when deploying from index with pre-built bundles
+      ansible.builtin.set_fact:
+        sto_bundle_image_path: "{{ __internal_registry_path }}/{{ namespace }}/service-telemetry-operator-bundle:{{ sto_bundle_image_tag }}"
+        sgo_bundle_image_path: "{{ __internal_registry_path }}/{{ namespace }}/smart-gateway-operator-bundle:{{ sgo_bundle_image_tag }}"
+
+    - name: Create ImageStream for STO
+      ansible.builtin.command:
+        cmd: |
+          oc import-image -n {{ namespace }} service-telemetry-operator-bundle:{{ sto_bundle_image_tag }} --from={{ __service_telemetry_bundle_image_path }} --confirm --insecure
+      register: sto_is
+
+    - name: Create ImageStream for SGO
+      ansible.builtin.command:
+        cmd: |
+          oc import-image -n {{ namespace }} smart-gateway-operator-bundle:{{ sgo_bundle_image_tag }} --from={{ __smart_gateway_bundle_image_path }} --confirm --insecure
+      register: sgo_is
+
 - name: Get the builder-dockercfg Secret name
   ansible.builtin.command: oc get secret -n {{ namespace }} --field-selector='type==kubernetes.io/dockercfg' -ojsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="builder")].metadata.name}'
   register: secret_builder_dockercfg_name

build/stf-run-ci/tasks/main.yml

@@ -28,27 +28,6 @@
     - __local_build_enabled | bool
     - __deploy_from_bundles_enabled | bool
 
-- name: Fail when deploying from index image and local build disabled
-  ansible.builtin.fail:
-    msg: __deploy_from_index_enabled must also have __local_build_enabled
-  when:
-   - __deploy_from_index_enabled | bool
-   - not __local_build_enabled | bool
-
-- name: Fail when deploying from index images and deployment from bundles also requested (mutually exclusive methods)
-  ansible.builtin.fail:
-    msg: __deploy_from_index_enabled can not be used with __deploy_from_bundles_enabled
-  when:
-   - __deploy_from_index_enabled | bool
-   - __deploy_from_bundles_enabled | bool
-
-- name: Fail when disconnected deploy and other deployment options also requested
-  ansible.builtin.fail:
-    msg: __disconnected_deploy cannot be used if __deploy_from_bundles_enabled, __deploy_from_index_enabled or __local_build_enabled
-  when:
-   - __disconnected_deploy | bool
-   - __deploy_from_bundles_enabled | bool or __deploy_from_index_enabled | bool or __local_build_enabled | bool 
-
 - name: Get the list of nodes
   kubernetes.core.k8s_info:
     kind: Node
@@ -159,7 +138,7 @@
           pod-security.kubernetes.io/audit: restricted
           pod-security.kubernetes.io/warn: restricted
 
-- when: __deploy_from_index_enabled | bool
+- when: __deploy_from_index_enabled | bool and __local_build_enabled | bool
   tags:
     - create_bundles
   block:
@@ -180,11 +159,13 @@
       tags:
         - build
 
+- when: __deploy_from_index_enabled | bool
+  block:
     - name: Create file-based catalog
       ansible.builtin.include_tasks: create_catalog.yml
 
 # -- deploy
-- when: not __local_build_enabled | bool
+- when: not __local_build_enabled | bool and not __deploy_from_index_enabled | bool
   block:
   - name: Setup Service Telemetry Framework from supplied bundle URLs
     ansible.builtin.include_tasks: setup_stf_from_bundles.yml
@@ -206,7 +187,7 @@
         name: service-telemetry-operator
         namespace: "{{ namespace }}"
       spec:
-        channel: unstable
+        channel: "{{ stf_channel }}"
         installPlanApproval: Automatic
         name: service-telemetry-operator
         source: service-telemetry-framework-operators

ci/vars-nightly_bundles-index_deploy.yml

@@ -0,0 +1,5 @@
+---
+# ansible-playbook -e __local_build_enabled=false -e __deploy_from_index_enabled=true -e __deploy_from_bundles_enabled=true -e __service_telemetry_bundle_image_path=quay.io/infrawatch-operators/service-telemetry-operator-bundle:nightly-head -e __smart_gateway_bundle_image_path=quay.io/infrawatch-operators/smart-gateway-operator-bundle:nightly-head --skip-tags bundle_registry_tls_ca --skip-tags bundle_registry_auth build/run-ci.yaml
+__local_build_enabled: false
+__deploy_from_bundles_enabled: true
+__deploy_from_index_enabled: true

deploy/olm-catalog/service-telemetry-operator/manifests/service-telemetry-operator.clusterserviceversion.yaml

@@ -203,6 +203,15 @@ spec:
       - kind: ServiceMonitors
         name: servicemonitors.monitoring.coreos.com
         version: v1
+      - kind: ScrapeConfigs
+        name: scrapeconfigs.monitoring.coreos.com
+        version: v1alpha1
+      - kind: ServiceMonitors
+        name: servicemonitors.monitoring.rhobs
+        version: v1
+      - kind: ScrapeConfigs
+        name: scrapeconfigs.monitoring.rhobs
+        version: v1alpha1
       version: v1beta1
   description: Service Telemetry Operator for monitoring clouds
   displayName: Service Telemetry Operator
@@ -378,17 +387,21 @@ spec:
         - apiGroups:
           - monitoring.coreos.com
           resources:
+          - scrapeconfigs
           - servicemonitors
           verbs:
           - get
           - create
+          - delete
         - apiGroups:
           - monitoring.rhobs
           resources:
+          - scrapeconfigs
           - servicemonitors
           verbs:
           - get
           - create
+          - delete
         - apiGroups:
           - apps
           resourceNames:

deploy/role.yaml

@@ -129,17 +129,21 @@ rules:
 - apiGroups:
   - monitoring.coreos.com
   resources:
+  - scrapeconfigs
   - servicemonitors
   verbs:
   - get
   - create
+  - delete
 - apiGroups:
   - monitoring.rhobs
   resources:
+  - scrapeconfigs
   - servicemonitors
   verbs:
   - get
   - create
+  - delete
 - apiGroups:
   - apps
   resourceNames:

roles/servicetelemetry/tasks/base_smartgateway.yml

@@ -6,8 +6,8 @@
   k8s:
     definition: "{{ lookup('template', manifest) | from_yaml }}"
 
-- name: Deploy SG-specific ServiceMonitor for metrics SGs
-  include_tasks: component_servicemonitor.yml
+- name: Deploy SG-specific ScrapeConfig for metrics SGs
+  include_tasks: component_scrapeconfig.yml
   when:
     - data_type == 'metrics'
     - has_monitoring_api | bool