[DEV-13929] Add harness build

.harness/charts.yaml

@@ -0,0 +1,64 @@
+pipeline:
+  name: devops-charts
+  identifier: devopscharts
+  projectIdentifier: Devops
+  orgIdentifier: default
+  tags: {}
+  properties:
+    ci:
+      codebase:
+        connectorRef: account.Indico
+        repoName: devops-charts
+        build: <+input>
+  stages:
+    - parallel:
+        - stage:
+            name: push helm charts
+            identifier: push_helm_charts
+            description: ""
+            type: CI
+            spec:
+              cloneCodebase: true
+              infrastructure:
+                type: KubernetesDirect
+                spec:
+                  connectorRef: account.cicdharness
+                  namespace: harness
+                  automountServiceAccountToken: true
+                  nodeSelector:
+                    node_group: cpuxl
+                  os: Linux
+              execution:
+                steps:
+                  - step:
+                      type: Run
+                      name: push helm charts
+                      identifier: push_helm_charts
+                      spec:
+                        connectorRef: account.dockerhubproxy
+                        image: alpine/helm:3.16.2
+                        shell: Sh
+                        command: |-
+                          echo https://dl-cdn.alpinelinux.org/alpine/edge/community >> /etc/apk/repositories
+                          apk update
+                          apk add yq jq bash curl wget parallel
+                          wget -c https://github.com/shteou/helm-dependency-fetch/releases/download/v1.1.3/helm-dependency-fetch_1.1.3_Linux_x86_64.tar.gz -O - | tar -xz
+                          export PATH=${CWD}:${PATH}
+                          helm version
+                          helm plugin install https://github.com/chartmuseum/helm-push --version 0
+                          helm plugin install https://github.com/nikhilsbhat/helm-images
+                          export DRONE_TAG=<+<+codebase.tag>?<+codebase.tag>:"">
+                          ./upload_helm_charts.sh "<+<+codebase.branch>?<+codebase.branch>:<+codebase.tag>>-<+codebase.commitSha.substring(0,8)>"
+                        envVariables:
+                          PASSWORD: <+secrets.getValue("account.hashicorpvault://vaulttoolsdrone/harbor-push-token#password")>
+                          USERNAME: <+secrets.getValue("account.hashicorpvault://vaulttoolsdrone/harbor-push-token#username")>
+                          PYTHONUNBUFFERED: "1"
+                          DRONE_BRANCH: <+codebase.branch>
+                          HELM_EXPERIMENTAL_OCI: "1"
+                        resources:
+                          limits:
+                            memory: 20G
+                            cpu: "4"
+            when:
+              pipelineStatus: Success
+  allowStageExecutions: true

argo-mutator/indico.chart

@@ -0,0 +1 @@
+

argo-mutator/templates/cert.yaml

@@ -12,9 +12,9 @@ spec:
     -  {{ include "argo-mutator.fullname" . }}.{{ .Release.Namespace }}.svc
   commonName: {{ include "argo-mutator.fullname" . }}.{{ .Release.Namespace }}.svc
   secretName: {{ include "argo-mutator.fullname" . }}
-  duration: 87600h
+  duration: 8760h
   renewBefore: 720h
   issuerRef:
     name: selfsigned-issuer
     kind: ClusterIssuer
-    group: cert-manager.io
+    group: cert-manager.io

argo-mutator/templates/mutate.yaml

@@ -3,7 +3,7 @@ kind: MutatingWebhookConfiguration
 apiVersion: admissionregistration.k8s.io/v1
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.admissionController.certificateName }}
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "argo-mutator.fullname" . }}
   name: {{ include "argo-mutator.fullname" . }}-mutating-webhook
 webhooks:
   - name: {{ .Values.mutate.serviceName }}.{{ .Release.Namespace }}.svc
@@ -45,4 +45,4 @@ webhooks:
     sideEffects: "None"
     admissionReviewVersions: ["v1"]
     timeoutSeconds: 5
-{{- end }}
+{{- end }}

argo-mutator/templates/validate.yaml

@@ -3,7 +3,7 @@ kind: ValidatingWebhookConfiguration
 apiVersion: admissionregistration.k8s.io/v1
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.admissionController.certificateName }}
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "argo-mutator.fullname" . }}
   name: {{ include "argo-mutator.fullname" . }}-validating-webhook
 webhooks:
   - name: {{ .Values.validate.serviceName }}.{{ .Release.Namespace }}.svc

argo-mutator/values.yaml

@@ -4,9 +4,6 @@
 
 replicaCount: 2
 
-admissionController:
-  certificateName: adm-argo-mutator
-
 image:
   repository: harbor.devops.indico.io/indico/devops-argo-mutator
   pullPolicy: Always

devops-keycloak-service/indico.chart

@@ -0,0 +1 @@
+

required_helm_repos.txt

@@ -0,0 +1 @@
+	https://prometheus-community.github.io/helm-charts

scripts/build_helm_dependencies_new.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# with the new method we know that all dependent charts are already built
+set -euxo pipefail
+dir=$1
+
+if [ -d "$dir/charts" ]; then
+  rm -rf $dir/charts
+  mkdir $dir/charts
+else 
+  mkdir $dir/charts
+fi
+
+for local_chart in $(helm dependency list $dir | grep "file://" | awk '{print $2}' | sed -n -e 's/^.*file:\/\/..\///p' )
+do
+  local_chart_tgz=$(ls .built_local_charts | grep "^$local_chart-v*[0-9]*\.[0-9]*\.[0-9]*\.tgz")
+  cp ./.built_local_charts/$local_chart_tgz $dir/charts
+done
+
+IFS=$'\n' # make newlines the only separator
+for external_chart in $(helm dependency list $dir | grep -E 'https://|oci://' )
+do
+  chart=$(echo $external_chart | awk '{print $1}')
+  version=$(echo $external_chart | awk '{print $2}')
+  source_registry=$(echo $external_chart | awk '{print $3}')
+
+  if [[ $source_registry == oci://* ]]; then
+    helm pull $source_registry/$chart --version $version --destination $dir/charts
+  else
+    registry_name=$(cat .helm_repo_map | grep "$source_registry" | uniq | awk '{print $1}')
+
+    helm pull $registry_name/$chart --version $version --destination $dir/charts
+  fi
+done
+unset IFS
+
+helm package $dir --destination ./.built_local_charts
+
+mkdir -p .built_oci_charts
+
+branch=${BRANCH_NAME//\//\-} # replace slashes with -
+branch=${branch//_/\-} # replace underscores with -
+
+chart_version=$(helm show chart $dir | grep '^version:' | awk '{print $2}')
+
+if [ ! -z "$DRONE_TAG" ]; then
+  version=$chart_version-${DRONE_TAG}
+else
+  version=$chart_version-$branch
+fi
+
+helm package $dir --version "$version" --destination ./.built_oci_charts

scripts/dependency_map_and_build.sh

@@ -0,0 +1,46 @@
+rm -f .charts
+rm -f .charts-tmp
+rm -f .tmp
+
+for fullpath in $(find . -name Chart.yaml | sort)
+do
+  echo $(dirname $fullpath) >> .charts
+done
+
+count=0
+
+cp .charts .charts-ext
+
+while [ -s ".charts" ]; do
+cp .charts .charts-tmp
+rm -f ".layer${count}"
+touch ".layer${count}"
+for dir in $(cat .charts)
+do
+  waiting=0
+  # check all local dependencies, if they aren't marked completed then this chart can't be either
+  for dep in $(helm dependency list $dir | grep 'file://' | awk '{print $2}' | sed -n -e 's/^.*file:\/\/..\///p')
+  do
+    if [ $(grep -Rx "./$dep" .charts) ]; then
+      waiting=1
+    fi
+  done
+  if [ $waiting = "0" ]; then
+    echo $dir >> ".layer${count}"
+    cat .charts-tmp | grep -vx $dir > .tmp 
+    mv .tmp .charts-tmp
+  fi
+done
+count=$((count + 1))
+mv .charts-tmp .charts
+done
+
+echo "Done building dependency map"
+
+for layerfile in $(ls -lart | grep .layer | awk '{print $9}' | sort)
+do
+  cat $layerfile | parallel --halt-on-error 1 -k --joblog .dependent-results-${i} -j 16 ./scripts/build_helm_dependencies_new.sh
+  cat .dependent-results-${i}
+done
+
+

scripts/push_helm_chart.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e
+fullpath=${1#"./"}
+BRANCH_NAME=$2  # e.g: "main-xxxxxx"
+
+dir=$(dirname $fullpath)
+name=$(helm show chart $dir | yq '.name')
+version=$(helm show chart $dir | yq '.version')
+chart_version=$version
+
+echo " "
+echo "-------------------------------------------------------------------------------------------"
+
+branch=${BRANCH_NAME//\//\-} # replace slashes with -
+branch=${branch//_/\-} # replace underscores with -
+
+if [ ! -z "$DRONE_TAG" ]; then
+  version=$version-${DRONE_TAG}
+else
+  version=$version-$branch
+fi
+
+num_charts=$((num_charts+1))
+
+echo "Working on Chart $name, Version: $version"
+
+if [ -d "$dir/tests" ]
+then
+  for testfile in $(find $dir/tests -name '*.yaml')
+  do
+    testname=$(basename "$testfile")
+    echo "Running test with $testfile"
+
+    echo helm template ./$dir --dependency-update --name-template $testname --namespace default --kube-version 1.27 --values $testfile --include-crds --debug > /dev/null
+    helm template ./$dir --dependency-update --name-template $testname --namespace default --kube-version 1.27 --values $testfile --include-crds --debug > /dev/null
+
+
+    echo "Linting chart"
+    helm lint ./$dir --values $testfile
+
+    echo "Images referenced"
+    helm template ./$dir --dependency-update --name-template $testname --namespace default --kube-version 1.27 --values $testfile --include-crds | yq '..|.image? | select(.)' | sort -u
+  done
+fi
+
+#Push chart, check if it succeeded, if not, retry.
+oci_chart_tgz=$(ls .built_oci_charts | grep "^$dir-v*[0-9]*\.[0-9]*\.[0-9]*-.*.tgz")
+pushed="false"
+retry_attempts=10
+until [ $pushed == "true" ] || [ $retry_attempts -le 0 ]
+do
+  if [ $retry_attempts -ne 10 ]; then
+    echo "Retry push ${dir} [$retry_attempts]"
+    sleep 10
+  fi
+
+  set +e
+  echo "helm push .built_oci_charts/$oci_chart_tgz oci://harbor.devops.indico.io/indico-charts [$retry_attempts]"
+  if helm push .built_oci_charts/$oci_chart_tgz oci://harbor.devops.indico.io/indico-charts; then
+    echo "\t--> oci://harbor.devops.indico.io/indico-charts/${name}:${version}\n" >> .pushed
+    pushed="true"
+  else
+    pushed="false"
+  fi
+  set -e
+  ((retry_attempts--))
+done
+
+echo "done"
+set -e
+# double-check that the chart was pushed.
+if [ $retry_attempts -le 0 ]; then
+  echo "Error: Unable to push oci://harbor.devops.indico.io/indico-charts/${name}:${version}"
+  exit 1
+fi
+
+
+
+
+
+
+