Migrate flyteconnector chart wiring and runtime defaults for Flyte Connectors

charts/flyte-binary/Chart.yaml

@@ -9,3 +9,9 @@ type: application
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
 version: v0.2.0  # VERSION
+
+dependencies:
+  - name: flyteconnector
+    version: v2.0.0
+    repository: file://../flyteconnector
+    condition: flyteconnector.enabled

charts/flyte-binary/README.md

@@ -8,7 +8,7 @@ Chart for basic single Flyte executable deployment
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://../flyteconnector | flyteconnector(flyteconnector) | v0.1.10 |
+| file://../flyteconnector | flyteconnector(flyteconnector) | v2.0.0 |
 
 ## Values
 
@@ -172,4 +172,3 @@ Chart for basic single Flyte executable deployment
 | serviceAccount.imagePullSecrets | list | `[]` |  |
 | serviceAccount.labels | object | `{}` |  |
 | serviceAccount.name | string | `""` |  |
-

charts/flyteconnector/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/flyteconnector/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: flyteconnector
+description: A Helm chart for Flyte connector
+type: application
+version: v2.0.0  # VERSION

charts/flyteconnector/README.md

@@ -0,0 +1,45 @@
+# flyteconnector
+
+![Version: v2.0.0](https://img.shields.io/badge/Version-v2.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+
+A Helm chart for Flyte connector
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. |
+| additionalVolumeMounts | list | `[]` | Appends additional volume mounts to the main container's spec. May include template values. |
+| additionalVolumes | list | `[]` | Appends additional volumes to the deployment spec. May include template values. |
+| affinity | object | `{}` | affinity for flyteconnector deployment |
+| commonAnnotations | object | `{}` |  |
+| commonLabels | object | `{}` |  |
+| configPath | string | `"/etc/flyteconnector/config/*.yaml"` | Default regex string for searching configuration files |
+| connectorSecret.secretData | object | `{}` | Optional Secret manifest fragment to merge into the generated Secret, for example `stringData:` or `data:`. |
+| extraArgs | object | `{}` | Appends extra command line arguments to the main command |
+| fullnameOverride | string | `""` |  |
+| image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
+| image.repository | string | `"ghcr.io/flyteorg/flyte-connectors"` | Docker image for flyteconnector deployment |
+| image.tag | string | `"latest"` | Docker image tag |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` | nodeSelector for flyteconnector deployment |
+| podAnnotations | object | `{}` | Annotations for flyteconnector pods |
+| podEnv | object | `{}` | Additional flyteconnector pod container environment variables |
+| podLabels | object | `{}` | Labels for flyteconnector pods |
+| podSecurityContext | object | `{}` |  |
+| ports.containerPort | int | `8000` |  |
+| ports.name | string | `"connector-grpc"` |  |
+| priorityClassName | string | `""` | Sets priorityClassName for flyteconnector pod(s). |
+| rbac | object | `{"enabled":false,"rules":[{"apiGroups":[""],"resources":["secrets"],"verbs":["get"]}]}` | RBAC configuration for flyteconnector Enable to allow the connector to read secrets across namespaces (required for per-project Databricks token authentication) |
+| rbac.enabled | bool | `false` | Should RBAC resources (ClusterRole/ClusterRoleBinding) be created for flyteconnector |
+| rbac.rules | list | `[{"apiGroups":[""],"resources":["secrets"],"verbs":["get"]}]` | Rules for the ClusterRole created for flyteconnector |
+| readinessProbe | object | `{"grpc":{"port":8000},"initialDelaySeconds":1,"periodSeconds":3}` | https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out |
+| replicaCount | int | `1` | Replicas count for flyteconnector deployment |
+| resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"300Mi"},"requests":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"}}` | Default resources requests and limits for flyteconnector deployment |
+| securityContext | object | `{"allowPrivilegeEscalation":false}` | Security context for container |
+| service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for flyteconnector |
+| serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for flyteconnector |
+| serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to flyteconnector pods |
+| serviceAccount.create | bool | `true` | Should a service account be created for flyteconnector |
+| serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| tolerations | list | `[]` | tolerations for flyteconnector deployment |

charts/flyteconnector/templates/_helpers.tpl

@@ -0,0 +1,53 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "flyte.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.namespace" -}}
+{{- default .Release.Namespace .Values.forceNamespace | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{- define "flyteconnector.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyteconnector.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "flyteconnector.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "flyteconnector.labels" -}}
+{{ include "flyteconnector.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- define "flyteconnector.podLabels" -}}
+{{ include "flyteconnector.labels" . }}
+{{- with .Values.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
+# Optional blocks for secret mount
+
+{{- define "connectorSecret.volume" -}}
+{{- with .Values.connectorSecret.secretData }}
+- name: {{ include "flyte.name" $ }}
+  secret:
+    secretName: {{ include "flyte.name" $ }}
+{{- end }}
+{{- end }}
+
+{{- define "connectorSecret.volumeMount" -}}
+{{- with .Values.connectorSecret.secretData }}
+- mountPath: /etc/secrets
+  name: {{ include "flyte.name" $ }}
+{{- end }}
+{{- end }}

charts/flyteconnector/templates/connector/deployment.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels: {{ include "flyteconnector.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels: {{ include "flyteconnector.podLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.priorityClassName }}
+      priorityClassName: {{ .Values.priorityClassName }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - command:
+        - flyte
+        - serve
+        - connector
+        {{- if .Values.podEnv }}
+        env:
+        {{- with .Values.podEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+        name: flyteconnector
+        {{- if or .Values.connectorSecret.secretData .Values.additionalVolumeMounts }}
+        volumeMounts:
+        {{- include "connectorSecret.volumeMount" . | nindent 8 }}
+        {{- with .Values.additionalVolumeMounts -}}
+        {{ tpl (toYaml .) $ | nindent 8 }}
+        {{- end }}
+        {{- end }}
+        ports:
+        - containerPort: {{ .Values.ports.containerPort }}
+          name: {{ .Values.ports.name }}
+        readinessProbe:
+        {{- with .Values.readinessProbe -}}
+          {{ tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+        resources: {{- toYaml .Values.resources | nindent 10 }}
+      {{- with .Values.additionalContainers -}}
+      {{- tpl (toYaml .) $ | nindent 6}}
+      {{- end }}
+      serviceAccountName: {{ template "flyteconnector.name" . }}
+      {{- if or .Values.connectorSecret.secretData .Values.additionalVolumes }}
+      volumes: {{- include "connectorSecret.volume" . | nindent 6 }}
+      {{- with .Values.additionalVolumes -}}
+      {{ tpl (toYaml .) $ | nindent 6 }}
+      {{- end }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}

charts/flyteconnector/templates/connector/rbac.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.rbac.enabled }}
+{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+kind: ClusterRole
+metadata:
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+rules:
+  {{- toYaml .Values.rbac.rules | nindent 2 }}
+---
+{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+{{- end }}

charts/flyteconnector/templates/connector/secret.yaml

@@ -0,0 +1,9 @@
+{{- with .Values.connectorSecret.secretData }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "flyteconnector.name" $ }}
+  namespace: {{ template "flyte.namespace" $ }}
+type: Opaque
+{{ tpl (toYaml .) $ | nindent 0 }}
+{{- end }}

charts/flyteconnector/templates/connector/service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.service.type}}
+  type: {{ . }}
+  {{- end }}
+  ports:
+  - name: {{ .Values.ports.name }}
+    port: {{ .Values.ports.containerPort }}
+    protocol: TCP
+    appProtocol: TCP
+    targetPort: {{ .Values.ports.name }}
+  selector: {{ include "flyteconnector.selectorLabels" . | nindent 4 }}

charts/flyteconnector/templates/connector/serviceaccount.yaml

@@ -0,0 +1,15 @@
+---
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end}}
+{{- with .Values.serviceAccount.imagePullSecrets }}
+imagePullSecrets: {{ tpl (toYaml .) $ | nindent 2 }}
+{{- end }}
+{{- end }}