diff --git a/charts/flyte-core/Chart.yaml b/charts/flyte-core/Chart.yaml new file mode 100644 index 0000000000..d5a7d8ec0d --- /dev/null +++ b/charts/flyte-core/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: flyte-core +description: Helm chart for Flyte core that each component deployed as separate services +type: application +version: "0.1.0" +appVersion: "1.0.0" diff --git a/charts/flyte-core/templates/_helpers.tpl b/charts/flyte-core/templates/_helpers.tpl new file mode 100644 index 0000000000..204cc1dcee --- /dev/null +++ b/charts/flyte-core/templates/_helpers.tpl @@ -0,0 +1,164 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "flyte-core.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "flyte-core.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart label value. +*/}} +{{- define "flyte-core.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels applied to every resource. +*/}} +{{- define "flyte-core.labels" -}} +helm.sh/chart: {{ include "flyte-core.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: flyte +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Component labels — call with (dict "ctx" $ "component" "runs"). +*/}} +{{- define "flyte-core.componentLabels" -}} +{{ include "flyte-core.labels" .ctx }} +app.kubernetes.io/name: {{ include "flyte-core.name" .ctx }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{/* +Selector labels for a component — call with (dict "ctx" $ "component" "runs"). +*/}} +{{- define "flyte-core.selectorLabels" -}} +app.kubernetes.io/name: {{ include "flyte-core.name" .ctx }} +app.kubernetes.io/instance: {{ .ctx.Release.Name }} +app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{/* +ServiceAccount name. +*/}} +{{- define "flyte-core.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "flyte-core.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Component fullname — call with (dict "ctx" $ "component" "runs"). +*/}} +{{- define "flyte-core.componentFullname" -}} +{{- printf "%s-%s" (include "flyte-core.fullname" .ctx) .component | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Per-component ConfigMap name — call with (dict "ctx" $ "component" "runs"). +*/}} +{{- define "flyte-core.componentConfigMapName" -}} +{{- printf "%s-%s-config" (include "flyte-core.fullname" .ctx) .component | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Database Secret name. +*/}} +{{- define "flyte-core.dbSecretName" -}} +{{- printf "%s-db-password" (include "flyte-core.fullname" .) }} +{{- end }} + +{{/* +Webhook service name. +*/}} +{{- define "flyte-core.webhook.serviceName" -}} +{{- printf "%s-webhook" (include "flyte-core.fullname" .) }} +{{- end }} + +{{/* +ClusterRole name. +*/}} +{{- define "flyte-core.clusterRoleName" -}} +{{- printf "%s-cluster-role" (include "flyte-core.fullname" .) }} +{{- end }} + +{{/* +Storage stow config block — reused across component configmaps. +*/}} +{{- define "flyte-core.storageConfig" -}} +storage: + type: stow + stow: + {{- if eq .Values.storage.provider "s3" }} + {{- with .Values.storage.providerConfig.s3 }} + kind: s3 + config: + region: {{ .region }} + disable_ssl: {{ .disableSSL }} + v2_signing: {{ .v2Signing }} + {{- if .endpoint }} + endpoint: {{ .endpoint }} + {{- end }} + auth_type: {{ .authType }} + {{- end }} + {{- else if eq .Values.storage.provider "gcs" }} + kind: google + config: + json: "" + project_id: {{ .Values.storage.providerConfig.gcs.project }} + scopes: https://www.googleapis.com/auth/cloud-platform + {{- else if eq .Values.storage.provider "azure" }} + {{- with .Values.storage.providerConfig.azure }} + kind: azure + config: + account: {{ .account }} + {{- if .key }} + key: {{ .key }} + {{- end }} + {{- end }} + {{- end }} + container: {{ .Values.storage.metadataContainer }} +{{- end }} + +{{/* +Database config block — reused by runs and cache_service. +*/}} +{{- define "flyte-core.databaseConfig" -}} +database: + postgres: + username: {{ .Values.database.username }} + host: {{ tpl .Values.database.host . }} + port: {{ .Values.database.port }} + dbname: {{ .Values.database.dbname }} + options: {{ .Values.database.options | quote }} + {{- if .Values.database.password }} + passwordPath: /etc/flyte/db-password/password + {{- else if .Values.database.passwordPath }} + passwordPath: {{ .Values.database.passwordPath }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/actions/configmap.yaml b/charts/flyte-core/templates/actions/configmap.yaml new file mode 100644 index 0000000000..14c51e0cb7 --- /dev/null +++ b/charts/flyte-core/templates/actions/configmap.yaml @@ -0,0 +1,27 @@ +{{- if .Values.actions.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "actions") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "actions") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 010-actions.yaml: | + actions: + server: + host: "0.0.0.0" + port: {{ .Values.actions.port }} + kubernetes: + namespace: {{ .Release.Namespace }} + watchBufferSize: {{ .Values.actions.config.watchBufferSize }} + watchWorkers: {{ .Values.actions.config.watchWorkers }} + runServiceUrl: "http://{{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "runs") }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.runs.port }}" + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/actions/deployment.yaml b/charts/flyte-core/templates/actions/deployment.yaml new file mode 100644 index 0000000000..799f62e0af --- /dev/null +++ b/charts/flyte-core/templates/actions/deployment.yaml @@ -0,0 +1,78 @@ +{{- if .Values.actions.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "actions") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "actions") | nindent 4 }} +spec: + replicas: {{ .Values.actions.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "actions") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/actions/configmap.yaml") . | sha256sum }} + {{- with .Values.actions.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "actions") | nindent 8 }} + {{- with .Values.actions.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.actions.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.actions.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.actions.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: actions + image: "{{ .Values.image.repository }}/actions:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-actions"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.actions.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.actions.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- with .Values.actions.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "actions") }} +{{- end }} diff --git a/charts/flyte-core/templates/actions/service.yaml b/charts/flyte-core/templates/actions/service.yaml new file mode 100644 index 0000000000..e3d2bf914b --- /dev/null +++ b/charts/flyte-core/templates/actions/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.actions.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "actions") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "actions") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.actions.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "actions") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/cache_service/configmap.yaml b/charts/flyte-core/templates/cache_service/configmap.yaml new file mode 100644 index 0000000000..b08ed860d7 --- /dev/null +++ b/charts/flyte-core/templates/cache_service/configmap.yaml @@ -0,0 +1,26 @@ +{{- if .Values.cacheService.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "cache-service") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "cache-service") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 001-storage.yaml: | + {{- include "flyte-core.storageConfig" . | nindent 4 }} + 002-database.yaml: | + {{- include "flyte-core.databaseConfig" . | nindent 4 }} + 010-cache-service.yaml: | + cache_service: + server: + host: "0.0.0.0" + port: {{ .Values.cacheService.port }} + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/cache_service/deployment.yaml b/charts/flyte-core/templates/cache_service/deployment.yaml new file mode 100644 index 0000000000..41a23959dc --- /dev/null +++ b/charts/flyte-core/templates/cache_service/deployment.yaml @@ -0,0 +1,88 @@ +{{- if .Values.cacheService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "cache-service") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "cache-service") | nindent 4 }} +spec: + replicas: {{ .Values.cacheService.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "cache-service") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/cache_service/configmap.yaml") . | sha256sum }} + {{- with .Values.cacheService.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "cache-service") | nindent 8 }} + {{- with .Values.cacheService.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.cacheService.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cacheService.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cacheService.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: cache-service + image: "{{ .Values.image.repository }}/cache_service:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-cache-service"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.cacheService.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.cacheService.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- if .Values.database.password }} + - name: db-password + mountPath: /etc/flyte/db-password + readOnly: true + {{- end }} + {{- with .Values.cacheService.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "cache-service") }} + {{- if .Values.database.password }} + - name: db-password + secret: + secretName: {{ include "flyte-core.dbSecretName" . }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/cache_service/service.yaml b/charts/flyte-core/templates/cache_service/service.yaml new file mode 100644 index 0000000000..364998c914 --- /dev/null +++ b/charts/flyte-core/templates/cache_service/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.cacheService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "cache-service") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "cache-service") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.cacheService.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "cache-service") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/clusterrole.yaml b/charts/flyte-core/templates/clusterrole.yaml new file mode 100644 index 0000000000..1f960613e1 --- /dev/null +++ b/charts/flyte-core/templates/clusterrole.yaml @@ -0,0 +1,37 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "flyte-core.clusterRoleName" . }} + labels: {{- include "flyte-core.labels" . | nindent 4 }} + {{- with .Values.rbac.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.rbac.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Executor: manage pods + - apiGroups: [""] + resources: [pods] + verbs: [create, delete, get, list, patch, update, watch] + # Executor: create events + - apiGroups: [""] + resources: [events] + verbs: [create, delete, patch, update] + # Executor: manage TaskAction CRDs + - apiGroups: [flyte.org] + resources: [taskactions] + verbs: [create, delete, get, list, patch, update, watch] + # Executor: manage webhook configurations + - apiGroups: [admissionregistration.k8s.io] + resources: [mutatingwebhookconfigurations] + verbs: [create, get, list, patch, update, watch] + # Secret service: manage secrets + - apiGroups: [""] + resources: [secrets] + verbs: [create, get, update] + {{- with .Values.rbac.extraRules }} + {{- toYaml . | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/clusterrolebinding.yaml b/charts/flyte-core/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..40955b0549 --- /dev/null +++ b/charts/flyte-core/templates/clusterrolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "flyte-core.fullname" . }}-cluster-role-binding + labels: {{- include "flyte-core.labels" . | nindent 4 }} + {{- with .Values.rbac.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.rbac.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "flyte-core.clusterRoleName" . }} +subjects: + - kind: ServiceAccount + name: {{ include "flyte-core.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/flyte-core/templates/crds/flyte.org_taskactions.yaml b/charts/flyte-core/templates/crds/flyte.org_taskactions.yaml new file mode 100644 index 0000000000..969d28e00a --- /dev/null +++ b/charts/flyte-core/templates/crds/flyte.org_taskactions.yaml @@ -0,0 +1,287 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: taskactions.flyte.org +spec: + group: flyte.org + names: + kind: TaskAction + listKind: TaskActionList + plural: taskactions + singular: taskaction + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.runName + name: Run + type: string + - jsonPath: .spec.actionName + name: Action + type: string + - jsonPath: .spec.taskType + name: TaskType + type: string + - jsonPath: .status.conditions[?(@.type=='Progressing')].reason + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=='Progressing')].status + name: Progressing + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=='Succeeded')].status + name: Succeeded + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=='Failed')].status + name: Failed + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: TaskAction is the Schema for the taskactions API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of TaskAction + properties: + actionName: + description: ActionName is the unique name of this action within the + run + maxLength: 30 + minLength: 1 + type: string + cacheKey: + description: |- + CacheKey enables cache lookup/writeback for this task action when set. + This is propagated from workflow.TaskAction.cache_key. + maxLength: 256 + type: string + domain: + description: Domain this action belongs to + maxLength: 63 + minLength: 1 + type: string + envVars: + additionalProperties: + type: string + description: EnvVars are run-scoped environment variables projected + from RunSpec for executor runtime use. + type: object + group: + description: Group is the group this action belongs to, if applicable. + maxLength: 256 + type: string + inputUri: + description: InputURI is the path to the input data for this action + minLength: 1 + type: string + interruptible: + description: Interruptible is the run-scoped interruptibility override + projected from RunSpec. + type: boolean + parentActionName: + description: ParentActionName is the optional name of the parent action + maxLength: 30 + minLength: 1 + type: string + project: + description: Project this action belongs to + maxLength: 63 + minLength: 1 + type: string + runName: + description: RunName is the name of the run this action belongs to + maxLength: 30 + minLength: 1 + type: string + runOutputBase: + description: RunOutputBase is the base path where this action should + write its output + minLength: 1 + type: string + shortName: + description: ShortName is the human-readable display name for this + task + maxLength: 63 + type: string + taskTemplate: + description: TaskTemplate is the proto-serialized core.TaskTemplate + stored inline in etcd + format: byte + type: string + taskType: + description: TaskType identifies which plugin handles this task (e.g. + "container", "spark", "ray") + maxLength: 63 + minLength: 1 + type: string + required: + - actionName + - domain + - inputUri + - project + - runName + - runOutputBase + - taskTemplate + - taskType + type: object + status: + description: status defines the observed state of TaskAction + properties: + attempts: + description: Attempts is the latest observed action attempt number, + starting from 1. + format: int32 + type: integer + cacheStatus: + description: CacheStatus is the latest observed cache lookup result + for this action. + format: int32 + type: integer + conditions: + description: |- + conditions represent the current state of the TaskAction resource. + Each condition has a unique type and reflects the status of a specific aspect of the resource. + + Standard condition types include: + - "Available": the resource is fully functional + - "Progressing": the resource is being created or updated + - "Degraded": the resource failed to reach or maintain its desired state + + The status of each condition is one of True, False, or Unknown. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + phaseHistory: + description: |- + PhaseHistory is an append-only log of phase transitions. Unlike conditions + (which are updated in-place by type), this preserves the full timeline: + Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp. + items: + description: PhaseTransition records a phase change with its timestamp. + properties: + message: + description: Message is an optional human-readable message about + the transition. + type: string + occurredAt: + description: OccurredAt is when this phase transition happened. + format: date-time + type: string + phase: + description: Phase is the phase that was entered (e.g. "Queued", + "Initializing", "Executing", "Succeeded", "Failed"). + type: string + required: + - occurredAt + - phase + type: object + type: array + pluginPhase: + description: PluginPhase is a human-readable representation of the + plugin's current phase. + type: string + pluginPhaseVersion: + description: PluginPhaseVersion is the version of the current plugin + phase. + format: int32 + type: integer + pluginState: + description: PluginState is the Gob-encoded plugin state from the + last reconciliation round. + format: byte + type: string + pluginStateVersion: + description: PluginStateVersion tracks the version of the plugin state + schema for compatibility. + type: integer + stateJson: + description: StateJSON is the JSON serialized NodeStatus that was + last sent to the State Service + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/flyte-core/templates/dataproxy/configmap.yaml b/charts/flyte-core/templates/dataproxy/configmap.yaml new file mode 100644 index 0000000000..8c4516aa51 --- /dev/null +++ b/charts/flyte-core/templates/dataproxy/configmap.yaml @@ -0,0 +1,30 @@ +{{- if .Values.dataproxy.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "dataproxy") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "dataproxy") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 001-storage.yaml: | + {{- include "flyte-core.storageConfig" . | nindent 4 }} + 010-dataproxy.yaml: | + dataproxy: + {{- with .Values.dataproxy.config }} + upload: + maxSize: {{ .upload.maxSize | quote }} + maxExpiresIn: {{ .upload.maxExpiresIn }} + defaultFileNameLength: {{ .upload.defaultFileNameLength }} + storagePrefix: {{ .upload.storagePrefix | quote }} + download: + maxExpiresIn: {{ .download.maxExpiresIn }} + {{- end }} + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/dataproxy/deployment.yaml b/charts/flyte-core/templates/dataproxy/deployment.yaml new file mode 100644 index 0000000000..c4ac319040 --- /dev/null +++ b/charts/flyte-core/templates/dataproxy/deployment.yaml @@ -0,0 +1,78 @@ +{{- if .Values.dataproxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "dataproxy") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "dataproxy") | nindent 4 }} +spec: + replicas: {{ .Values.dataproxy.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "dataproxy") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/dataproxy/configmap.yaml") . | sha256sum }} + {{- with .Values.dataproxy.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "dataproxy") | nindent 8 }} + {{- with .Values.dataproxy.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.dataproxy.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dataproxy.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dataproxy.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: dataproxy + image: "{{ .Values.image.repository }}/dataproxy:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-dataproxy"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.dataproxy.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.dataproxy.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- with .Values.dataproxy.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "dataproxy") }} +{{- end }} diff --git a/charts/flyte-core/templates/dataproxy/service.yaml b/charts/flyte-core/templates/dataproxy/service.yaml new file mode 100644 index 0000000000..a8096eccb2 --- /dev/null +++ b/charts/flyte-core/templates/dataproxy/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.dataproxy.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "dataproxy") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "dataproxy") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.dataproxy.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "dataproxy") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/db-secret.yaml b/charts/flyte-core/templates/db-secret.yaml new file mode 100644 index 0000000000..c45bd28822 --- /dev/null +++ b/charts/flyte-core/templates/db-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.database.password }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "flyte-core.dbSecretName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.labels" . | nindent 4 }} +type: Opaque +data: + password: {{ .Values.database.password | b64enc | quote }} +{{- end }} diff --git a/charts/flyte-core/templates/events/configmap.yaml b/charts/flyte-core/templates/events/configmap.yaml new file mode 100644 index 0000000000..622e5b07f7 --- /dev/null +++ b/charts/flyte-core/templates/events/configmap.yaml @@ -0,0 +1,23 @@ +{{- if .Values.events.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "events") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "events") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 010-events.yaml: | + events: + server: + host: "0.0.0.0" + port: {{ .Values.events.port }} + runServiceUrl: "http://{{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "runs") }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.runs.port }}" + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/events/deployment.yaml b/charts/flyte-core/templates/events/deployment.yaml new file mode 100644 index 0000000000..9a609328f7 --- /dev/null +++ b/charts/flyte-core/templates/events/deployment.yaml @@ -0,0 +1,78 @@ +{{- if .Values.events.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "events") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "events") | nindent 4 }} +spec: + replicas: {{ .Values.events.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "events") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/events/configmap.yaml") . | sha256sum }} + {{- with .Values.events.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "events") | nindent 8 }} + {{- with .Values.events.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.events.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.events.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.events.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: events + image: "{{ .Values.image.repository }}/events:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-events"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.events.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.events.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- with .Values.events.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "events") }} +{{- end }} diff --git a/charts/flyte-core/templates/events/service.yaml b/charts/flyte-core/templates/events/service.yaml new file mode 100644 index 0000000000..eeaf668ac3 --- /dev/null +++ b/charts/flyte-core/templates/events/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.events.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "events") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "events") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.events.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "events") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/executor/configmap.yaml b/charts/flyte-core/templates/executor/configmap.yaml new file mode 100644 index 0000000000..6507936242 --- /dev/null +++ b/charts/flyte-core/templates/executor/configmap.yaml @@ -0,0 +1,33 @@ +{{- if .Values.executor.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "executor") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "executor") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 001-storage.yaml: | + {{- include "flyte-core.storageConfig" . | nindent 4 }} + 010-executor.yaml: | + executor: + healthProbeBindAddress: ":{{ .Values.executor.healthProbePort }}" + leaderElect: {{ .Values.executor.config.leaderElect }} + eventsServiceUrl: "http://{{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "events") }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.events.port }}" + cacheServiceUrl: "http://{{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "cache-service") }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.cacheService.port }}" + gc: + interval: {{ .Values.executor.config.gc.interval }} + maxTTL: {{ .Values.executor.config.gc.maxTTL }} + webhook: + certDir: {{ .Values.webhook.certDir }} + localCert: {{ .Values.webhook.localCert }} + serviceName: {{ include "flyte-core.webhook.serviceName" . }} + servicePort: {{ .Values.webhook.servicePort }} + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/executor/deployment.yaml b/charts/flyte-core/templates/executor/deployment.yaml new file mode 100644 index 0000000000..ee05196956 --- /dev/null +++ b/charts/flyte-core/templates/executor/deployment.yaml @@ -0,0 +1,86 @@ +{{- if .Values.executor.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "executor") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "executor") | nindent 4 }} +spec: + replicas: {{ .Values.executor.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "executor") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/executor/configmap.yaml") . | sha256sum }} + {{- with .Values.executor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "executor") | nindent 8 }} + {{- with .Values.executor.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.executor.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.executor.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.executor.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: executor + image: "{{ .Values.image.repository }}/executor:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-executor"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: health + containerPort: {{ .Values.executor.healthProbePort }} + protocol: TCP + - name: webhook + containerPort: 9443 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.executor.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + - name: webhook-certs + mountPath: {{ .Values.webhook.certDir }} + readOnly: false + {{- with .Values.executor.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "executor") }} + - name: webhook-certs + emptyDir: {} +{{- end }} diff --git a/charts/flyte-core/templates/executor/service.yaml b/charts/flyte-core/templates/executor/service.yaml new file mode 100644 index 0000000000..ca268de2b5 --- /dev/null +++ b/charts/flyte-core/templates/executor/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.executor.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.webhook.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "executor") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: webhook + port: {{ .Values.webhook.servicePort }} + targetPort: webhook + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "executor") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/runs/configmap.yaml b/charts/flyte-core/templates/runs/configmap.yaml new file mode 100644 index 0000000000..90686fbd73 --- /dev/null +++ b/charts/flyte-core/templates/runs/configmap.yaml @@ -0,0 +1,34 @@ +{{- if .Values.runs.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "runs") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "runs") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 001-storage.yaml: | + {{- include "flyte-core.storageConfig" . | nindent 4 }} + 002-database.yaml: | + {{- include "flyte-core.databaseConfig" . | nindent 4 }} + 010-runs.yaml: | + runs: + server: + host: "0.0.0.0" + port: {{ .Values.runs.port }} + storagePrefix: {{ .Values.runs.config.storagePrefix | quote }} + actionsServiceUrl: "http://{{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "actions") }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.actions.port }}" + {{- with .Values.runs.config.seedProjects }} + seedProjects: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runs.config.domains }} + domains: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/runs/deployment.yaml b/charts/flyte-core/templates/runs/deployment.yaml new file mode 100644 index 0000000000..f70e72891d --- /dev/null +++ b/charts/flyte-core/templates/runs/deployment.yaml @@ -0,0 +1,88 @@ +{{- if .Values.runs.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "runs") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "runs") | nindent 4 }} +spec: + replicas: {{ .Values.runs.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "runs") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/runs/configmap.yaml") . | sha256sum }} + {{- with .Values.runs.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "runs") | nindent 8 }} + {{- with .Values.runs.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.runs.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runs.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runs.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: runs + image: "{{ .Values.image.repository }}/runs:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-runs"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.runs.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.runs.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- if .Values.database.password }} + - name: db-password + mountPath: /etc/flyte/db-password + readOnly: true + {{- end }} + {{- with .Values.runs.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "runs") }} + {{- if .Values.database.password }} + - name: db-password + secret: + secretName: {{ include "flyte-core.dbSecretName" . }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/runs/service.yaml b/charts/flyte-core/templates/runs/service.yaml new file mode 100644 index 0000000000..0a6613f830 --- /dev/null +++ b/charts/flyte-core/templates/runs/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.runs.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "runs") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "runs") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.runs.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "runs") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/secret-service/configmap.yaml b/charts/flyte-core/templates/secret-service/configmap.yaml new file mode 100644 index 0000000000..c406f0d18d --- /dev/null +++ b/charts/flyte-core/templates/secret-service/configmap.yaml @@ -0,0 +1,24 @@ +{{- if .Values.secretService.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "secret") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "secret") | nindent 4 }} +data: + 000-core.yaml: | + logger: + show-source: true + level: {{ .Values.logging.level }} + 010-secret.yaml: | + secret: + server: + host: "0.0.0.0" + port: {{ .Values.secretService.port }} + kubernetes: + namespace: {{ .Release.Namespace }} + {{- with .Values.inlineConfig }} + 100-inline.yaml: | + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/flyte-core/templates/secret-service/deployment.yaml b/charts/flyte-core/templates/secret-service/deployment.yaml new file mode 100644 index 0000000000..ec2f207c53 --- /dev/null +++ b/charts/flyte-core/templates/secret-service/deployment.yaml @@ -0,0 +1,78 @@ +{{- if .Values.secretService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "secret") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "secret") | nindent 4 }} +spec: + replicas: {{ .Values.secretService.replicaCount }} + selector: + matchLabels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "secret") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/secret-service/configmap.yaml") . | sha256sum }} + {{- with .Values.secretService.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "secret") | nindent 8 }} + {{- with .Values.secretService.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "flyte-core.serviceAccountName" . }} + {{- with .Values.secretService.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.secretService.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.secretService.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: secret + image: "{{ .Values.image.repository }}/secret:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/flyte-secret"] + args: ["--config", "/etc/flyte/config.d/*.yaml"] + ports: + - name: http + containerPort: {{ .Values.secretService.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 30 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.secretService.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /etc/flyte/config.d + {{- with .Values.secretService.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ include "flyte-core.componentConfigMapName" (dict "ctx" $ "component" "secret") }} +{{- end }} diff --git a/charts/flyte-core/templates/secret-service/service.yaml b/charts/flyte-core/templates/secret-service/service.yaml new file mode 100644 index 0000000000..291bab285d --- /dev/null +++ b/charts/flyte-core/templates/secret-service/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.secretService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "flyte-core.componentFullname" (dict "ctx" $ "component" "secret") }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.componentLabels" (dict "ctx" $ "component" "secret") | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: {{ .Values.secretService.port }} + targetPort: http + protocol: TCP + selector: {{- include "flyte-core.selectorLabels" (dict "ctx" $ "component" "secret") | nindent 4 }} +{{- end }} diff --git a/charts/flyte-core/templates/serviceaccount.yaml b/charts/flyte-core/templates/serviceaccount.yaml new file mode 100644 index 0000000000..724ddb0639 --- /dev/null +++ b/charts/flyte-core/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "flyte-core.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "flyte-core.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- with .Values.serviceAccount.imagePullSecrets }} +imagePullSecrets: {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml new file mode 100644 index 0000000000..44524fc6bf --- /dev/null +++ b/charts/flyte-core/values.yaml @@ -0,0 +1,187 @@ +# -- Override chart name +nameOverride: "" +# -- Override fully qualified chart name +fullnameOverride: "" +# -- Common labels applied to all resources +commonLabels: {} +# -- Common annotations applied to all resources +commonAnnotations: {} + +# -- Image configuration shared by all components +image: + repository: ghcr.flyte.org/flyteorg/flyte-core + tag: nightly + pullPolicy: IfNotPresent + +# -- Database configuration (used by runs and cache_service) +database: + username: postgres + password: "" + passwordPath: "" + host: postgres + port: 5432 + dbname: flyte + options: "sslmode=disable" + +# -- Object storage configuration +storage: + metadataContainer: "flyte-data" + provider: s3 + providerConfig: + s3: + region: "us-east-1" + disableSSL: false + v2Signing: false + endpoint: "" + authType: "iam" + accessKey: "" + secretKey: "" + gcs: + project: "" + azure: + account: "" + key: "" + configDomainSuffix: "" + configUploadConcurrency: 4 + +# -- Logging configuration +logging: + level: 1 + +# -- Per-component configuration +# Each component becomes a separate Deployment. + +runs: + enabled: true + port: 8090 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + config: + storagePrefix: "s3://flyte-data" + seedProjects: + - flytesnacks + domains: + - id: development + name: Development + - id: production + name: Production + - id: staging + name: Staging + +actions: + enabled: true + port: 8091 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + config: + watchBufferSize: 100 + watchWorkers: 10 + +cacheService: + enabled: true + port: 8094 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + +dataproxy: + enabled: true + port: 8088 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + config: + upload: + maxSize: "100Mi" + maxExpiresIn: 1h + defaultFileNameLength: 20 + storagePrefix: "uploads" + download: + maxExpiresIn: 1h + +events: + enabled: true + port: 8092 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + +executor: + enabled: true + healthProbePort: 8081 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + config: + leaderElect: false + gc: + interval: 30m + maxTTL: 1h + +secretService: + enabled: true + port: 8093 + replicaCount: 1 + resources: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + tolerations: [] + affinity: {} + extraEnvVars: [] + +# -- Webhook configuration for the executor's mutating admission webhook +webhook: + certDir: /var/run/flyte/certs + localCert: true + servicePort: 443 + +# -- RBAC configuration +rbac: + create: true + labels: {} + annotations: {} + extraRules: [] + +# -- ServiceAccount configuration +serviceAccount: + create: true + name: "" + labels: {} + annotations: {} + imagePullSecrets: [] + +# -- Additional inline configuration merged into all component configs +inlineConfig: {}