Add makefile to generat a full mlflow app deployment

.gitignore

@@ -5,3 +5,5 @@ charts/*/charts/
 charts/mlflow/deployed-values
 .idea/
 
+# Ignore files produced by the deploy template makefile
+**/make-deploy/*/*

charts/mlflow/.helmignore

@@ -21,3 +21,5 @@
 .idea/
 *.tmproj
 .vscode/
+# Makefile and templates for creating a deployment
+make-deploy/

charts/mlflow/make-deploy/LDAPGroupsMapper.json

@@ -0,0 +1,28 @@
+{
+  "id": "da07497d-8833-48ee-a189-fc6bbd85b950",
+  "name": "ldap_groups",
+  "description": "LDAP Groups",
+  "protocol": "openid-connect",
+  "attributes": {
+    "include.in.token.scope": "true",
+    "display.on.consent.screen": "true",
+    "gui.order": "",
+    "consent.screen.text": "LDAP Groups"
+  },
+  "protocolMappers": [
+    {
+      "id": "b82b4c4e-feee-44bb-9ede-820beb5d4d18",
+      "name": "LDAP Group Membership",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-group-membership-mapper",
+      "consentRequired": false,
+      "config": {
+        "full.path": "true",
+        "id.token.claim": "true",
+        "access.token.claim": "true",
+        "claim.name": "groups",
+        "userinfo.token.claim": "true"
+      }
+    }
+  ]
+}

charts/mlflow/make-deploy/Makefile

@@ -0,0 +1,42 @@
+.PHONY: values-template
+values-template: $(name)
+	NAME=$(name) gomplate < gomplate-values.yaml > $(name)/gomplate-values.yaml
+
+clean:
+	rm -f $(name)/$(name)-postgres.secret.yaml
+	rm -f $(name)/$(name)-minio.secret.yaml
+	rm -f $(name)/$(name)-oauth2.secret.yaml
+	rm -f $(name)/$(name)-postgres.sealed.yaml
+	rm -f $(name)/$(name)-minio.sealed.yaml
+	rm -f $(name)/$(name)-oauth2.sealed.yaml
+	rm -f $(name)/$(name)-application.yaml
+	rm -f $(name)/values.yaml
+	rm -f $(name)/keycloak-client.json
+
+.PHONY: mlflow
+mlflow: $(name) $(name)/$(name)-postgres.secret.yaml $(name)/$(name)-minio.secret.yaml $(name)/$(name)-oauth2.secret.yaml $(name)/keycloak-client.json $(name)/$(name)-application.yaml $(name)/values.yaml
+
+$(name):
+	mkdir $(name)
+
+$(name)/$(name)-application.yaml: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < application.yaml > $(name)/$(name)-application.yaml
+
+$(name)/values.yaml: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < values.yaml > $(name)/values.yaml
+
+$(name)/keycloak-client.json: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < keycloak-client.json > $(name)/keycloak-client.json
+
+
+$(name)/$(name)-postgres.secret.yaml: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < postgres.secret.yaml > $(name)/$(name)-postgres.secret.yaml
+	kubeseal < $(name)/$(name)-postgres.secret.yaml > $(name)/$(name)-postgres.sealed.yaml
+
+$(name)/$(name)-minio.secret.yaml: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < minio.secret.yaml > $(name)/$(name)-minio.secret.yaml
+	kubeseal < $(name)/$(name)-minio.secret.yaml > $(name)/$(name)-minio.sealed.yaml
+
+$(name)/$(name)-oauth2.secret.yaml: $(name)/gomplate-values.yaml
+	gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < oauth2.secret.yaml > $(name)/$(name)-oauth2.secret.yaml
+	kubeseal < $(name)/$(name)-oauth2.secret.yaml > $(name)/$(name)-oauth2.sealed.yaml

charts/mlflow/make-deploy/README.md

@@ -0,0 +1,71 @@
+# Scripts to Make an MLFlow Deployment
+Deploying MLFlow using the ArgoCD setup still requires several files and a bit
+of carefully copied and pasted config. This directory holds a Makefile and a 
+set of Go templates to produce that from just a small handful of config settings.
+
+## Prerequisites
+This Makefile assumes you have a go templating engine installed in your shell.
+We are using [gomplate](https://docs.gomplate.ca/installing/).
+
+The Makefile uses the `kubeseal` command to encrypt the generated secrets. You
+will need to have it installed, and your KubeConfig set to point to the 
+cluster you want to deploy to.
+
+## Makefile Argument: name
+Each of the Makefile steps expect a single variable setting from the command
+line: `name` - this value represents the name of the mlflow deployment and will
+be used in a number of contexts:
+1. The subdirectory where the files will be generated
+2. Root name of the files that eventually will be copied to the `kubernetes` or `charts` directories in the Argo repository
+3. Default application name and values group
+
+## Create Initial Settings
+The first step in this process is to create the basic values used to drive the
+entire templating process.
+```shell
+% make name=my-mlflow values-template
+```
+This will create a directory named `my-mlflow` and populate a go template values
+file there. This values file will have some defaults based on the name, and show
+you all the other fields you can change for your specific deployment.
+
+## Gomplate-values.yaml
+Here are the values you can configure for your deployment:
+
+| Property Name                       | Description                                                                                                 | Default                                   |
+|-------------------------------------|-------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| name                                | Name for this deployment. Will be used for the Keycloak client ID as well as naming the various K8s objects | The name provided with the `make` command |
+| namespace                           | Kubernetes namespace where mlflow will be deployed                                                          | mlflow                                    |
+| MLFlow.artifacts.bucketName         | Bucket name where the MLFlow artifacts will be persisted. Must follow bucket naming conventions             | Name provided with make command           |
+| OAuth2.enabled                      | Enable the OAuth2-proxy sidecar?                                                                            | true                                      |
+| OAuth2.secret                       | Name of a K8s holding the OAuth2 secrets. Leave blank to create public client                               | <blank>                                   |
+| OAuth2.allowedGroups                | YAML List of Keycloak groups that will be allowed access                                                    | Some default NCSA LDAP groups             |
+| OAuth2.client_secret                | A generated random secret that could be included in the oauth2-secret file if used                          | Random string                             |
+| OAuth2.keycloak_realm_url           | The URL to the Keycloak realm we will be authenticating to                                                  | The software-dev MLFlow realm             |
+| chartVersion                        | The MLFlow helm chart to use                                                                                | 1.*                                       |
+| postgresql.persistence.storageClass | Storage class to be used for the postgres db                                                                | csi-cinder-sc-delete                      |
+| minio.persistence.storageClass      | Storage class for minio data                                                                                | nfs-taiga                                 |
+| ingress.enabled                     | Enable an ingress to the MLFlow tracking server?                                                            | true                                      |
+| ingress.host                        | Hostname for the tracking server ingress                                                                    | A suggested host on software-dev domain   |
+
+Looks the defaulted values over and make changes as needed.
+
+## Generate Deployment
+Once you have the values you want, you can generate the deployment files with
+```shell
+% make name=my-mlflow mlflow
+```
+This will produce the following files for you:
+1. keycloak-client.json - A file you can import into Keycloak to setup the OIDC client
+2. $(name)-application.yaml - the Helm application manifest to copy to your `charts/templates` directory in Argo repo
+3. $(name)-minio.secret.yaml and $(name)-minio.sealed.yaml - randomly generated secret values for minio as well as the encrypted version of these
+4. $(name)-postgres.secret.yaml and $(name)-postgres.sealed.yaml - randomly generated secret values for postgres as well as the encrypted version of these
+5. $(name)-oauth2.secret.yaml and $(name)-oauth2.sealed.yaml - randomly generated secret values for oauth2-proxy as well as the encrypted version of these. Delete these if you are not using OAuth2 secrets
+6. values.yaml - Fragment of values.yaml to be carefully pasted into the Argo repository values.yaml
+
+
+
+
+
+
+

charts/mlflow/make-deploy/_helpers.tpl

@@ -0,0 +1,23 @@
+{{/*
+Make a go template safe version of the deployment name
+*/}}
+{{- define "application.values.name"}}{{ strings.ReplaceAll "-" " " (ds "Template").name | strings.CamelCase}}
+{{- end }}
+
+{{/*
+Emit a reference to this application's values
+*/}}
+{{- define "application.values.root"}}.Values.{{ template "application.values.name" }}
+{{- end }}
+
+{{/*
+Emit a reference to this application's values as expanded map
+*/}}
+{{- define "application.values"}}{{ "{{- toYaml " }}{{- template "application.values.root" . }}{{ ".values | nindent 8 }} " }}
+{{- end }}
+
+{{/*
+Emit the host name as a url
+*/}}
+{{- define "url"}}{{ "https://"}}{{ (ds "Template").ingress.host }}
+{{- end }}

charts/mlflow/make-deploy/application.yaml

@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ "{{ .Values.cluster }}-" }}{{ (ds "Template").name }}
+  labels:
+    cluster: {{ "{{ .Values.cluster }}" }}
+    app: {{ (ds "Template").name }}
+  namespace: argocd
+  annotations:
+    {{ "{{- toYaml .Values.notifications | nindent 4 }}" }}
+spec:
+  project: {{ "{{ .Values.cluster }}" }}
+  destination:
+    name: {{ "{{ .Values.cluster }}" }}
+    namespace: {{ (ds "Template").namespace }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://opensource.ncsa.illinois.edu/charts
+    chart: mlflow
+    targetRevision: "{{ (ds "Template").chartVersion }}"
+    helm:
+      version: v3
+      releaseName: {{ (ds "Template").name }}
+      values: |
+        {{ template "application.values" . }}

charts/mlflow/make-deploy/gomplate-values.yaml

@@ -0,0 +1,36 @@
+# This name is used for the client-id as well as the helm deployment
+name: {{ env.Getenv "NAME"}}
+
+namespace: mlflow
+
+MLFlow:
+  artifacts:
+    # Specify the bucket name for models and other artifacts
+    #    Bucket names must be between 3 and 63 characters long.
+    #    Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-).
+    #    Bucket names must begin and end with a letter or number.
+    bucketName: {{ strings.ReplaceAll "-" " " (env.Getenv "NAME") | strings.CamelCase}}
+
+
+OAuth2:
+  enabled: true
+  # Kubernetes secret holding the client auth secret. Leave blank to work with
+  # a public client. This is needed for the device flow to work
+  secret:
+
+  # Generated secret key if you want to use a confidential client instead.
+  # Ignore this otherwise
+  client_secret: {{ random.String 24}}
+  keycloak_realm_url: "https://keycloak.software-dev.ncsa.illinois.edu/realms/mlflow"
+
+chartVersion: 1.*
+postgresql:
+  persistence:
+    storageClass: csi-cinder-sc-delete
+minio:
+  persistence:
+    storageClass: nfs-taiga
+
+ingress:
+  enabled: true
+  host: "{{ env.Getenv "NAME"}}.software-dev.ncsa.illinois.edu"

charts/mlflow/make-deploy/keycloak-client.json

@@ -0,0 +1,97 @@
+{
+  "clientId": "{{ (ds "Template").name }}",
+  "name": "{{ (ds "Template").name }}",
+  "description": "",
+  "rootUrl": "{{ template "url" . }}",
+  "adminUrl": "",
+  "baseUrl": "{{ template "url" . }}",
+  "surrogateAuthRequired": false,
+  "enabled": true,
+  "alwaysDisplayInConsole": false,
+  "clientAuthenticatorType": "client-secret",
+  "redirectUris": [
+    "{{ template "url" . }}/oauth2/callback"
+  ],
+  "webOrigins": [],
+  "notBefore": 0,
+  "bearerOnly": false,
+  "consentRequired": false,
+  "standardFlowEnabled": true,
+  "implicitFlowEnabled": false,
+  "directAccessGrantsEnabled": true,
+  "serviceAccountsEnabled": false,
+  "publicClient": true,
+  "frontchannelLogout": true,
+  "protocol": "openid-connect",
+  "attributes": {
+    "client.secret.creation.time": "1676592784",
+    "access.token.lifespan": 172800,
+    "oauth2.device.authorization.grant.enabled": "true",
+    "use.jwks.url": "false",
+    "backchannel.logout.revoke.offline.tokens": "false",
+    "use.refresh.tokens": "true",
+    "tls-client-certificate-bound-access-tokens": "false",
+    "oidc.ciba.grant.enabled": "false",
+    "backchannel.logout.session.required": "true",
+    "client_credentials.use_refresh_token": "false",
+    "acr.loa.map": "{}",
+    "require.pushed.authorization.requests": "false",
+    "display.on.consent.screen": "false",
+    "token.response.type.bearer.lower-case": "false"
+  },
+  "authenticationFlowBindingOverrides": {},
+  "fullScopeAllowed": true,
+  "nodeReRegistrationTimeout": -1,
+  "protocolMappers": [
+    {
+      "name": "Audience for OAut2",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-audience-mapper",
+      "consentRequired": false,
+      "config": {
+        "id.token.claim": "false",
+        "access.token.claim": "true",
+        "included.custom.audience": "{{ (ds "Template").name }}"
+      }
+    },
+    {
+      "name": "audience resolve",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-audience-resolve-mapper",
+      "consentRequired": false,
+      "config": {}
+    },
+    {
+      "name": "email",
+      "protocol": "openid-connect",
+      "protocolMapper": "oidc-usermodel-property-mapper",
+      "consentRequired": false,
+      "config": {
+        "userinfo.token.claim": "true",
+        "user.attribute": "email",
+        "id.token.claim": "true",
+        "access.token.claim": "true",
+        "claim.name": "email",
+        "jsonType.label": "String"
+      }
+    }
+  ],
+  "defaultClientScopes": [
+    "web-origins",
+    "acr",
+    "profile",
+    "roles",
+    "email"
+  ],
+  "optionalClientScopes": [
+    "address",
+    "phone",
+    "offline_access",
+    "microprofile-jwt"
+  ],
+  "access": {
+    "view": true,
+    "configure": true,
+    "manage": true
+  }
+}

charts/mlflow/make-deploy/minio.secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ (ds "Template").name }}-minio-auth
+stringData:
+    root-password: {{ random.String 24}}
+    root-user: {{ random.String 8}}
+
+

charts/mlflow/make-deploy/oauth2.secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ (ds "Template").name }}-oauth2-secrets
+stringData:
+  client_secret: "{{ (ds "Template").OAuth2.client_secret }}"
+

charts/mlflow/make-deploy/postgres.secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ (ds "Template").name }}-postgresql-auth
+stringData:
+    password: {{ random.String 24}}
+    postgres-password: {{ random.String 24}}