diff --git a/.gitignore b/.gitignore index 78664ca..8a1b0d3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ charts/*/charts/ charts/mlflow/deployed-values .idea/ +# Ignore files produced by the deploy template makefile +**/make-deploy/*/* diff --git a/charts/mlflow/.helmignore b/charts/mlflow/.helmignore index 0e8a0eb..04df7dc 100644 --- a/charts/mlflow/.helmignore +++ b/charts/mlflow/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# Makefile and templates for creating a deployment +make-deploy/ diff --git a/charts/mlflow/make-deploy/LDAPGroupsMapper.json b/charts/mlflow/make-deploy/LDAPGroupsMapper.json new file mode 100644 index 0000000..e2f2697 --- /dev/null +++ b/charts/mlflow/make-deploy/LDAPGroupsMapper.json @@ -0,0 +1,28 @@ +{ + "id": "da07497d-8833-48ee-a189-fc6bbd85b950", + "name": "ldap_groups", + "description": "LDAP Groups", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "LDAP Groups" + }, + "protocolMappers": [ + { + "id": "b82b4c4e-feee-44bb-9ede-820beb5d4d18", + "name": "LDAP Group Membership", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "full.path": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "userinfo.token.claim": "true" + } + } + ] +} \ No newline at end of file diff --git a/charts/mlflow/make-deploy/Makefile b/charts/mlflow/make-deploy/Makefile new file mode 100644 index 0000000..9dcafa5 --- /dev/null +++ b/charts/mlflow/make-deploy/Makefile @@ -0,0 +1,42 @@ +.PHONY: values-template +values-template: $(name) + NAME=$(name) gomplate < gomplate-values.yaml > $(name)/gomplate-values.yaml + +clean: + rm -f $(name)/$(name)-postgres.secret.yaml + rm -f $(name)/$(name)-minio.secret.yaml + rm -f $(name)/$(name)-oauth2.secret.yaml + rm -f $(name)/$(name)-postgres.sealed.yaml + rm -f $(name)/$(name)-minio.sealed.yaml + rm -f $(name)/$(name)-oauth2.sealed.yaml + rm -f $(name)/$(name)-application.yaml + rm -f $(name)/values.yaml + rm -f $(name)/keycloak-client.json + +.PHONY: mlflow +mlflow: $(name) $(name)/$(name)-postgres.secret.yaml $(name)/$(name)-minio.secret.yaml $(name)/$(name)-oauth2.secret.yaml $(name)/keycloak-client.json $(name)/$(name)-application.yaml $(name)/values.yaml + +$(name): + mkdir $(name) + +$(name)/$(name)-application.yaml: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < application.yaml > $(name)/$(name)-application.yaml + +$(name)/values.yaml: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < values.yaml > $(name)/values.yaml + +$(name)/keycloak-client.json: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < keycloak-client.json > $(name)/keycloak-client.json + + +$(name)/$(name)-postgres.secret.yaml: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < postgres.secret.yaml > $(name)/$(name)-postgres.secret.yaml + kubeseal < $(name)/$(name)-postgres.secret.yaml > $(name)/$(name)-postgres.sealed.yaml + +$(name)/$(name)-minio.secret.yaml: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < minio.secret.yaml > $(name)/$(name)-minio.secret.yaml + kubeseal < $(name)/$(name)-minio.secret.yaml > $(name)/$(name)-minio.sealed.yaml + +$(name)/$(name)-oauth2.secret.yaml: $(name)/gomplate-values.yaml + gomplate -t _helpers.tpl -d Template=$(name)/gomplate-values.yaml < oauth2.secret.yaml > $(name)/$(name)-oauth2.secret.yaml + kubeseal < $(name)/$(name)-oauth2.secret.yaml > $(name)/$(name)-oauth2.sealed.yaml diff --git a/charts/mlflow/make-deploy/README.md b/charts/mlflow/make-deploy/README.md new file mode 100644 index 0000000..7949beb --- /dev/null +++ b/charts/mlflow/make-deploy/README.md @@ -0,0 +1,71 @@ +# Scripts to Make an MLFlow Deployment +Deploying MLFlow using the ArgoCD setup still requires several files and a bit +of carefully copied and pasted config. This directory holds a Makefile and a +set of Go templates to produce that from just a small handful of config settings. + +## Prerequisites +This Makefile assumes you have a go templating engine installed in your shell. +We are using [gomplate](https://docs.gomplate.ca/installing/). + +The Makefile uses the `kubeseal` command to encrypt the generated secrets. You +will need to have it installed, and your KubeConfig set to point to the +cluster you want to deploy to. + +## Makefile Argument: name +Each of the Makefile steps expect a single variable setting from the command +line: `name` - this value represents the name of the mlflow deployment and will +be used in a number of contexts: +1. The subdirectory where the files will be generated +2. Root name of the files that eventually will be copied to the `kubernetes` or `charts` directories in the Argo repository +3. Default application name and values group + +## Create Initial Settings +The first step in this process is to create the basic values used to drive the +entire templating process. +```shell +% make name=my-mlflow values-template +``` +This will create a directory named `my-mlflow` and populate a go template values +file there. This values file will have some defaults based on the name, and show +you all the other fields you can change for your specific deployment. + +## Gomplate-values.yaml +Here are the values you can configure for your deployment: + +| Property Name | Description | Default | +|-------------------------------------|-------------------------------------------------------------------------------------------------------------|-------------------------------------------| +| name | Name for this deployment. Will be used for the Keycloak client ID as well as naming the various K8s objects | The name provided with the `make` command | +| namespace | Kubernetes namespace where mlflow will be deployed | mlflow | +| MLFlow.artifacts.bucketName | Bucket name where the MLFlow artifacts will be persisted. Must follow bucket naming conventions | Name provided with make command | +| OAuth2.enabled | Enable the OAuth2-proxy sidecar? | true | +| OAuth2.secret | Name of a K8s holding the OAuth2 secrets. Leave blank to create public client | | +| OAuth2.allowedGroups | YAML List of Keycloak groups that will be allowed access | Some default NCSA LDAP groups | +| OAuth2.client_secret | A generated random secret that could be included in the oauth2-secret file if used | Random string | +| OAuth2.keycloak_realm_url | The URL to the Keycloak realm we will be authenticating to | The software-dev MLFlow realm | +| chartVersion | The MLFlow helm chart to use | 1.* | +| postgresql.persistence.storageClass | Storage class to be used for the postgres db | csi-cinder-sc-delete | +| minio.persistence.storageClass | Storage class for minio data | nfs-taiga | +| ingress.enabled | Enable an ingress to the MLFlow tracking server? | true | +| ingress.host | Hostname for the tracking server ingress | A suggested host on software-dev domain | + +Looks the defaulted values over and make changes as needed. + +## Generate Deployment +Once you have the values you want, you can generate the deployment files with +```shell +% make name=my-mlflow mlflow +``` +This will produce the following files for you: +1. keycloak-client.json - A file you can import into Keycloak to setup the OIDC client +2. $(name)-application.yaml - the Helm application manifest to copy to your `charts/templates` directory in Argo repo +3. $(name)-minio.secret.yaml and $(name)-minio.sealed.yaml - randomly generated secret values for minio as well as the encrypted version of these +4. $(name)-postgres.secret.yaml and $(name)-postgres.sealed.yaml - randomly generated secret values for postgres as well as the encrypted version of these +5. $(name)-oauth2.secret.yaml and $(name)-oauth2.sealed.yaml - randomly generated secret values for oauth2-proxy as well as the encrypted version of these. Delete these if you are not using OAuth2 secrets +6. values.yaml - Fragment of values.yaml to be carefully pasted into the Argo repository values.yaml + + + + + + + diff --git a/charts/mlflow/make-deploy/_helpers.tpl b/charts/mlflow/make-deploy/_helpers.tpl new file mode 100644 index 0000000..7f24310 --- /dev/null +++ b/charts/mlflow/make-deploy/_helpers.tpl @@ -0,0 +1,23 @@ +{{/* +Make a go template safe version of the deployment name +*/}} +{{- define "application.values.name"}}{{ strings.ReplaceAll "-" " " (ds "Template").name | strings.CamelCase}} +{{- end }} + +{{/* +Emit a reference to this application's values +*/}} +{{- define "application.values.root"}}.Values.{{ template "application.values.name" }} +{{- end }} + +{{/* +Emit a reference to this application's values as expanded map +*/}} +{{- define "application.values"}}{{ "{{- toYaml " }}{{- template "application.values.root" . }}{{ ".values | nindent 8 }} " }} +{{- end }} + +{{/* +Emit the host name as a url +*/}} +{{- define "url"}}{{ "https://"}}{{ (ds "Template").ingress.host }} +{{- end }} diff --git a/charts/mlflow/make-deploy/application.yaml b/charts/mlflow/make-deploy/application.yaml new file mode 100644 index 0000000..e7a270c --- /dev/null +++ b/charts/mlflow/make-deploy/application.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ "{{ .Values.cluster }}-" }}{{ (ds "Template").name }} + labels: + cluster: {{ "{{ .Values.cluster }}" }} + app: {{ (ds "Template").name }} + namespace: argocd + annotations: + {{ "{{- toYaml .Values.notifications | nindent 4 }}" }} +spec: + project: {{ "{{ .Values.cluster }}" }} + destination: + name: {{ "{{ .Values.cluster }}" }} + namespace: {{ (ds "Template").namespace }} + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + source: + repoURL: https://opensource.ncsa.illinois.edu/charts + chart: mlflow + targetRevision: "{{ (ds "Template").chartVersion }}" + helm: + version: v3 + releaseName: {{ (ds "Template").name }} + values: | + {{ template "application.values" . }} diff --git a/charts/mlflow/make-deploy/gomplate-values.yaml b/charts/mlflow/make-deploy/gomplate-values.yaml new file mode 100644 index 0000000..141ed03 --- /dev/null +++ b/charts/mlflow/make-deploy/gomplate-values.yaml @@ -0,0 +1,36 @@ +# This name is used for the client-id as well as the helm deployment +name: {{ env.Getenv "NAME"}} + +namespace: mlflow + +MLFlow: + artifacts: + # Specify the bucket name for models and other artifacts + # Bucket names must be between 3 and 63 characters long. + # Bucket names can consist only of lowercase letters, numbers, dots (.), and hyphens (-). + # Bucket names must begin and end with a letter or number. + bucketName: {{ strings.ReplaceAll "-" " " (env.Getenv "NAME") | strings.CamelCase}} + + +OAuth2: + enabled: true + # Kubernetes secret holding the client auth secret. Leave blank to work with + # a public client. This is needed for the device flow to work + secret: + + # Generated secret key if you want to use a confidential client instead. + # Ignore this otherwise + client_secret: {{ random.String 24}} + keycloak_realm_url: "https://keycloak.software-dev.ncsa.illinois.edu/realms/mlflow" + +chartVersion: 1.* +postgresql: + persistence: + storageClass: csi-cinder-sc-delete +minio: + persistence: + storageClass: nfs-taiga + +ingress: + enabled: true + host: "{{ env.Getenv "NAME"}}.software-dev.ncsa.illinois.edu" diff --git a/charts/mlflow/make-deploy/keycloak-client.json b/charts/mlflow/make-deploy/keycloak-client.json new file mode 100644 index 0000000..ab64709 --- /dev/null +++ b/charts/mlflow/make-deploy/keycloak-client.json @@ -0,0 +1,97 @@ +{ + "clientId": "{{ (ds "Template").name }}", + "name": "{{ (ds "Template").name }}", + "description": "", + "rootUrl": "{{ template "url" . }}", + "adminUrl": "", + "baseUrl": "{{ template "url" . }}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "{{ template "url" . }}/oauth2/callback" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "client.secret.creation.time": "1676592784", + "access.token.lifespan": 172800, + "oauth2.device.authorization.grant.enabled": "true", + "use.jwks.url": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "use.refresh.tokens": "true", + "tls-client-certificate-bound-access-tokens": "false", + "oidc.ciba.grant.enabled": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "acr.loa.map": "{}", + "require.pushed.authorization.requests": "false", + "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "name": "Audience for OAut2", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "false", + "access.token.claim": "true", + "included.custom.audience": "{{ (ds "Template").name }}" + } + }, + { + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "profile", + "roles", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ], + "access": { + "view": true, + "configure": true, + "manage": true + } +} diff --git a/charts/mlflow/make-deploy/minio.secret.yaml b/charts/mlflow/make-deploy/minio.secret.yaml new file mode 100644 index 0000000..6ac2a78 --- /dev/null +++ b/charts/mlflow/make-deploy/minio.secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ (ds "Template").name }}-minio-auth +stringData: + root-password: {{ random.String 24}} + root-user: {{ random.String 8}} + + diff --git a/charts/mlflow/make-deploy/oauth2.secret.yaml b/charts/mlflow/make-deploy/oauth2.secret.yaml new file mode 100644 index 0000000..6c5e67e --- /dev/null +++ b/charts/mlflow/make-deploy/oauth2.secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ (ds "Template").name }}-oauth2-secrets +stringData: + client_secret: "{{ (ds "Template").OAuth2.client_secret }}" + diff --git a/charts/mlflow/make-deploy/postgres.secret.yaml b/charts/mlflow/make-deploy/postgres.secret.yaml new file mode 100644 index 0000000..ed6a00d --- /dev/null +++ b/charts/mlflow/make-deploy/postgres.secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ (ds "Template").name }}-postgresql-auth +stringData: + password: {{ random.String 24}} + postgres-password: {{ random.String 24}} diff --git a/charts/mlflow/make-deploy/realm-template.json b/charts/mlflow/make-deploy/realm-template.json new file mode 100644 index 0000000..3eb25ef --- /dev/null +++ b/charts/mlflow/make-deploy/realm-template.json @@ -0,0 +1,165 @@ +{ + "realm": "mlflow", + "enabled": true, + "displayNameHtml": "

Login to MLFlow

", + "clients": [ + ], + "components": { + "org.keycloak.storage.UserStorageProvider": [ + { + "name": "NCSA LDAP", + "providerId": "ldap", + "subComponents": { + "org.keycloak.storage.ldap.mappers.LDAPStorageMapper": [ + { + "name": "email", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "mail" + ], + "user.model.attribute": [ + "email" + ] + } + }, + { + "name": "last name", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "sn" + ], + "user.model.attribute": [ + "lastName" + ] + } + }, + { + "name": "username", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "uid" + ], + "user.model.attribute": [ + "username" + ] + } + }, + { + "name": "first name", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "givenName" + ], + "user.model.attribute": [ + "firstName" + ] + } + }, + { + "name": "uidNumber", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "uidNumber" + ], + "user.model.attribute": [ + "uidNumber" + ] + } + }, + { + "name": "gidNumber", + "providerId": "user-attribute-ldap-mapper", + "config": { + "ldap.attribute": [ + "gidNumber" + ], + "user.model.attribute": [ + "gidNumber" + ] + } + }, + { + "name": "groups", + "providerId": "group-ldap-mapper", + "config": { + "membership.attribute.type": [ + "DN" + ], + "group.name.ldap.attribute": [ + "cn" + ], + "membership.user.ldap.attribute": [ + "uid" + ], + "groups.dn": [ + "ou=Groups,dc=ncsa,dc=illinois,dc=edu" + ], + "mode": [ + "READ_ONLY" + ], + "user.roles.retrieve.strategy": [ + "GET_GROUPS_FROM_USER_MEMBEROF_ATTRIBUTE" + ], + "membership.ldap.attribute": [ + "uniqueMember" + ], + "mapped.group.attributes": [ + "gidNumber" + ], + "group.object.classes": [ + "groupofuniquenames" + ] + } + } + ] + }, + "config": { + "customUserSearchFilter": [ + "(&(objectClass=inetorgperson)(|(memberOf=cn=org_software,ou=Groups,dc=ncsa,dc=illinois,dc=edu)(memberOf=cn=sd_class_admin,ou=Groups,dc=ncsa,dc=illinois,dc=edu)))" + ], + "fullSyncPeriod": [ + "-1" + ], + "usersDn": [ + "ou=People,dc=ncsa,dc=illinois,dc=edu" + ], + "trustEmail": [ + "true" + ], + "enabled": [ + "true" + ], + "changedSyncPeriod": [ + "-1" + ], + "usernameLDAPAttribute": [ + "uid" + ], + "rdnLDAPAttribute": [ + "uid" + ], + "vendor": [ + "other" + ], + "editMode": [ + "READ_ONLY" + ], + "uuidLDAPAttribute": [ + "uid" + ], + "connectionUrl": [ + "ldaps://ldap1.ncsa.illinois.edu:636 ldaps://ldap2.ncsa.illinois.edu:636 ldaps://ldap3.ncsa.illinois.edu:636 ldaps://ldap4.ncsa.illinois.edu:636" + ], + "authType": [ + "none" + ] + } + } + ] + } +} diff --git a/charts/mlflow/make-deploy/values.yaml b/charts/mlflow/make-deploy/values.yaml new file mode 100644 index 0000000..712fc36 --- /dev/null +++ b/charts/mlflow/make-deploy/values.yaml @@ -0,0 +1,47 @@ +# {{ (ds "Template").name }} mlflow application +{{ template "application.values.name" . }}: + version: {{ (ds "Template").chartVersion }} + values: + MLFlow: + artifacts: + bucketName: {{ (ds "Template").MLFlow.artifacts.bucketName }} + + oauth2Proxy: + secret: {{ (ds "Template").OAuth2.secret }} + enabled: {{ (ds "Template").OAuth2.enabled }} + allowedGroups: +{{ (ds "Template").OAuth2.allowedGroups | toYAML | strings.Indent 8}} + + oidc: + oidc_email_claim: "email" + oidc_issuer_url: "{{ (ds "Template").OAuth2.keycloak_realm_url }}" + oidc_groups_claim: "groups" + + provider: keycloak-oidc + clientID: {{ (ds "Template").name }} + + postgresql: + auth: + existingSecret: {{ (ds "Template").name }}-postgresql-auth + persistence: + storageClass: {{ (ds "Template").postgresql.persistence.storageClass }} + minio: + ingress: + enabled: false + apiIngress: + enabled: false + auth: + existingSecret: {{ (ds "Template").name }}-minio-auth + persistence: + storageClass: {{ (ds "Template").minio.persistence.storageClass }} + ingress: + enabled: {{ (ds "Template").ingress.enabled }} + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: {{ (ds "Template").ingress.host }} + paths: + - path: / + pathType: ImplementationSpecific + ingress: