diff --git a/.env.example b/.env.example index 71bc19bf3..980c91fe5 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,14 @@ PHARIA_KERNEL_METRICS_ADDRESS=127.0.0.1:9000 # You can retrieve a Pharia AI Token from Pharia Studio PHARIA_AI_TOKEN= # User name may be anything, in case you are using a token as password -SKILL_REGISTRY_USER= -# Skill registry token needs to have read and write access. Some tests push stuff to teh registry -SKILL_REGISTRY_PASSWORD= -SKILL_REGISTRY=registry.gitlab.aleph-alpha.de -SKILL_REPOSITORY=engineering/pharia-skills/skills +NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER= +# Skill registry token needs to have read and write access. Some tests push stuff to the registry +NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD= +NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY=registry.gitlab.aleph-alpha.de +NAMESPACES__PHARIA_KERNEL_TEAM__BASE_REPOSITORY=engineering/pharia-skills/skills LOG_LEVEL=info # Best use your personal Gitlab token -GITLAB_CONFIG_ACCESS_TOKEN= +NAMESPACES__PHARIA_KERNEL_TEAM__CONFIG_ACCESS_TOKEN= OPEN_TELEMETRY_ENDPOINT=http://127.0.0.1:4317 AA_INFERENCE_ADDRESS=https://inference-api.product.pharia.com DOCUMENT_INDEX_ADDRESS=https://document-index.product.pharia.com diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2df2db51b..6efc53b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,10 +35,10 @@ jobs: - name: Run cargo test run: cargo test --workspace --all-features env: - SKILL_REGISTRY: ghcr.io - SKILL_REPOSITORY: aleph-alpha/pharia-kernel - SKILL_REGISTRY_USER: ${{ github.actor }} - SKILL_REGISTRY_PASSWORD: ${{ secrets.SKILL_REGISTRY_PASSWORD }} + NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY: ghcr.io + NAMESPACES__PHARIA_KERNEL_TEAM__BASE_REPOSITORY: aleph-alpha/pharia-kernel + NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER: ${{ github.actor }} + NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD: ${{ secrets.SKILL_REGISTRY_PASSWORD }} PHARIA_AI_TOKEN: ${{ secrets.PHARIA_AI_TOKEN }} AA_INFERENCE_ADDRESS: https://inference-api.product.pharia.com AUTHORIZATION_ADDRESS: https://pharia-iam.product.pharia.com @@ -101,9 +101,9 @@ jobs: - name: Integration test run: | (podman run -p 8081:8081 -v ./operator-config.toml:/app/operator-config.toml \ - -e SKILL_REGISTRY_USER=dummy \ - -e SKILL_REGISTRY_PASSWORD=dummy \ - -e GITLAB_CONFIG_ACCESS_TOKEN=dummy \ + -e NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER=dummy \ + -e NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD=dummy \ + -e NAMESPACES__PHARIA_KERNEL_TEAM__CONFIG_ACCESS_TOKEN=dummy \ -e PHARIA_AI_TOKEN=dummy \ pharia-kernel:$IMAGE_ID | cat) & bash -x ./tests/test-image.sh 8081 127.0.0.1 diff --git a/Cargo.lock b/Cargo.lock index ea68ae31d..e60acf2a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,6 +130,12 @@ dependencies = [ "syn", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.6" @@ -432,6 +438,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitstream-io" @@ -702,6 +711,25 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "config" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d84f8d224ac58107d53d3ec2b9ad39fd8c8c4e285d3c9cb35485ffd2ca88cb3" +dependencies = [ + "async-trait", + "convert_case", + "json5", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "winnow", + "yaml-rust2", +] + [[package]] name = "const-random" version = "0.1.18" @@ -722,6 +750,15 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1077,6 +1114,15 @@ dependencies = [ "syn", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "doc-metadata-skill" version = "0.1.0" @@ -1512,6 +1558,15 @@ dependencies = [ "serde", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "headers" version = "0.4.0" @@ -2105,6 +2160,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jwt" version = "0.16.0" @@ -2791,6 +2857,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "overload" version = "0.1.1" @@ -2803,12 +2879,63 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror 2.0.9", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pharia-kernel" version = "0.4.14" @@ -2818,9 +2945,11 @@ dependencies = [ "async-trait", "axum 0.8.1", "axum-extra", + "config", "dotenvy", "fake", "futures", + "heck 0.5.0", "http-body-util", "humantime", "itertools 0.14.0", @@ -3339,6 +3468,29 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.6.0", + "serde", + "serde_derive", +] + +[[package]] +name = "rust-ini" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" +dependencies = [ + "cfg-if", + "ordered-multimap", + "trim-in-place", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -4321,6 +4473,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trim-in-place" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" + [[package]] name = "try-lock" version = "0.2.5" @@ -4333,6 +4491,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicase" version = "2.8.1" @@ -5482,6 +5646,17 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "yaml-rust2" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a1a1c0bc9823338a3bdf8c61f994f23ac004c6fa32c08cd152984499b445e8d" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index e556a1e7e..7e568f4b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,6 +93,8 @@ metrics-exporter-prometheus = { version = "0.16.0", default-features = false, fe ] } itertools = "0.14.0" urlencoding = "2.1.3" +config = "0.15.4" +heck = "0.5.0" [dev-dependencies] test-skills = { path = "./test-skills" } diff --git a/doc/src/local-setup.md b/doc/src/local-setup.md index a15120273..0f062aebb 100644 --- a/doc/src/local-setup.md +++ b/doc/src/local-setup.md @@ -33,7 +33,7 @@ In order to run Pharia Kernel, you need to provide a namespace configuration: mkdir skills ``` - All skills in this folder are exposed in the namespace "dev". + All skills in this folder are exposed in the namespace "dev" with the environment variable `NAMESPACES__DEV__DIRECTORY`. Any changes in this folder will be picked up by the Pharia Kernel automatically. The `operator-config.toml` and `namespace.toml` should not be provided. 2. Start the container: @@ -41,6 +41,7 @@ In order to run Pharia Kernel, you need to provide a namespace configuration: ```shell podman run \ -v ./skills:/app/skills \ + -e NAMESPACES__DEV__DIRECTORY = "/app/skills" -e PHARIA_AI_TOKEN=$PHARIA_AI_TOKEN \ -e NAMESPACE_UPDATE_INTERVAL=1s \ -e LOG_LEVEL="pharia_kernel=debug" \ diff --git a/doc/src/operating-kernel.md b/doc/src/operating-kernel.md index f7f4089e2..77067bcee 100644 --- a/doc/src/operating-kernel.md +++ b/doc/src/operating-kernel.md @@ -26,48 +26,49 @@ Pharia Kernel wants to enable teams outside of the Pharia Kernel Operators to de Each namespace configuration typically would reside in a Git repository owned by the Team which owns the namespace. Changes in this file will be automatically detected by the Kernel. -A namespace is also associated with a registry to load the skills from. These skill registries can either be directories in filesystem (mostly used for a development setup) or point to an OCI registry (recommended for production). +A [namespace](skill-deployment.md#configuring-namespace) is also associated with a registry to load the skills from. These skill registries can either be directories in filesystem (mostly used for a development setup) or point to an OCI registry (recommended for production). ### Namespace with Local Config and Local Registry ```toml -[namespaces.local] -config_url = "file://namespace.toml" +# `my-team` is the name of this namespace +[namespaces.my-team] +# Path to the local namespace configuration file +config-url = "file://namespace.toml" -[namespaces.local.registry] -type = "file" +# Path to the local skill directory path = "skills" ``` -With the local configuration above, Pharia Kernel will serve any skill deployed at the `skills` subdirectory of its working directory under the namespace "local". This is mostly intended for local development of skills without a remote instance of Pharia Kernel. To deploy skills in production it is recommended to use a remote namespace. +With the local configuration above, Pharia Kernel will serve any skill deployed at the `skills` subdirectory of its working directory under the namespace `my-team`. This is mostly intended for local development of skills without a remote instance of Pharia Kernel. To deploy skills in production it is recommended to use a remote namespace. ### Namespace with Remote Config and Remote Registry ```toml +# `my-team` is the name of this namespace [namespaces.my-team] # The URL to the configuration listing the skills of this namespace -config_url = "https://gitlab.aleph-alpha.de/api/v4/projects/966/repository/files/config.toml/raw?ref=main" -# Pharia kernel will use the contents of this environment variable to access (authorize) the above URL -config_access_token_env_var = "GITLAB_CONFIG_ACCESS_TOKEN" - -# Registry to load skills from -[namespaces.my-team.registry] -type = "oci" -registry = "registry.gitlab.aleph-alpha.de" -repository = "engineering/pharia-skills/skills" -user_env_var = "MY_TEAM_REGISTRY_USER" -password_env_var = "MY_TEAM_REGISTRY_PASSWORD" +# Pharia kernel will use the contents of the `NAMESPACES__MY_TEAM__CONFIG_ACCESS_TOKEN` environment variable to access (authorize) the config +config-url = "https://github.com/Aleph-Alpha/my-team/blob/main/config.toml" + +# OCI Registry to load skills from +# Pharia kernel will use the contents of the `NAMESPACES__MY_TEAM__REGISTRY_USER` and `NAMESPACES__MY_TEAM__REGISTRY_PASSWORD` environment variables to access (authorize) the registry +registry = "registry.acme.com" + +# This is the common prefix added to the skill name when composing the OCI repository. +# e.g. ${base-repository}/${skill_name} +base-repository = "my-org/my-team/skills" ``` -With the remote configuration above, Pharia Kernel will serve any skill deployed on the specified OCI registry under the namespace "my-team". +With the remote configuration above, Pharia Kernel will serve any skill deployed on the specified OCI registry under the namespace `my-team`. ### Authentication against OCI Registries You can provide each namespace in Pharia Kernel with credentials to authenticate against the specified OCI registry. Set the environment variables that are expected from the operator config: ```shell -MY_TEAM_REGISTRY_USER=Joe.Plumber -MY_TEAM_REGISTRY_PASSWORD=**** +NAMESPACES__MY_TEAM__REGISTRY_USER=Joe.Plumber +NAMESPACES__MY_TEAM__REGISTRY_PASSWORD=**** ``` ## Update interval diff --git a/helm-charts/pharia-kernel/templates/deployment.yaml b/helm-charts/pharia-kernel/templates/deployment.yaml index 4d5024ecf..64deaf6ef 100644 --- a/helm-charts/pharia-kernel/templates/deployment.yaml +++ b/helm-charts/pharia-kernel/templates/deployment.yaml @@ -70,6 +70,10 @@ spec: env: - name: LOG_LEVEL value: {{ .Values.logLevel | quote }} + {{- if .Values.openTelemetryEndpoint }} + - name: OPEN_TELEMETRY_ENDPOINT + value: {{ .Values.openTelemetryEndpoint | quote }} + {{- end }} {{- if .Values.authorizationAddress }} - name: AUTHORIZATION_ADDRESS value: {{ .Values.authorizationAddress | quote }} @@ -100,9 +104,16 @@ spec: name: {{ .Values.global.phariaAIConfigMap }} key: DOCUMENT_INDEX_URL {{- end }} - {{- if .Values.openTelemetryEndpoint }} - - name: OPEN_TELEMETRY_ENDPOINT - value: {{ .Values.openTelemetryEndpoint | quote }} + {{- if and .Values.defaultNamespacesEnabled .Values.global.phariaAIConfigMap }} + - name: NAMESPACES__ASSISTANT__CONFIG_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.global.phariaAIConfigMap }} + key: PHARIA_ASSISTANT_API_SKILL_CONFIG + - name: NAMESPACES__ASSISTANT__REGISTRY + value: "alephalpha.jfrog.io" + - name: NAMESPACES__ASSISTANT__BASE_REPOSITORY + value: "assistant-skills" {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 12 }} diff --git a/helm-charts/pharia-kernel/templates/operator-config.yaml b/helm-charts/pharia-kernel/templates/operator-config.yaml index 5dcbc695d..e91607dd6 100644 --- a/helm-charts/pharia-kernel/templates/operator-config.yaml +++ b/helm-charts/pharia-kernel/templates/operator-config.yaml @@ -4,21 +4,26 @@ metadata: name: pharia-kernel-operator-config data: operator-config.toml: | - {{- $namespaces := .Values.defaultNamespacesEnabled | ternary (merge .Values.defaultNamespaces .Values.namespaces) .Values.namespaces -}} - {{- if $namespaces }} - {{- range $k, $v := $namespaces }} - [namespaces.{{ $k }}] - config_url = {{ $v.config_url | quote }} - {{- if $v.config_access_token_env_var }} - config_access_token_env_var = {{ $v.config_access_token_env_var | quote }} + {{- if .Values.namespaces }} + {{- range $k, $v := .Values.namespaces }} + [namespaces.{{ $k | kebabcase }}] + {{- if $v.configUrl }} + config-url = {{ $v.configUrl | quote }} {{- end }} - [namespaces.{{ $k }}.registry] - type = "oci" + {{- if $v.configAccessToken }} + config-access-token = {{ $v.configAccessToken | quote }} + {{- end }} + {{- if $v.registry }} registry = {{ $v.registry | quote }} - repository = {{ $v.repository | quote }} - user_env_var = {{ $v.user_env_var | quote }} - password_env_var = {{ $v.password_env_var | quote }} + {{- end }} + {{- if $v.baseRepository }} + base-repository = {{ $v.baseRepository | quote }} + {{- end }} + {{- if $v.registryUser }} + registry-user = {{ $v.registryUser | quote }} + {{- end }} + {{- if $v.registryPassword }} + registry-password = {{ $v.registryPassword | quote }} + {{- end }} {{ end }} - {{- else }} - [namespaces] {{- end }} diff --git a/helm-charts/pharia-kernel/values.yaml b/helm-charts/pharia-kernel/values.yaml index 3e977850d..433152ae9 100644 --- a/helm-charts/pharia-kernel/values.yaml +++ b/helm-charts/pharia-kernel/values.yaml @@ -46,27 +46,29 @@ service: metricsProtocol: TCP serviceMonitor: enabled: false -# -- If the default namespaces are enabled, the provided namespaces with existing keys are ignored +# -- If the default namespaces are enabled, the provided namespaces with existing keys are overwritten defaultNamespacesEnabled: false -defaultNamespaces: - assistant: - config_url: "http://pharia-assistant-api.pharia-ai.svc.cluster.local/skills/skill-config" - registry: alephalpha.jfrog.io - repository: assistant-skills - user_env_var: ALEPH_ALPHA_OCI_SKILL_REGISTRY_USER - password_env_var: ALEPH_ALPHA_OCI_SKILL_REGISTRY_PASSWORD # -- Active namespaces for Pharia Kernel. e.g.: # playground: -# config_url: "https://gitlab.aleph-alpha.de/api/v4/projects/997/repository/files/namespace.toml/raw?ref=main" -# config_access_token_env_var: "GITLAB_CONFIG_ACCESS_TOKEN" +# configUrl: "https://gitlab.aleph-alpha.de/api/v4/projects/997/repository/files/namespace.toml/raw?ref=main" +# configAccessToken: "GITLAB_CONFIG_ACCESS_TOKEN" # registry: "registry.gitlab.aleph-alpha.de" -# repository: "engineering/pharia-kernel-playground/skills" +# baseRepository: "engineering/pharia-kernel-playground/skills" +# registryUser: "REGISTRY_USER" +# registryPassword: "REGISTRY_PASSWORD" +# Each of the value can alternatively be provided as environment variables, which have higher precedence: +# NAMESPACES__PLAYGROUND__CONFIG_URL +# NAMESPACES__PLAYGROUND__CONFIG_ACCESS_TOKEN +# NAMESPACES__PLAYGROUND__REGISTRY +# NAMESPACES__PLAYGROUND__BASE_REPOSITORY +# NAMESPACES__PLAYGROUND__REGISTRY_USER +# NAMESPACES__PLAYGROUND__REGISTRY_PASSWORD namespaces: {} logLevel: info +openTelemetryEndpoint: "" authorizationAddress: "" inferenceAddress: "" documentIndexAddress: "" -openTelemetryEndpoint: "" ingress: enabled: true diff --git a/operator-config.toml b/operator-config.toml index f30c42ad0..f34580d79 100644 --- a/operator-config.toml +++ b/operator-config.toml @@ -1,14 +1,9 @@ [namespaces.pharia-kernel-team] # The URL to the configuration listing the skills of this namespace -config_url = "https://gitlab.aleph-alpha.de/api/v4/projects/966/repository/files/config.toml/raw?ref=main" - -# Pharia kernel will use the contents of this environment variable to access (authorize) the above URL -config_access_token_env_var = "GITLAB_CONFIG_ACCESS_TOKEN" +# Pharia kernel will use the contents of the `NAMESPACES__PHARIA_KERNEL_TEAM__CONFIG_ACCESS_TOKEN` environment variable to access (authorize) the config +config-url = "https://gitlab.aleph-alpha.de/api/v4/projects/966/repository/files/config.toml/raw?ref=main" # Registry to load skills from -[namespaces.pharia-kernel-team.registry] -type = "oci" +# Pharia kernel will use the contents of the `NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER` and `NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD` environment variables to access (authorize) the registry registry = "registry.gitlab.aleph-alpha.de" -repository = "engineering/pharia-skills/skills" -user_env_var = "SKILL_REGISTRY_USER" -password_env_var = "SKILL_REGISTRY_PASSWORD" +base-repository = "engineering/pharia-skills/skills" diff --git a/src/config.rs b/src/config.rs index 7a86ac5a5..b1856e7af 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use std::{env, io, net::SocketAddr, time::Duration}; +use std::{env, net::SocketAddr, time::Duration}; use crate::namespace_watcher::OperatorConfig; @@ -71,26 +71,8 @@ impl AppConfig { .unwrap_or("10s") .parse()?; - let operator_config = match OperatorConfig::from_file("operator-config.toml") { - Ok(operator_config) => operator_config, - Err(err) => match err.downcast::() { - Ok(ioerror) if ioerror.kind() == io::ErrorKind::NotFound => { - // println! as the logger is not yet instantiated - println!("Info: The 'operator-config.toml' is not found, fallback to the namespace 'dev' with the path 'skills' as the registry."); - OperatorConfig::dev() - } - Ok(err) => { - return Err(anyhow!( - "The provided operator configuration must be valid: {err}" - )) - } - Err(err) => { - return Err(anyhow!( - "The provided operator configuration must be valid: {err}" - )) - } - }, - }; + let operator_config = OperatorConfig::new("operator-config.toml") + .map_err(|err| anyhow!("The provided operator configuration must be valid: {err}"))?; let use_pooling_allocator = env::var("USE_POOLING_ALLOCATOR") .as_deref() diff --git a/src/namespace_watcher/actor.rs b/src/namespace_watcher/actor.rs index 2936440bb..de3cd02b9 100644 --- a/src/namespace_watcher/actor.rs +++ b/src/namespace_watcher/actor.rs @@ -34,9 +34,9 @@ impl NamespaceDescriptionLoaders { config .loader() .with_context(|| { - format!("Unable to load configuration of namespace: '{namespace}'") + format!("Unable to load configuration of namespace: '{namespace:?}'") }) - .map(|loader| (namespace, loader)) + .map(|loader| (namespace.into_string(), loader)) }) .collect::>>()?; Ok(Self { namespaces }) @@ -264,6 +264,7 @@ pub mod tests { use tokio::sync::{mpsc, Mutex}; use tokio::time::timeout; + use crate::namespace_watcher::config::Namespace; use crate::namespace_watcher::tests::NamespaceConfig; use crate::skill_store::tests::SkillStoreMessage; use crate::skills::SkillPath; @@ -407,7 +408,7 @@ pub mod tests { async fn watch_skills_in_empty_directory() { let temp_dir = tempdir().unwrap(); let namespaces = [( - "local".to_owned(), + Namespace::new("local"), NamespaceConfig::Watch { directory: temp_dir.path().to_owned(), }, @@ -430,7 +431,7 @@ pub mod tests { fs::File::create(directory.join("skill_1.wasm")).unwrap(); fs::File::create(directory.join("skill_2.wasm")).unwrap(); let namespaces = [( - "local".to_owned(), + Namespace::new("local"), NamespaceConfig::Watch { directory: directory.to_owned(), }, diff --git a/src/namespace_watcher/config.rs b/src/namespace_watcher/config.rs index 8d0550d08..647900326 100644 --- a/src/namespace_watcher/config.rs +++ b/src/namespace_watcher/config.rs @@ -1,11 +1,9 @@ -use std::{ - collections::HashMap, - env, fs, - path::{Path, PathBuf}, -}; +use std::{collections::HashMap, path::PathBuf}; -use anyhow::{anyhow, Context}; -use serde::Deserialize; +use anyhow::anyhow; +use config::{Case, Config, Environment, File, FileFormat, FileSourceFile}; +use heck::ToKebabCase; +use serde::{Deserialize, Deserializer}; use url::Url; use crate::skill_loader::RegistryConfig; @@ -17,23 +15,63 @@ use super::{ NamespaceDescriptionLoader, }; +#[derive(PartialEq, Eq, Hash, Debug, Clone)] +pub struct Namespace(String); + +impl Namespace { + pub fn new(input: impl Into) -> Self { + Self(input.into().to_kebab_case()) + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl<'de> Deserialize<'de> for Namespace { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(Namespace::new(s)) + } +} + #[derive(Deserialize, PartialEq, Eq, Debug, Clone)] pub struct OperatorConfig { - pub namespaces: HashMap, + #[serde(default)] + pub namespaces: HashMap, } impl OperatorConfig { /// # Errors - /// Cannot parse config or cannot read from file. - pub fn from_file>(p: P) -> anyhow::Result { - let config = fs::read_to_string(p)?; - Self::from_toml(&config) + /// Cannot parse operator config from the provided file or the environment variables. + pub fn new(config_file: &str) -> anyhow::Result { + let file = File::with_name(config_file).required(false); + let env = Self::environment(); + Self::from_sources(file, env) } - /// # Errors - /// Cannot parse config. - pub fn from_toml(config: &str) -> anyhow::Result { - Ok(toml::from_str(config)?) + /// A namespace can contain the characters `[a-z0-9-]` e.g. `pharia-kernel-team`. + /// + /// As only `SCREAMING_SNAKE_CASE` is widely supported for environment variable keys, + /// we support it by converting each key into `kebab-case`. + /// Because we have a nested configuration, we use double underscores as the separators. + fn environment() -> Environment { + Environment::with_convert_case(Case::Kebab).separator("__") + } + + fn from_sources( + file: File, + env: Environment, + ) -> anyhow::Result { + let config = Config::builder() + .add_source(file) + .add_source(env) + .build()? + .try_deserialize::()?; + Ok(config) } /// Create an operator config which checks the local `skills` directory for @@ -44,7 +82,7 @@ impl OperatorConfig { pub fn local(skills: &[&str]) -> Self { OperatorConfig { namespaces: [( - "local".to_owned(), + Namespace::new("local"), NamespaceConfig::InPlace { skills: skills .iter() @@ -68,7 +106,7 @@ impl OperatorConfig { RegistryConfig::new( self.namespaces .iter() - .map(|(k, v)| (k.to_owned(), v.registry())) + .map(|(k, v)| (k.to_owned().0, v.registry())) .collect(), ) } @@ -76,7 +114,7 @@ impl OperatorConfig { #[must_use] pub fn dev() -> Self { let namespaces = [( - "dev".to_owned(), + Namespace::new("dev"), NamespaceConfig::Watch { directory: "skills".into(), }, @@ -87,42 +125,24 @@ impl OperatorConfig { } #[derive(Deserialize, Clone, PartialEq, Eq, Debug)] -pub struct RegistryAuth { - user_env_var: String, - password_env_var: String, -} - -impl RegistryAuth { - pub fn user(&self) -> String { - env::var(self.user_env_var.as_str()).unwrap_or_else(|_| { - panic!( - "{} must be set if OCI registry is used.", - self.user_env_var.as_str() - ) - }) - } - - pub fn password(&self) -> String { - env::var(self.password_env_var.as_str()).unwrap_or_else(|_| { - panic!( - "{} must be set if OCI registry is used.", - self.password_env_var.as_str() - ) - }) - } -} - -#[derive(Deserialize, Clone, PartialEq, Eq, Debug)] -#[serde(rename_all = "snake_case", tag = "type")] +#[serde(untagged)] pub enum Registry { - File { - path: String, - }, + // https://serde.rs/enum-representations.html#untagged + // Serde will try to match the data against each variant in order and the first one that + // deserializes successfully is the one returned. + // + // Therefore, we put `Oci` first as this is most likely the variant someone will use in production. + #[serde(rename_all = "kebab-case")] Oci { registry: String, - repository: String, - #[serde(flatten)] - auth: RegistryAuth, + base_repository: String, + #[serde(rename = "registry-user")] + user: String, + #[serde(rename = "registry-password")] + password: String, + }, + File { + path: String, }, } @@ -132,9 +152,11 @@ pub enum NamespaceConfig { /// Namespaces are our way to enable teams to deploy skills in self service via Git Ops. This /// implies that the skills in team owned namespaces are configured by a team rather than the /// operators of Pharia Kernel, which in turn means we only refer the teams documentation here. + #[serde(rename_all = "kebab-case")] TeamOwned { config_url: String, - config_access_token_env_var: Option, + config_access_token: Option, + #[serde(flatten)] registry: Registry, }, /// For development it is convenient to just watch a local repository for changing skills @@ -146,6 +168,7 @@ pub enum NamespaceConfig { /// tests. InPlace { skills: Vec, + #[serde(flatten)] registry: Registry, }, } @@ -165,28 +188,16 @@ impl NamespaceConfig { match self { NamespaceConfig::TeamOwned { config_url, - config_access_token_env_var, + config_access_token, // Registry does not change dynamically at the moment registry: _, } => { let url = Url::parse(config_url)?; match url.scheme() { - "https" | "http" => { - let config_access_token = config_access_token_env_var - .as_ref() - .map(env::var) - .transpose() - .with_context(|| { - format!( - "Missing environment variable: {}", - config_access_token_env_var.as_ref().unwrap() - ) - })?; - Ok(Box::new(HttpLoader::from_url( - config_url, - config_access_token, - ))) - } + "https" | "http" => Ok(Box::new(HttpLoader::from_url( + config_url, + config_access_token.clone(), + ))), "file" => { // remove leading "file://" let file_path = &config_url[7..]; @@ -211,11 +222,12 @@ impl NamespaceConfig { #[cfg(test)] mod tests { - use crate::namespace_watcher::config::{Registry, RegistryAuth}; + use std::fs; + use std::io::Write; - use crate::namespace_watcher::tests::NamespaceConfig; + use tempfile::tempdir; - use super::OperatorConfig; + use super::*; impl OperatorConfig { /// # Panics @@ -224,12 +236,195 @@ mod tests { pub fn empty() -> Self { Self::from_toml("[namespaces]").unwrap() } + + /// # Errors + /// Cannot parse config. + pub fn from_toml(config: &str) -> anyhow::Result { + Ok(toml::from_str(config)?) + } + } + + #[test] + fn load_from_two_empty_sources() -> anyhow::Result<()> { + // Given a TOML file and environment variables + let dir = tempdir()?; + let file_path = dir.path().join("operator-config.toml"); + fs::File::create_new(&file_path)?; + let file_source = File::with_name(file_path.to_str().unwrap()); + let env_vars = HashMap::new(); + let env_source = OperatorConfig::environment().source(Some(env_vars)); + + // When loading from the sources + let config = OperatorConfig::from_sources(file_source, env_source)?; + + // Then both sources are applied, with the values from environment variables having precedence + assert_eq!(config.namespaces.len(), 0); + Ok(()) + } + + #[test] + fn load_two_namespaces_from_independent_sources() -> anyhow::Result<()> { + // Given a TOML file and environment variables + let dir = tempdir()?; + let file_path = dir.path().join("operator-config.toml"); + let mut file = fs::File::create_new(&file_path)?; + writeln!( + file, + r#"[namespaces.a] +config-url = "a" +config-access-token = "a" +registry = "a" +base-repository = "a" +registry-user = "a" +registry-password = "a""# + )?; + let file_source = File::with_name(file_path.to_str().unwrap()); + let env_vars = HashMap::from([ + ("NAMESPACES__B__CONFIG_URL".to_owned(), "b".to_owned()), + ( + "NAMESPACES__B__CONFIG_ACCESS_TOKEN".to_owned(), + "b".to_owned(), + ), + ("NAMESPACES__B__REGISTRY".to_owned(), "b".to_owned()), + ("NAMESPACES__B__BASE_REPOSITORY".to_owned(), "b".to_owned()), + ("NAMESPACES__B__REGISTRY_USER".to_owned(), "b".to_owned()), + ( + "NAMESPACES__B__REGISTRY_PASSWORD".to_owned(), + "b".to_owned(), + ), + ]); + let env_source = OperatorConfig::environment().source(Some(env_vars)); + + // When loading from the sources + let config = OperatorConfig::from_sources(file_source, env_source)?; + + // Then both namespaces are loaded + assert_eq!(config.namespaces.len(), 2); + let namespace_a = Namespace::new("a"); + assert!(config.namespaces.contains_key(&namespace_a)); + let namespace_b = Namespace::new("b"); + assert!(config.namespaces.contains_key(&namespace_b)); + Ok(()) + } + + #[test] + fn load_one_namespace_from_two_partial_sources() -> anyhow::Result<()> { + // Given a TOML file and environment variables + let config_url = "https://acme.com/latest/config.toml"; + let config_access_token = "ACME_CONFIG_ACCESS_TOKEN"; + let registry = "registry.acme.com"; + let base_repository = "engineering/skills"; + let user = "DUMMY_USER"; + let password = "DUMMY_PASSWORD"; + let dir = tempdir()?; + let file_path = dir.path().join("operator-config.toml"); + let mut file = fs::File::create_new(&file_path)?; + writeln!( + file, + "[namespaces.acme] +config-access-token = \"{config_access_token}\" +registry = \"{registry}\" +base-repository = \"{base_repository}\" +registry-password = \"{password}\" + " + )?; + let file_source = File::with_name(file_path.to_str().unwrap()); + let env_vars = HashMap::from([ + ( + "NAMESPACES__ACME__CONFIG_URL".to_owned(), + config_url.to_owned(), + ), + ( + "NAMESPACES__ACME__REGISTRY_USER".to_owned(), + user.to_owned(), + ), + ]); + let env_source = OperatorConfig::environment().source(Some(env_vars)); + + // When loading from the sources + let config = OperatorConfig::from_sources(file_source, env_source)?; + + // Then both sources are applied, with the values from environment variables having higher precedence + assert_eq!(config.namespaces.len(), 1); + let namespace_config = NamespaceConfig::TeamOwned { + config_url: config_url.to_owned(), + config_access_token: Some(config_access_token.to_owned()), + registry: Registry::Oci { + registry: registry.to_owned(), + base_repository: base_repository.to_owned(), + user: user.to_owned(), + password: password.to_owned(), + }, + }; + let namespace = Namespace::new("acme"); + assert_eq!( + config.namespaces.get(&namespace).unwrap(), + &namespace_config + ); + Ok(()) + } + + #[test] + fn deserialize_empty_operator_config() -> anyhow::Result<()> { + // Given a hashmap with variables + let env_vars = HashMap::from([]); + + // When we build the source from the environment variables + let source = OperatorConfig::environment().source(Some(env_vars)); + let config = Config::builder() + .add_source(source) + .build()? + .try_deserialize::()?; + + assert_eq!(config, OperatorConfig::empty()); + Ok(()) + } + + #[test] + fn deserialize_operator_config_with_namespaces() -> anyhow::Result<()> { + // Given a hashmap with variables + let env_vars = HashMap::from([ + ( + "NAMESPACES__PLAY_GROUND__CONFIG_URL".to_owned(), + "https://gitlab.aleph-alpha.de/playground".to_owned(), + ), + ( + "NAMESPACES__PLAY_GROUND__CONFIG_ACCESS_TOKEN".to_owned(), + "GITLAB_CONFIG_ACCESS_TOKEN".to_owned(), + ), + ( + "NAMESPACES__PLAY_GROUND__REGISTRY".to_owned(), + "registry.gitlab.aleph-alpha.de".to_owned(), + ), + ( + "NAMESPACES__PLAY_GROUND__BASE_REPOSITORY".to_owned(), + "engineering/pharia-skills/skills".to_owned(), + ), + ( + "NAMESPACES__PLAY_GROUND__REGISTRY_USER".to_owned(), + "SKILL_REGISTRY_USER".to_owned(), + ), + ( + "NAMESPACES__PLAY_GROUND__REGISTRY_PASSWORD".to_owned(), + "SKILL_REGISTRY_PASSWORD".to_owned(), + ), + ]); + + // When we build the source from the environment variables + let source = OperatorConfig::environment().source(Some(env_vars)); + let config = Config::builder() + .add_source(source) + .build()? + .try_deserialize::()?; + assert_eq!(config.namespaces.len(), 1); + Ok(()) } #[test] fn deserialize_config_with_file_registry() { let config = OperatorConfig::local(&[]); - assert!(config.namespaces.contains_key("local")); + let namespace = Namespace::new("local"); + assert!(config.namespaces.contains_key(&namespace)); } #[test] @@ -241,7 +436,8 @@ mod tests { "#, ) .unwrap(); - let local_namespace = config.namespaces.get("local").unwrap(); + let namespace = Namespace::new("local"); + let local_namespace = config.namespaces.get(&namespace).unwrap(); let registry = local_namespace.registry(); @@ -253,27 +449,23 @@ mod tests { let config = OperatorConfig::from_toml( r#" [namespaces.pharia-kernel-team] - config_url = "https://dummy_url" - - [namespaces.pharia-kernel-team.registry] - type = "oci" + config-url = "https://dummy_url" registry = "registry.gitlab.aleph-alpha.de" - repository = "engineering/pharia-skills/skills" - user_env_var = "SKILL_REGISTRY_USER" - password_env_var = "SKILL_REGISTRY_PASSWORD" + base-repository = "engineering/pharia-skills/skills" + registry-user = "DUMMY_USER" + registry-password = "DUMMY_PASSWORD" "#, ) .unwrap(); - let pharia_kernel_team = config.namespaces.get("pharia-kernel-team").unwrap(); + let namespace = Namespace::new("pharia-kernel-team"); + let pharia_kernel_team = config.namespaces.get(&namespace).unwrap(); assert_eq!( pharia_kernel_team.registry(), Registry::Oci { registry: "registry.gitlab.aleph-alpha.de".to_owned(), - repository: "engineering/pharia-skills/skills".to_owned(), - auth: RegistryAuth { - user_env_var: "SKILL_REGISTRY_USER".to_owned(), - password_env_var: "SKILL_REGISTRY_PASSWORD".to_owned(), - }, + base_repository: "engineering/pharia-skills/skills".to_owned(), + user: "DUMMY_USER".to_owned(), + password: "DUMMY_PASSWORD".to_owned(), } ); } @@ -282,20 +474,18 @@ mod tests { fn deserialize_config_with_config_access_token() { let config = OperatorConfig::from_toml( r#" - [namespaces.dummy_team] - config_url = "file://dummy_config_url" - config_access_token_env_var = "GITLAB_CONFIG_ACCESS_TOKEN" - - [namespaces.dummy_team.registry] - type = "file" + [namespaces.dummy-team] + config-url = "file://dummy_config_url" + config-access-token = "GITLAB_CONFIG_ACCESS_TOKEN" path = "dummy_file_path" "#, ) .unwrap(); - let namespace_cfg = config.namespaces.get("dummy_team").unwrap(); + let namespace = Namespace::new("dummy-team"); + let namespace_cfg = config.namespaces.get(&namespace).unwrap(); let expected = NamespaceConfig::TeamOwned { config_url: "file://dummy_config_url".to_owned(), - config_access_token_env_var: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), + config_access_token: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), registry: Registry::File { path: "dummy_file_path".to_owned(), }, @@ -305,31 +495,52 @@ mod tests { #[test] fn reads_from_file() { - let config = OperatorConfig::from_file("operator-config.toml").unwrap(); - assert!(config.namespaces.contains_key("pharia-kernel-team")); + drop(dotenvy::dotenv()); + let config = OperatorConfig::new("operator-config.toml").unwrap(); + let namespace = Namespace::new("pharia-kernel-team"); + assert!(config.namespaces.contains_key(&namespace)); } #[test] - fn deserializes_multiple_namespaces() { + fn prioritizes_oci_registry() { + // When deserializing a config which contains both, an oci and a file registry + // for the same namespace let config = toml::from_str::( r#" [namespaces.pharia-kernel-team] - config_url = "https://dummy_url" - config_access_token_env_var = "GITLAB_CONFIG_ACCESS_TOKEN" + config-url = "https://dummy_url" + config-access-token = "GITLAB_CONFIG_ACCESS_TOKEN" + registry = "registry.gitlab.aleph-alpha.de" + base-repository = "engineering/pharia-skills/skills" + registry-user = "PHARIA_KERNEL_TEAM_REGISTRY_USER" + registry-password = "PHARIA_KERNEL_TEAM_REGISTRY_PASSWORD" + path = "local-path" + "#, + ) + .unwrap(); + + let key = Namespace::new("pharia-kernel-team"); + let namespace = config.namespaces.get(&key).unwrap(); - [namespaces.pharia-kernel-team.registry] - type = "oci" + // Then the `Oci` variants is prioritized + assert!(matches!(namespace.registry(), Registry::Oci { .. })); + } + + #[test] + fn deserializes_multiple_namespaces() { + let config = toml::from_str::( + r#" + [namespaces.pharia-kernel-team] + config-url = "https://dummy_url" + config-access-token = "GITLAB_CONFIG_ACCESS_TOKEN" registry = "registry.gitlab.aleph-alpha.de" - repository = "engineering/pharia-skills/skills" - user_env_var = "PHARIA_KERNEL_TEAM_REGISTRY_USER" - password_env_var = "PHARIA_KERNEL_TEAM_REGISTRY_PASSWORD" + base-repository = "engineering/pharia-skills/skills" + registry-user = "PHARIA_KERNEL_TEAM_REGISTRY_USER" + registry-password = "PHARIA_KERNEL_TEAM_REGISTRY_PASSWORD" [namespaces.pharia-kernel-team-local] - config_url = "https://dummy_url" - config_access_token_env_var = "GITLAB_CONFIG_ACCESS_TOKEN" - - [namespaces.pharia-kernel-team-local.registry] - type = "file" + config-url = "https://dummy_url" + config-access-token = "GITLAB_CONFIG_ACCESS_TOKEN" path = "/temp/skills" "#, ) @@ -338,25 +549,23 @@ mod tests { let expected = OperatorConfig { namespaces: [ ( - "pharia-kernel-team".to_owned(), + Namespace::new("pharia-kernel-team"), NamespaceConfig::TeamOwned { config_url: "https://dummy_url".to_owned(), - config_access_token_env_var: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), + config_access_token: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), registry: Registry::Oci { registry: "registry.gitlab.aleph-alpha.de".to_owned(), - repository: "engineering/pharia-skills/skills".to_owned(), - auth: RegistryAuth { - user_env_var: "PHARIA_KERNEL_TEAM_REGISTRY_USER".to_owned(), - password_env_var: "PHARIA_KERNEL_TEAM_REGISTRY_PASSWORD".to_owned(), - }, + base_repository: "engineering/pharia-skills/skills".to_owned(), + user: "PHARIA_KERNEL_TEAM_REGISTRY_USER".to_owned(), + password: "PHARIA_KERNEL_TEAM_REGISTRY_PASSWORD".to_owned(), }, }, ), ( - "pharia-kernel-team-local".to_owned(), + Namespace::new("pharia-kernel-team-local"), NamespaceConfig::TeamOwned { config_url: "https://dummy_url".to_owned(), - config_access_token_env_var: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), + config_access_token: Some("GITLAB_CONFIG_ACCESS_TOKEN".to_owned()), registry: Registry::File { path: "/temp/skills".to_owned(), }, diff --git a/src/registries/oci.rs b/src/registries/oci.rs index 5b49a27c0..381679cd8 100644 --- a/src/registries/oci.rs +++ b/src/registries/oci.rs @@ -14,22 +14,41 @@ use crate::registries::SkillRegistry; pub struct OciRegistry { client: WasmClient, registry: String, - repository: String, + base_repository: String, username: String, password: String, } impl OciRegistry { + pub fn new( + registry: String, + base_repository: String, + username: String, + password: String, + ) -> Self { + let client = Client::new(ClientConfig::default()); + let client = WasmClient::new(client); + + Self { + client, + registry, + base_repository: base_repository.trim_matches('/').to_owned(), + username, + password, + } + } + fn auth(&self) -> RegistryAuth { RegistryAuth::Basic(self.username.clone(), self.password.clone()) } - fn reference(&self, name: &str, tag: String) -> Reference { - Reference::with_tag( - self.registry.clone(), - format!("{}/{name}", self.repository), - tag, - ) + fn reference(&self, name: &str, tag: impl Into) -> Reference { + let repository = if self.base_repository.is_empty() { + name.to_owned() + } else { + format!("{}/{name}", self.base_repository) + }; + Reference::with_tag(self.registry.clone(), repository, tag.into()) } } @@ -39,7 +58,7 @@ impl SkillRegistry for OciRegistry { name: &'a str, tag: &'a str, ) -> DynFuture<'a, anyhow::Result>> { - let image = self.reference(name, tag.to_owned()); + let image = self.reference(name, tag); Box::pin(async move { // We want to match on the specific type of result. @@ -82,7 +101,7 @@ impl SkillRegistry for OciRegistry { Box::pin(async move { let result = self .client - .fetch_manifest_digest(&self.reference(name, tag.to_owned()), &self.auth()) + .fetch_manifest_digest(&self.reference(name, tag), &self.auth()) .await; match result { Ok(digest) => Ok(Some(Digest(digest))), @@ -100,21 +119,6 @@ impl SkillRegistry for OciRegistry { } } -impl OciRegistry { - pub fn new(repository: String, registry: String, username: String, password: String) -> Self { - let client = Client::new(ClientConfig::default()); - let client = WasmClient::new(client); - - Self { - client, - registry, - repository, - username, - password, - } - } -} - fn anyhow_is_skill_not_found(error: &anyhow::Error) -> bool { let Some(error) = error.downcast_ref::() else { return false; @@ -150,34 +154,34 @@ mod tests { impl OciRegistry { fn from_env() -> Option { - let maybe_repository = env::var("SKILL_REPOSITORY"); - let maybe_registry = env::var("SKILL_REGISTRY"); - let maybe_username = env::var("SKILL_REGISTRY_USER"); - let maybe_password = env::var("SKILL_REGISTRY_PASSWORD"); + let maybe_registry = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY"); + let maybe_repository = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__BASE_REPOSITORY"); + let maybe_username = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER"); + let maybe_password = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD"); match ( - maybe_repository, maybe_registry, + maybe_repository, maybe_username, maybe_password, ) { - (Ok(repository), Ok(registry), Ok(username), Ok(password)) => { - Some(OciRegistry::new(repository, registry, username, password)) + (Ok(registry), Ok(repository), Ok(username), Ok(password)) => { + Some(OciRegistry::new(registry, repository, username, password)) } _ => None, } } async fn store_skill(&self, path: impl AsRef, skill_name: &str, tag: &str) { - let repository = format!("{}/{skill_name}", self.repository); + let repository = format!("{}/{skill_name}", self.base_repository); let image = Reference::with_tag(self.registry.clone(), repository, tag.to_owned()); let (config, component_layer) = WasmConfig::from_component(path, None) .await .expect("component must be valid"); - let username = - env::var("SKILL_REGISTRY_USER").expect("SKILL_REGISTRY_USER variable not set"); - let password = env::var("SKILL_REGISTRY_PASSWORD") - .expect("SKILL_REGISTRY_PASSWORD variable not set"); + let username = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER") + .expect("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_USER variable not set"); + let password = env::var("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD") + .expect("NAMESPACES__PHARIA_KERNEL_TEAM__REGISTRY_PASSWORD variable not set"); let auth = RegistryAuth::Basic(username, password); self.client @@ -270,4 +274,31 @@ mod tests { // then skill can not be found assert!(bytes.is_err()); } + + #[test] + fn oci_registry_sanitizes_base_repository_input() { + let registry = OciRegistry::new( + "127.0.0.1:6000".to_owned(), + "/skills/".to_owned(), + "dummy-user".to_owned(), + "dummy-password".to_owned(), + ); + + assert_eq!(registry.base_repository, "skills"); + } + + #[test] + fn oci_registry_handles_empty_base_repository() { + let skill_name = "skill"; + let registry = OciRegistry::new( + "127.0.0.1:6000".to_owned(), + String::new(), + "dummy-user".to_owned(), + "dummy-password".to_owned(), + ); + + let reference = registry.reference(skill_name, "tag"); + + assert_eq!(reference.repository(), skill_name); + } } diff --git a/src/shell.rs b/src/shell.rs index c51dae290..0a9b731d5 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -306,7 +306,6 @@ async fn serve_docs() -> Json { struct ExecuteSkillArgs { /// The qualified name of the skill to invoke. The qualified name consists of a namespace and /// a skill name (e.g. "acme/summarize"). - /// If the namespace is omitted, the default 'pharia-kernel-team' namespace is used. /// skill: String, /// The expected input for the skill in JSON format. Examples: diff --git a/src/skill_loader.rs b/src/skill_loader.rs index 0642d65d9..353a110e3 100644 --- a/src/skill_loader.rs +++ b/src/skill_loader.rs @@ -76,14 +76,15 @@ impl From<&Registry> for Arc { match val { Registry::File { path } => Arc::new(FileRegistry::with_dir(path)), Registry::Oci { - repository, registry, - auth, + base_repository, + user, + password, } => Arc::new(OciRegistry::new( - repository.clone(), registry.clone(), - auth.user(), - auth.password(), + base_repository.clone(), + user.clone(), + password.clone(), )), } }