From 1cfda3945d565327d790e62ef3eef3e1d964ceda Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 11:20:04 -0500 Subject: [PATCH 01/20] changing schema naming Signed-off-by: Alexa Griffith --- api/v1alpha1/api.go | 6 ++--- filterconfig/filterconfig.go | 18 +++++++-------- filterconfig/filterconfig_test.go | 14 ++++++------ internal/apischema/awsbedrock/awsbedrock.go | 10 ++++----- internal/controller/sink.go | 8 +++---- internal/controller/sink_test.go | 4 ++-- internal/extproc/processor.go | 6 ++--- internal/extproc/router/request_body.go | 6 ++--- internal/extproc/router/router.go | 14 ++++++------ internal/extproc/router/router_test.go | 22 +++++++++---------- internal/extproc/server.go | 6 ++--- internal/extproc/server_test.go | 10 ++++----- .../extproc/translator/openai_awsbedrock.go | 2 +- .../translator/openai_awsbedrock_test.go | 2 +- internal/extproc/watcher_test.go | 12 +++++----- .../aigateway.envoyproxy.io_llmbackends.yaml | 4 ++-- .../aigateway.envoyproxy.io_llmroutes.yaml | 6 ++--- tests/cel-validation/main_test.go | 6 ++--- .../llmbackends/basic-eg-backend.yaml | 2 +- .../testdata/llmbackends/basic.yaml | 2 +- .../testdata/llmbackends/unknown_schema.yaml | 2 +- .../testdata/llmroutes/basic.yaml | 2 +- .../testdata/llmroutes/non_openai_schema.yaml | 2 +- .../testdata/llmroutes/unknown_schema.yaml | 2 +- .../testdata/llmroutes/unsupported_match.yaml | 2 +- tests/extproc/extproc_test.go | 6 ++--- 26 files changed, 88 insertions(+), 88 deletions(-) diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go index 01fbaac4..6196fcf9 100644 --- a/api/v1alpha1/api.go +++ b/api/v1alpha1/api.go @@ -14,7 +14,7 @@ import ( // This serves as a way to define a "unified" LLM API for a Gateway which allows downstream // clients to use a single schema API to interact with multiple LLM backends. // -// The inputSchema field is used to determine the structure of the requests that the Gateway will +// The schema field is used to determine the structure of the requests that the Gateway will // receive. And then the Gateway will route the traffic to the appropriate LLMBackend based // on the output schema of the LLMBackend while doing the other necessary jobs like // upstream authentication, rate limit, etc. @@ -52,7 +52,7 @@ type LLMRouteSpec struct { // // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self.schema == 'OpenAI'" - APISchema LLMAPISchema `json:"inputSchema"` + APISchema LLMAPISchema `json:"schema"` // Rules is the list of LLMRouteRule that this LLMRoute will match the traffic to. // Each rule is a subset of the HTTPRoute in the Gateway API (https://gateway-api.sigs.k8s.io/api-types/httproute/). // @@ -158,7 +158,7 @@ type LLMBackendSpec struct { // This is required to be set. // // +kubebuilder:validation:Required - APISchema LLMAPISchema `json:"outputSchema"` + APISchema LLMAPISchema `json:"schema"` // BackendRef is the reference to the Backend resource that this LLMBackend corresponds to. // // A backend can be of either k8s Service or Backend resource of Envoy Gateway. diff --git a/filterconfig/filterconfig.go b/filterconfig/filterconfig.go index 6fe69a11..87c9024a 100644 --- a/filterconfig/filterconfig.go +++ b/filterconfig/filterconfig.go @@ -19,7 +19,7 @@ import ( // DefaultConfig is the default configuration that can be used as a // fallback when the configuration is not explicitly provided. const DefaultConfig = ` -inputSchema: +schema: schema: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-envoy-ai-gateway-model @@ -29,7 +29,7 @@ modelNameHeaderKey: x-envoy-ai-gateway-model // // # Example configuration: // -// inputSchema: +// schema: // schema: OpenAI // selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend // modelNameHeaderKey: x-envoy-ai-gateway-model @@ -40,18 +40,18 @@ modelNameHeaderKey: x-envoy-ai-gateway-model // - backends: // - name: kserve // weight: 1 -// outputSchema: +// schema: // schema: OpenAI // - name: awsbedrock // weight: 10 -// outputSchema: +// schema: // schema: AWSBedrock // headers: // - name: x-envoy-ai-gateway-model // value: llama3.3333 // - backends: // - name: openai -// outputSchema: +// schema: // schema: OpenAI // headers: // - name: x-envoy-ai-gateway-model @@ -70,8 +70,8 @@ type Config struct { // If this is provided, the filter will populate the usage token in the filter metadata at the end of the // response body processing. TokenUsageMetadata *TokenUsageMetadata `yaml:"tokenUsageMetadata,omitempty"` - // InputSchema specifies the API schema of the input format of requests to the filter. - InputSchema VersionedAPISchema `yaml:"inputSchema"` + // Schema specifies the API schema of the input format of requests to the filter. + Schema VersionedAPISchema `yaml:"schema"` // ModelNameHeaderKey is the header key to be populated with the model name by the filter. ModelNameHeaderKey string `yaml:"modelNameHeaderKey"` // SelectedBackendHeaderKey is the header key to be populated with the backend name by the filter @@ -130,8 +130,8 @@ type Backend struct { // Name of the backend, which is the value in the final routing decision // matching the header key specified in the [Config.BackendRoutingHeaderKey]. Name string `yaml:"name"` - // OutputSchema specifies the API schema of the output format of requests from. - OutputSchema VersionedAPISchema `yaml:"outputSchema"` + // Schema specifies the API schema of the output format of requests from. + Schema VersionedAPISchema `yaml:"schema"` // Weight is the weight of the backend in the routing decision. Weight int `yaml:"weight"` // Auth is the authn/z configuration for the backend. Optional. diff --git a/filterconfig/filterconfig_test.go b/filterconfig/filterconfig_test.go index 48095e09..3e855dc5 100644 --- a/filterconfig/filterconfig_test.go +++ b/filterconfig/filterconfig_test.go @@ -29,7 +29,7 @@ func TestDefaultConfig(t *testing.T) { func TestUnmarshalConfigYaml(t *testing.T) { configPath := path.Join(t.TempDir(), "config.yaml") const config = ` -inputSchema: +schema: schema: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-envoy-ai-gateway-model @@ -40,18 +40,18 @@ rules: - backends: - name: kserve weight: 1 - outputSchema: + schema: schema: OpenAI - name: awsbedrock weight: 10 - outputSchema: + schema: schema: AWSBedrock headers: - name: x-envoy-ai-gateway-model value: llama3.3333 - backends: - name: openai - outputSchema: + schema: schema: OpenAI headers: - name: x-envoy-ai-gateway-model @@ -62,7 +62,7 @@ rules: require.NoError(t, err) require.Equal(t, "ai_gateway_llm_ns", cfg.TokenUsageMetadata.Namespace) require.Equal(t, "token_usage_key", cfg.TokenUsageMetadata.Key) - require.Equal(t, "OpenAI", string(cfg.InputSchema.Schema)) + require.Equal(t, "OpenAI", string(cfg.schema.Schema)) require.Equal(t, "x-envoy-ai-gateway-selected-backend", cfg.SelectedBackendHeaderKey) require.Equal(t, "x-envoy-ai-gateway-model", cfg.ModelNameHeaderKey) require.Len(t, cfg.Rules, 2) @@ -70,7 +70,7 @@ rules: require.Equal(t, "gpt4.4444", cfg.Rules[1].Headers[0].Value) require.Equal(t, "kserve", cfg.Rules[0].Backends[0].Name) require.Equal(t, 10, cfg.Rules[0].Backends[1].Weight) - require.Equal(t, "AWSBedrock", string(cfg.Rules[0].Backends[1].OutputSchema.Schema)) + require.Equal(t, "AWSBedrock", string(cfg.Rules[0].Backends[1].Schema.Schema)) require.Equal(t, "openai", cfg.Rules[1].Backends[0].Name) - require.Equal(t, "OpenAI", string(cfg.Rules[1].Backends[0].OutputSchema.Schema)) + require.Equal(t, "OpenAI", string(cfg.Rules[1].Backends[0].Schema.Schema)) } diff --git a/internal/apischema/awsbedrock/awsbedrock.go b/internal/apischema/awsbedrock/awsbedrock.go index 25de5ba2..203fcdfc 100644 --- a/internal/apischema/awsbedrock/awsbedrock.go +++ b/internal/apischema/awsbedrock/awsbedrock.go @@ -448,8 +448,8 @@ type Tool struct { ToolSpec *ToolSpecification `json:"toolSpec"` } -// ToolInputSchema The schema for the tool. The top level schema type must be an object. -type ToolInputSchema struct { +// ToolSchema The schema for the tool. The top level schema type must be an object. +type ToolSchema struct { JSON any `json:"json"` } @@ -458,10 +458,10 @@ type ToolSpecification struct { // The description for the tool. Description *string `json:"description,omitempty"` - // The input schema for the tool in JSON format. + // The schema for the tool in JSON format. // - // InputSchema is a required field - InputSchema *ToolInputSchema `json:"inputSchema"` + // Schema is a required field + Schema *ToolSchema `json:"schema"` // The name for the tool. // diff --git a/internal/controller/sink.go b/internal/controller/sink.go index c8699fe8..145e44a5 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -226,8 +226,8 @@ func (c *configSink) updateExtProcConfigMap(llmRoute *aigv1a1.LLMRoute) error { ec := &filterconfig.Config{} spec := &llmRoute.Spec - ec.InputSchema.Schema = filterconfig.APISchema(spec.APISchema.Schema) - ec.InputSchema.Version = spec.APISchema.Version + ec.Schema.Schema = filterconfig.APISchema(spec.APISchema.Schema) + ec.Schema.Version = spec.APISchema.Version ec.ModelNameHeaderKey = aigv1a1.LLMModelHeaderKey ec.SelectedBackendHeaderKey = selectedBackendHeaderKey ec.Rules = make([]filterconfig.RouteRule, len(spec.Rules)) @@ -242,8 +242,8 @@ func (c *configSink) updateExtProcConfigMap(llmRoute *aigv1a1.LLMRoute) error { err = fmt.Errorf("backend %s not found", key) return err } else { - ec.Rules[i].Backends[j].OutputSchema.Schema = filterconfig.APISchema(backendObj.Spec.APISchema.Schema) - ec.Rules[i].Backends[j].OutputSchema.Version = backendObj.Spec.APISchema.Version + ec.Rules[i].Backends[j].Schema.Schema = filterconfig.APISchema(backendObj.Spec.APISchema.Schema) + ec.Rules[i].Backends[j].Schema.Version = backendObj.Spec.APISchema.Version } } ec.Rules[i].Headers = make([]filterconfig.HeaderMatch, len(rule.Matches)) diff --git a/internal/controller/sink_test.go b/internal/controller/sink_test.go index 7aa29b38..5be13c5e 100644 --- a/internal/controller/sink_test.go +++ b/internal/controller/sink_test.go @@ -446,13 +446,13 @@ func Test_updateExtProcConfigMap(t *testing.T) { }, }, exp: &filterconfig.Config{ - InputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, + Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, ModelNameHeaderKey: aigv1a1.LLMModelHeaderKey, SelectedBackendHeaderKey: selectedBackendHeaderKey, Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "apple.ns", Weight: 1, OutputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, + {Name: "apple.ns", Weight: 1, Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, }, Headers: []filterconfig.HeaderMatch{{Name: aigv1a1.LLMModelHeaderKey, Value: "some-ai"}}, }, diff --git a/internal/extproc/processor.go b/internal/extproc/processor.go index 39ca6630..0320a2a2 100644 --- a/internal/extproc/processor.go +++ b/internal/extproc/processor.go @@ -73,14 +73,14 @@ func (p *Processor) ProcessRequestBody(_ context.Context, rawBody *extprocv3.Htt } p.requestHeaders[p.config.ModelNameHeaderKey] = model - backendName, outputSchema, err := p.config.router.Calculate(p.requestHeaders) + backendName, schema, err := p.config.router.Calculate(p.requestHeaders) if err != nil { return nil, fmt.Errorf("failed to calculate route: %w", err) } - factory, ok := p.config.factories[outputSchema] + factory, ok := p.config.factories[schema] if !ok { - return nil, fmt.Errorf("failed to find factory for output schema %q", outputSchema) + return nil, fmt.Errorf("failed to find factory for output schema %q", schema) } t, err := factory(path) diff --git a/internal/extproc/router/request_body.go b/internal/extproc/router/request_body.go index 84c800f6..91661420 100644 --- a/internal/extproc/router/request_body.go +++ b/internal/extproc/router/request_body.go @@ -14,11 +14,11 @@ import ( type RequestBodyParser func(path string, body *extprocv3.HttpBody) (modelName string, rb RequestBody, err error) // NewRequestBodyParser creates a new RequestBodyParser based on the schema. -func NewRequestBodyParser(inputSchema filterconfig.VersionedAPISchema) (RequestBodyParser, error) { - if inputSchema.Schema == filterconfig.APISchemaOpenAI { +func NewRequestBodyParser(schema filterconfig.VersionedAPISchema) (RequestBodyParser, error) { + if schema.Schema == filterconfig.APISchemaOpenAI { return openAIParseBody, nil } - return nil, fmt.Errorf("unsupported API schema: %s", inputSchema) + return nil, fmt.Errorf("unsupported API schema: %s", schema) } // RequestBody is the union of all request body types. diff --git a/internal/extproc/router/router.go b/internal/extproc/router/router.go index a604a023..537ece45 100644 --- a/internal/extproc/router/router.go +++ b/internal/extproc/router/router.go @@ -13,7 +13,7 @@ import ( type Router interface { // Calculate determines the backend to route to based on the headers. // Returns the backend name and the output schema. - Calculate(headers map[string]string) (backendName string, outputSchema filterconfig.VersionedAPISchema, err error) + Calculate(headers map[string]string) (backendName string, schema filterconfig.VersionedAPISchema, err error) } // router implements [Router]. @@ -28,7 +28,7 @@ func NewRouter(config *filterconfig.Config) (Router, error) { } // Calculate implements [Router.Calculate]. -func (r *router) Calculate(headers map[string]string) (backendName string, outputSchema filterconfig.VersionedAPISchema, err error) { +func (r *router) Calculate(headers map[string]string) (backendName string, schema filterconfig.VersionedAPISchema, err error) { var rule *filterconfig.RouteRule for i := range r.rules { _rule := &r.rules[i] @@ -44,11 +44,11 @@ func (r *router) Calculate(headers map[string]string) (backendName string, outpu if rule == nil { return "", filterconfig.VersionedAPISchema{}, errors.New("no matching rule found") } - backendName, outputSchema = r.selectBackendFromRule(rule) + backendName, schema = r.selectBackendFromRule(rule) return } -func (r *router) selectBackendFromRule(rule *filterconfig.RouteRule) (backendName string, outputSchema filterconfig.VersionedAPISchema) { +func (r *router) selectBackendFromRule(rule *filterconfig.RouteRule) (backendName string, schema filterconfig.VersionedAPISchema) { // Each backend has a weight, so we randomly select depending on the weight. // This is a pretty naive implementation and can be buggy, so fix it later. totalWeight := 0 @@ -56,14 +56,14 @@ func (r *router) selectBackendFromRule(rule *filterconfig.RouteRule) (backendNam totalWeight += b.Weight } if totalWeight == 0 { - return rule.Backends[0].Name, rule.Backends[0].OutputSchema + return rule.Backends[0].Name, rule.Backends[0].schema } selected := r.rng.Intn(totalWeight) for _, b := range rule.Backends { if selected < b.Weight { - return b.Name, b.OutputSchema + return b.Name, b.schema } selected -= b.Weight } - return rule.Backends[0].Name, rule.Backends[0].OutputSchema + return rule.Backends[0].Name, rule.Backends[0].schema } diff --git a/internal/extproc/router/router_test.go b/internal/extproc/router/router_test.go index c20bbe1f..cc56a0b9 100644 --- a/internal/extproc/router/router_test.go +++ b/internal/extproc/router/router_test.go @@ -14,8 +14,8 @@ func TestRouter_Calculate(t *testing.T) { Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "foo", OutputSchema: outSchema, Weight: 1}, - {Name: "bar", OutputSchema: outSchema, Weight: 3}, + {Name: "foo", Schema: outSchema, Weight: 1}, + {Name: "bar", Schema: outSchema, Weight: 3}, }, Headers: []filterconfig.HeaderMatch{ {Name: "x-model-name", Value: "llama3.3333"}, @@ -23,7 +23,7 @@ func TestRouter_Calculate(t *testing.T) { }, { Backends: []filterconfig.Backend{ - {Name: "openai", OutputSchema: outSchema}, + {Name: "openai", Schema: outSchema}, }, Headers: []filterconfig.HeaderMatch{ {Name: "x-model-name", Value: "gpt4.4444"}, @@ -36,25 +36,25 @@ func TestRouter_Calculate(t *testing.T) { require.True(t, ok) t.Run("no matching rule", func(t *testing.T) { - backendName, outputSchema, err := r.Calculate(map[string]string{"x-model-name": "something-quirky"}) + backendName, Schema, err := r.Calculate(map[string]string{"x-model-name": "something-quirky"}) require.Error(t, err) require.Empty(t, backendName) - require.Empty(t, outputSchema) + require.Empty(t, Schema) }) t.Run("matching rule - single backend choice", func(t *testing.T) { - backendName, outputSchema, err := r.Calculate(map[string]string{"x-model-name": "gpt4.4444"}) + backendName, Schema, err := r.Calculate(map[string]string{"x-model-name": "gpt4.4444"}) require.NoError(t, err) require.Equal(t, "openai", backendName) - require.Equal(t, outSchema, outputSchema) + require.Equal(t, outSchema, Schema) }) t.Run("matching rule - multiple backend choices", func(t *testing.T) { chosenNames := make(map[string]int) for i := 0; i < 1000; i++ { - backendName, outputSchema, err := r.Calculate(map[string]string{"x-model-name": "llama3.3333"}) + backendName, Schema, err := r.Calculate(map[string]string{"x-model-name": "llama3.3333"}) require.NoError(t, err) chosenNames[backendName]++ require.Contains(t, []string{"foo", "bar"}, backendName) - require.Equal(t, outSchema, outputSchema) + require.Equal(t, outSchema, Schema) } require.Greater(t, chosenNames["bar"], chosenNames["foo"]) require.Greater(t, chosenNames["bar"], 700) @@ -72,8 +72,8 @@ func TestRouter_selectBackendFromRule(t *testing.T) { rule := &filterconfig.RouteRule{ Backends: []filterconfig.Backend{ - {Name: "foo", OutputSchema: outSchema, Weight: 1}, - {Name: "bar", OutputSchema: outSchema, Weight: 3}, + {Name: "foo", Schema: outSchema, Weight: 1}, + {Name: "bar", Schema: outSchema, Weight: 3}, }, } diff --git a/internal/extproc/server.go b/internal/extproc/server.go index 810d52cd..4325f41b 100644 --- a/internal/extproc/server.go +++ b/internal/extproc/server.go @@ -33,7 +33,7 @@ func NewServer[P ProcessorIface](logger *slog.Logger, newProcessor func(*process // LoadConfig updates the configuration of the external processor. func (s *Server[P]) LoadConfig(config *filterconfig.Config) error { - bodyParser, err := router.NewRequestBodyParser(config.InputSchema) + bodyParser, err := router.NewRequestBodyParser(config.Schema) if err != nil { return fmt.Errorf("cannot create request body parser: %w", err) } @@ -46,8 +46,8 @@ func (s *Server[P]) LoadConfig(config *filterconfig.Config) error { backendAuthHandlers := make(map[string]backendauth.Handler) for _, r := range config.Rules { for _, b := range r.Backends { - if _, ok := factories[b.OutputSchema]; !ok { - factories[b.OutputSchema], err = translator.NewFactory(config.InputSchema, b.OutputSchema) + if _, ok := factories[b.Schema]; !ok { + factories[b.Schema], err = translator.NewFactory(config.Schema, b.Schema) if err != nil { return fmt.Errorf("cannot create translator factory: %w", err) } diff --git a/internal/extproc/server_test.go b/internal/extproc/server_test.go index 39c49aaa..464d6f58 100644 --- a/internal/extproc/server_test.go +++ b/internal/extproc/server_test.go @@ -29,7 +29,7 @@ func TestServer_LoadConfig(t *testing.T) { t.Run("invalid input schema", func(t *testing.T) { s := requireNewServerWithMockProcessor(t) err := s.LoadConfig(&filterconfig.Config{ - InputSchema: filterconfig.VersionedAPISchema{Schema: "some-invalid-schema"}, + Schema: filterconfig.VersionedAPISchema{Schema: "some-invalid-schema"}, }) require.Error(t, err) require.ErrorContains(t, err, "cannot create request body parser") @@ -37,14 +37,14 @@ func TestServer_LoadConfig(t *testing.T) { t.Run("ok", func(t *testing.T) { config := &filterconfig.Config{ TokenUsageMetadata: &filterconfig.TokenUsageMetadata{Namespace: "ns", Key: "key"}, - InputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, + Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, SelectedBackendHeaderKey: "x-envoy-ai-gateway-selected-backend", ModelNameHeaderKey: "x-model-name", Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "kserve", OutputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, - {Name: "awsbedrock", OutputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, + {Name: "kserve", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, + {Name: "awsbedrock", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, }, Headers: []filterconfig.HeaderMatch{ { @@ -55,7 +55,7 @@ func TestServer_LoadConfig(t *testing.T) { }, { Backends: []filterconfig.Backend{ - {Name: "openai", OutputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, + {Name: "openai", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, }, Headers: []filterconfig.HeaderMatch{ { diff --git a/internal/extproc/translator/openai_awsbedrock.go b/internal/extproc/translator/openai_awsbedrock.go index ce2cc70b..5bc7704e 100644 --- a/internal/extproc/translator/openai_awsbedrock.go +++ b/internal/extproc/translator/openai_awsbedrock.go @@ -116,7 +116,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIToolsToBedrockToolC ToolSpec: &awsbedrock.ToolSpecification{ Name: &toolType, Description: &toolDefinition.Function.Description, - InputSchema: &awsbedrock.ToolInputSchema{ + Schema: &awsbedrock.ToolSchema{ JSON: toolDefinition.Function.Parameters, }, }, diff --git a/internal/extproc/translator/openai_awsbedrock_test.go b/internal/extproc/translator/openai_awsbedrock_test.go index f4bcc25f..70c8badb 100644 --- a/internal/extproc/translator/openai_awsbedrock_test.go +++ b/internal/extproc/translator/openai_awsbedrock_test.go @@ -350,7 +350,7 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T) ToolSpec: &awsbedrock.ToolSpecification{ Name: ptr.To("function"), Description: ptr.To("Get the current weather in a given location"), - InputSchema: &awsbedrock.ToolInputSchema{ + Schema: &awsbedrock.ToolSchema{ JSON: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ diff --git a/internal/extproc/watcher_test.go b/internal/extproc/watcher_test.go index fd55ed21..3628ca7e 100644 --- a/internal/extproc/watcher_test.go +++ b/internal/extproc/watcher_test.go @@ -42,7 +42,7 @@ func TestStartConfigWatcher(t *testing.T) { // Create the initial config file. cfg := ` -inputSchema: +schema: schema: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-model-name @@ -50,18 +50,18 @@ rules: - backends: - name: kserve weight: 1 - outputSchema: + schema: schema: OpenAI - name: awsbedrock weight: 10 - outputSchema: + schema: schema: AWSBedrock headers: - name: x-model-name value: llama3.3333 - backends: - name: openai - outputSchema: + schema: schema: OpenAI headers: - name: x-model-name @@ -82,14 +82,14 @@ rules: // Update the config file. cfg = ` -inputSchema: +schema: schema: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-model-name rules: - backends: - name: openai - outputSchema: + schema: schema: OpenAI headers: - name: x-model-name diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml index 71dc591f..2cb24f0d 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml @@ -160,7 +160,7 @@ spec: - kind - name type: object - outputSchema: + schema: description: |- APISchema specifies the API schema of the output format of requests from Envoy that this LLMBackend can accept as incoming requests. @@ -183,7 +183,7 @@ spec: type: object required: - backendRef - - outputSchema + - schema type: object type: object served: true diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml index a8b51ae7..2bab3724 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml @@ -23,7 +23,7 @@ spec: This serves as a way to define a "unified" LLM API for a Gateway which allows downstream clients to use a single schema API to interact with multiple LLM backends. - The inputSchema field is used to determine the structure of the requests that the Gateway will + The schema field is used to determine the structure of the requests that the Gateway will receive. And then the Gateway will route the traffic to the appropriate LLMBackend based on the output schema of the LLMBackend while doing the other necessary jobs like upstream authentication, rate limit, etc. @@ -51,7 +51,7 @@ spec: spec: description: Spec defines the details of the LLM policy. properties: - inputSchema: + schema: description: |- APISchema specifies the API schema of the input that the target Gateway(s) will receive. Based on this schema, the ai-gateway will perform the necessary transformation to the @@ -253,7 +253,7 @@ spec: maxItems: 128 type: array required: - - inputSchema + - schema - rules type: object type: object diff --git a/tests/cel-validation/main_test.go b/tests/cel-validation/main_test.go index bfd75126..0720caa8 100644 --- a/tests/cel-validation/main_test.go +++ b/tests/cel-validation/main_test.go @@ -31,11 +31,11 @@ func TestLLMRoutes(t *testing.T) { {name: "basic.yaml"}, { name: "non_openai_schema.yaml", - expErr: `spec.inputSchema: Invalid value: "object": failed rule: self.schema == 'OpenAI'`, + expErr: `spec.schema: Invalid value: "object": failed rule: self.schema == 'OpenAI'`, }, { name: "unknown_schema.yaml", - expErr: "spec.inputSchema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", + expErr: "spec.schema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", }, { name: "unsupported_match.yaml", @@ -74,7 +74,7 @@ func TestLLMBackends(t *testing.T) { {name: "basic-eg-backend.yaml"}, { name: "unknown_schema.yaml", - expErr: "spec.outputSchema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", + expErr: "spec.schema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml b/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml index e9c5843a..c2ece99b 100644 --- a/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml +++ b/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml @@ -4,7 +4,7 @@ metadata: name: eg-backend namespace: default spec: - outputSchema: + schema: schema: AWSBedrock backendRef: name: eg-backend diff --git a/tests/cel-validation/testdata/llmbackends/basic.yaml b/tests/cel-validation/testdata/llmbackends/basic.yaml index 08739e40..a305ea08 100644 --- a/tests/cel-validation/testdata/llmbackends/basic.yaml +++ b/tests/cel-validation/testdata/llmbackends/basic.yaml @@ -4,7 +4,7 @@ metadata: name: dog-backend namespace: default spec: - outputSchema: + schema: schema: AWSBedrock backendRef: name: dog-service diff --git a/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml b/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml index ea0ac408..f7ddb28a 100644 --- a/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml +++ b/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml @@ -4,6 +4,6 @@ metadata: name: cat-backend namespace: default spec: - outputSchema: + schema: # Schema must be one of the known schemas, so this is invalid. schema: SomeRandomVendor diff --git a/tests/cel-validation/testdata/llmroutes/basic.yaml b/tests/cel-validation/testdata/llmroutes/basic.yaml index e849f685..c85a24e8 100644 --- a/tests/cel-validation/testdata/llmroutes/basic.yaml +++ b/tests/cel-validation/testdata/llmroutes/basic.yaml @@ -4,7 +4,7 @@ metadata: name: apple namespace: default spec: - inputSchema: + schema: schema: OpenAI rules: - matches: diff --git a/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml b/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml index 65084a12..570ba11a 100644 --- a/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml +++ b/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml @@ -4,7 +4,7 @@ metadata: name: apple namespace: default spec: - inputSchema: + schema: # Input must be OpenAI schema at the moment, so this is invalid. schema: AWSBedrock rules: diff --git a/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml b/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml index dfdd1d90..2a360b91 100644 --- a/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml +++ b/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml @@ -4,7 +4,7 @@ metadata: name: apple namespace: default spec: - inputSchema: + schema: # Schema must be OpenAI schema at the moment, so this is invalid. schema: SomeRandomVendor rules: diff --git a/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml b/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml index d7cb103f..5b4e77f7 100644 --- a/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml +++ b/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml @@ -4,7 +4,7 @@ metadata: name: apple namespace: default spec: - inputSchema: + schema: schema: OpenAI rules: - matches: diff --git a/tests/extproc/extproc_test.go b/tests/extproc/extproc_test.go index 6cc743f8..05c3e971 100644 --- a/tests/extproc/extproc_test.go +++ b/tests/extproc/extproc_test.go @@ -51,18 +51,18 @@ func TestE2E(t *testing.T) { Namespace: "ai_gateway_llm_ns", Key: "used_token", }, - InputSchema: openAISchema, + Schema: openAISchema, // This can be any header key, but it must match the envoy.yaml routing configuration. SelectedBackendHeaderKey: "x-selected-backend-name", ModelNameHeaderKey: "x-model-name", Rules: []filterconfig.RouteRule{ { - Backends: []filterconfig.Backend{{Name: "openai", OutputSchema: openAISchema}}, + Backends: []filterconfig.Backend{{Name: "openai", Schema: openAISchema}}, Headers: []filterconfig.HeaderMatch{{Name: "x-model-name", Value: "gpt-4o-mini"}}, }, { Backends: []filterconfig.Backend{ - {Name: "aws-bedrock", OutputSchema: awsBedrockSchema, Auth: &filterconfig.BackendAuth{AWSAuth: &filterconfig.AWSAuth{}}}, + {Name: "aws-bedrock", Schema: awsBedrockSchema, Auth: &filterconfig.BackendAuth{AWSAuth: &filterconfig.AWSAuth{}}}, }, Headers: []filterconfig.HeaderMatch{{Name: "x-model-name", Value: "us.meta.llama3-2-1b-instruct-v1:0"}}, }, From 74498d132bce0367a6095c9ffdeedc88a39f48f8 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 12:20:56 -0500 Subject: [PATCH 02/20] fix typo Signed-off-by: Alexa Griffith --- filterconfig/filterconfig_test.go | 2 +- internal/extproc/router/router.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filterconfig/filterconfig_test.go b/filterconfig/filterconfig_test.go index 3e855dc5..e01c4308 100644 --- a/filterconfig/filterconfig_test.go +++ b/filterconfig/filterconfig_test.go @@ -62,7 +62,7 @@ rules: require.NoError(t, err) require.Equal(t, "ai_gateway_llm_ns", cfg.TokenUsageMetadata.Namespace) require.Equal(t, "token_usage_key", cfg.TokenUsageMetadata.Key) - require.Equal(t, "OpenAI", string(cfg.schema.Schema)) + require.Equal(t, "OpenAI", string(cfg.Schema.Schema)) require.Equal(t, "x-envoy-ai-gateway-selected-backend", cfg.SelectedBackendHeaderKey) require.Equal(t, "x-envoy-ai-gateway-model", cfg.ModelNameHeaderKey) require.Len(t, cfg.Rules, 2) diff --git a/internal/extproc/router/router.go b/internal/extproc/router/router.go index 537ece45..cdcaa6fa 100644 --- a/internal/extproc/router/router.go +++ b/internal/extproc/router/router.go @@ -56,14 +56,14 @@ func (r *router) selectBackendFromRule(rule *filterconfig.RouteRule) (backendNam totalWeight += b.Weight } if totalWeight == 0 { - return rule.Backends[0].Name, rule.Backends[0].schema + return rule.Backends[0].Name, rule.Backends[0].Schema } selected := r.rng.Intn(totalWeight) for _, b := range rule.Backends { if selected < b.Weight { - return b.Name, b.schema + return b.Name, b.Schema } selected -= b.Weight } - return rule.Backends[0].Name, rule.Backends[0].schema + return rule.Backends[0].Name, rule.Backends[0].Schema } From d8b94d025e9540cd3af25adad23618e540e39f71 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 12:45:43 -0500 Subject: [PATCH 03/20] fix lint Signed-off-by: Alexa Griffith --- internal/controller/sink_test.go | 2 +- internal/extproc/server_test.go | 2 +- .../aigateway.envoyproxy.io_llmroutes.yaml | 44 +++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/internal/controller/sink_test.go b/internal/controller/sink_test.go index 5be13c5e..5e37f861 100644 --- a/internal/controller/sink_test.go +++ b/internal/controller/sink_test.go @@ -446,7 +446,7 @@ func Test_updateExtProcConfigMap(t *testing.T) { }, }, exp: &filterconfig.Config{ - Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, + Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, ModelNameHeaderKey: aigv1a1.LLMModelHeaderKey, SelectedBackendHeaderKey: selectedBackendHeaderKey, Rules: []filterconfig.RouteRule{ diff --git a/internal/extproc/server_test.go b/internal/extproc/server_test.go index 464d6f58..bbcaac56 100644 --- a/internal/extproc/server_test.go +++ b/internal/extproc/server_test.go @@ -37,7 +37,7 @@ func TestServer_LoadConfig(t *testing.T) { t.Run("ok", func(t *testing.T) { config := &filterconfig.Config{ TokenUsageMetadata: &filterconfig.TokenUsageMetadata{Namespace: "ns", Key: "key"}, - Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, + Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, SelectedBackendHeaderKey: "x-envoy-ai-gateway-selected-backend", ModelNameHeaderKey: "x-model-name", Rules: []filterconfig.RouteRule{ diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml index 2bab3724..9ff10c87 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml @@ -51,28 +51,6 @@ spec: spec: description: Spec defines the details of the LLM policy. properties: - schema: - description: |- - APISchema specifies the API schema of the input that the target Gateway(s) will receive. - Based on this schema, the ai-gateway will perform the necessary transformation to the - output schema specified in the selected LLMBackend during the routing process. - - Currently, the only supported schema is OpenAI as the input schema. - properties: - schema: - description: Schema is the API schema of the LLMRoute or LLMBackend. - enum: - - OpenAI - - AWSBedrock - type: string - version: - description: Version is the version of the API schema. - type: string - required: - - schema - type: object - x-kubernetes-validations: - - rule: self.schema == 'OpenAI' rules: description: |- Rules is the list of LLMRouteRule that this LLMRoute will match the traffic to. @@ -197,6 +175,28 @@ spec: type: object maxItems: 128 type: array + schema: + description: |- + APISchema specifies the API schema of the input that the target Gateway(s) will receive. + Based on this schema, the ai-gateway will perform the necessary transformation to the + output schema specified in the selected LLMBackend during the routing process. + + Currently, the only supported schema is OpenAI as the input schema. + properties: + schema: + description: Schema is the API schema of the LLMRoute or LLMBackend. + enum: + - OpenAI + - AWSBedrock + type: string + version: + description: Version is the version of the API schema. + type: string + required: + - schema + type: object + x-kubernetes-validations: + - rule: self.schema == 'OpenAI' targetRefs: description: TargetRefs are the names of the Gateway resources this LLMRoute is being attached to. From 7d96b48a7b1119bb21f20077b74855656b2582fd Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 13:08:34 -0500 Subject: [PATCH 04/20] lint Signed-off-by: Alexa Griffith --- .../ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml index 9ff10c87..28870e63 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml @@ -253,8 +253,8 @@ spec: maxItems: 128 type: array required: - - schema - rules + - schema type: object type: object served: true From 2e997d40637a41faba8b72fac270930cb6ef6508 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 15:32:09 -0500 Subject: [PATCH 05/20] change nested schema to name Signed-off-by: Alexa Griffith change nested schema to name in structs Signed-off-by: Alexa Griffith fix factory logic Signed-off-by: Alexa Griffith rebase# This is a combination of 20 commits. fix some other schema names to be consistent Signed-off-by: Alexa Griffith gitignore idea Signed-off-by: Alexa Griffith fix missing name change Signed-off-by: Alexa Griffith fix filterconfig Signed-off-by: Alexa Griffith fix test Signed-off-by: Alexa Griffith renaming Signed-off-by: Alexa Griffith fix Signed-off-by: Alexa Griffith fix sink test naming Signed-off-by: Alexa Griffith fix controller test Signed-off-by: Alexa Griffith add name rule Signed-off-by: Alexa Griffith fix rule Signed-off-by: Alexa Griffith rename klog pkg Signed-off-by: Alexa Griffith fix cel tests Signed-off-by: Alexa Griffith re-add openai pkg Signed-off-by: Alexa Griffith fix llm backend yaml Signed-off-by: Alexa Griffith fix backend Signed-off-by: Alexa Griffith fix the code style Signed-off-by: Alexa Griffith fix order Signed-off-by: Alexa Griffith fix Signed-off-by: Alexa Griffith lint Signed-off-by: Alexa Griffith --- .gitignore | 1 + api/v1alpha1/api.go | 14 ++++++------ filterconfig/filterconfig.go | 22 +++++++++---------- filterconfig/filterconfig_test.go | 14 ++++++------ internal/controller/sink.go | 4 ++-- internal/controller/sink_test.go | 18 +++++++-------- internal/extproc/processor_test.go | 14 ++++++------ internal/extproc/router/request_body.go | 2 +- internal/extproc/router/request_body_test.go | 4 ++-- internal/extproc/router/router_test.go | 4 ++-- internal/extproc/server_test.go | 14 ++++++------ internal/extproc/translator/translator.go | 4 ++-- .../extproc/translator/translator_test.go | 12 +++++----- .../aigateway.envoyproxy.io_llmbackends.yaml | 7 +++--- .../aigateway.envoyproxy.io_llmroutes.yaml | 9 ++++---- tests/cel-validation/main_test.go | 6 ++--- .../llmbackends/basic-eg-backend.yaml | 2 +- .../testdata/llmbackends/basic.yaml | 2 +- .../testdata/llmbackends/unknown_schema.yaml | 4 ++-- .../testdata/llmroutes/basic.yaml | 2 +- .../testdata/llmroutes/non_openai_schema.yaml | 4 ++-- .../testdata/llmroutes/unknown_schema.yaml | 4 ++-- .../testdata/llmroutes/unsupported_match.yaml | 2 +- tests/controller/controller_test.go | 4 ++-- tests/extproc/extproc_test.go | 6 ++--- 25 files changed, 91 insertions(+), 88 deletions(-) diff --git a/.gitignore b/.gitignore index 2c2997cf..012aacfc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ out/ .DS_Store .terraform *for_tests.yaml +.idea/ # Files and directories to ignore in the site directory # dependencies diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go index 6196fcf9..6067ec50 100644 --- a/api/v1alpha1/api.go +++ b/api/v1alpha1/api.go @@ -51,7 +51,7 @@ type LLMRouteSpec struct { // Currently, the only supported schema is OpenAI as the input schema. // // +kubebuilder:validation:Required - // +kubebuilder:validation:XValidation:rule="self.schema == 'OpenAI'" + // +kubebuilder:validation:XValidation:rule="self.name == 'OpenAI'" APISchema LLMAPISchema `json:"schema"` // Rules is the list of LLMRouteRule that this LLMRoute will match the traffic to. // Each rule is a subset of the HTTPRoute in the Gateway API (https://gateway-api.sigs.k8s.io/api-types/httproute/). @@ -183,27 +183,27 @@ type LLMBackendSpec struct { // Note that this is vendor specific, and the stability of the API schema is not guaranteed by // the ai-gateway, but by the vendor via proper versioning. type LLMAPISchema struct { - // Schema is the API schema of the LLMRoute or LLMBackend. + // Name is the name of the API schema of the LLMRoute or LLMBackend. // // +kubebuilder:validation:Enum=OpenAI;AWSBedrock - Schema APISchema `json:"schema"` + Name APISchemaName `json:"name"` // Version is the version of the API schema. Version string `json:"version,omitempty"` } -// APISchema defines the API schema. -type APISchema string +// APISchemaName defines the name of the API schema. +type APISchemaName string const ( // APISchemaOpenAI is the OpenAI schema. // // https://github.com/openai/openai-openapi - APISchemaOpenAI APISchema = "OpenAI" + APISchemaOpenAI APISchemaName = "OpenAI" // APISchemaAWSBedrock is the AWS Bedrock schema. // // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_Operations_Amazon_Bedrock_Runtime.html - APISchemaAWSBedrock APISchema = "AWSBedrock" + APISchemaAWSBedrock APISchemaName = "AWSBedrock" ) const ( diff --git a/filterconfig/filterconfig.go b/filterconfig/filterconfig.go index 87c9024a..51e45561 100644 --- a/filterconfig/filterconfig.go +++ b/filterconfig/filterconfig.go @@ -20,7 +20,7 @@ import ( // fallback when the configuration is not explicitly provided. const DefaultConfig = ` schema: - schema: OpenAI + name: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-envoy-ai-gateway-model ` @@ -30,7 +30,7 @@ modelNameHeaderKey: x-envoy-ai-gateway-model // # Example configuration: // // schema: -// schema: OpenAI +// name: OpenAI // selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend // modelNameHeaderKey: x-envoy-ai-gateway-model // tokenUsageMetadata: @@ -41,18 +41,18 @@ modelNameHeaderKey: x-envoy-ai-gateway-model // - name: kserve // weight: 1 // schema: -// schema: OpenAI +// name: OpenAI // - name: awsbedrock // weight: 10 // schema: -// schema: AWSBedrock +// name: AWSBedrock // headers: // - name: x-envoy-ai-gateway-model // value: llama3.3333 // - backends: // - name: openai // schema: -// schema: OpenAI +// name: OpenAI // headers: // - name: x-envoy-ai-gateway-model // value: gpt4.4444 @@ -96,18 +96,18 @@ type TokenUsageMetadata struct { // VersionedAPISchema corresponds to LLMAPISchema in api/v1alpha1/api.go. type VersionedAPISchema struct { - // Schema is the API schema. - Schema APISchema `yaml:"schema"` + // Name is the name of the API schema. + Name APISchemaName `yaml:"name"` // Version is the version of the API schema. Optional. Version string `yaml:"version,omitempty"` } -// APISchema corresponds to APISchema in api/v1alpha1/api.go. -type APISchema string +// APISchemaName corresponds to APISchemaName in api/v1alpha1/api.go. +type APISchemaName string const ( - APISchemaOpenAI APISchema = "OpenAI" - APISchemaAWSBedrock APISchema = "AWSBedrock" + APISchemaOpenAI APISchemaName = "OpenAI" + APISchemaAWSBedrock APISchemaName = "AWSBedrock" ) // HeaderMatch is an alias for HTTPHeaderMatch of the Gateway API. diff --git a/filterconfig/filterconfig_test.go b/filterconfig/filterconfig_test.go index e01c4308..7d35618d 100644 --- a/filterconfig/filterconfig_test.go +++ b/filterconfig/filterconfig_test.go @@ -30,7 +30,7 @@ func TestUnmarshalConfigYaml(t *testing.T) { configPath := path.Join(t.TempDir(), "config.yaml") const config = ` schema: - schema: OpenAI + name: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-envoy-ai-gateway-model tokenUsageMetadata: @@ -41,18 +41,18 @@ rules: - name: kserve weight: 1 schema: - schema: OpenAI + name: OpenAI - name: awsbedrock weight: 10 schema: - schema: AWSBedrock + name: AWSBedrock headers: - name: x-envoy-ai-gateway-model value: llama3.3333 - backends: - name: openai schema: - schema: OpenAI + name: OpenAI headers: - name: x-envoy-ai-gateway-model value: gpt4.4444 @@ -62,7 +62,7 @@ rules: require.NoError(t, err) require.Equal(t, "ai_gateway_llm_ns", cfg.TokenUsageMetadata.Namespace) require.Equal(t, "token_usage_key", cfg.TokenUsageMetadata.Key) - require.Equal(t, "OpenAI", string(cfg.Schema.Schema)) + require.Equal(t, "OpenAI", string(cfg.Schema.Name)) require.Equal(t, "x-envoy-ai-gateway-selected-backend", cfg.SelectedBackendHeaderKey) require.Equal(t, "x-envoy-ai-gateway-model", cfg.ModelNameHeaderKey) require.Len(t, cfg.Rules, 2) @@ -70,7 +70,7 @@ rules: require.Equal(t, "gpt4.4444", cfg.Rules[1].Headers[0].Value) require.Equal(t, "kserve", cfg.Rules[0].Backends[0].Name) require.Equal(t, 10, cfg.Rules[0].Backends[1].Weight) - require.Equal(t, "AWSBedrock", string(cfg.Rules[0].Backends[1].Schema.Schema)) + require.Equal(t, "AWSBedrock", string(cfg.Rules[0].Backends[1].Schema.Name)) require.Equal(t, "openai", cfg.Rules[1].Backends[0].Name) - require.Equal(t, "OpenAI", string(cfg.Rules[1].Backends[0].Schema.Schema)) + require.Equal(t, "OpenAI", string(cfg.Rules[1].Backends[0].Schema.Name)) } diff --git a/internal/controller/sink.go b/internal/controller/sink.go index 145e44a5..5b45732f 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -226,7 +226,7 @@ func (c *configSink) updateExtProcConfigMap(llmRoute *aigv1a1.LLMRoute) error { ec := &filterconfig.Config{} spec := &llmRoute.Spec - ec.Schema.Schema = filterconfig.APISchema(spec.APISchema.Schema) + ec.Schema.Name = filterconfig.APISchemaName(spec.APISchema.Name) ec.Schema.Version = spec.APISchema.Version ec.ModelNameHeaderKey = aigv1a1.LLMModelHeaderKey ec.SelectedBackendHeaderKey = selectedBackendHeaderKey @@ -242,7 +242,7 @@ func (c *configSink) updateExtProcConfigMap(llmRoute *aigv1a1.LLMRoute) error { err = fmt.Errorf("backend %s not found", key) return err } else { - ec.Rules[i].Backends[j].Schema.Schema = filterconfig.APISchema(backendObj.Spec.APISchema.Schema) + ec.Rules[i].Backends[j].Schema.Name = filterconfig.APISchemaName(backendObj.Spec.APISchema.Name) ec.Rules[i].Backends[j].Schema.Version = backendObj.Spec.APISchema.Version } } diff --git a/internal/controller/sink_test.go b/internal/controller/sink_test.go index 5e37f861..d8d8e3c7 100644 --- a/internal/controller/sink_test.go +++ b/internal/controller/sink_test.go @@ -78,7 +78,7 @@ func TestConfigSink_init(t *testing.T) { BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend1", Namespace: ptr.To[gwapiv1.Namespace]("ns1")}, }, - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI}, }, }, { @@ -87,7 +87,7 @@ func TestConfigSink_init(t *testing.T) { BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend2", Namespace: ptr.To[gwapiv1.Namespace]("ns1")}, }, - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaAWSBedrock}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaAWSBedrock}, }, }, { @@ -96,7 +96,7 @@ func TestConfigSink_init(t *testing.T) { BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend3", Namespace: ptr.To[gwapiv1.Namespace]("ns1")}, }, - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI}, }, }, { @@ -105,7 +105,7 @@ func TestConfigSink_init(t *testing.T) { BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend4", Namespace: ptr.To[gwapiv1.Namespace]("ns1")}, }, - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI}, }, }, } { @@ -212,7 +212,7 @@ func TestConfigSink_syncLLMRoute(t *testing.T) { BackendRefs: []aigv1a1.LLMRouteRuleBackendRef{{Name: "apple", Weight: 1}, {Name: "orange", Weight: 1}}, }, }, - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v123"}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v123"}, }, } err := fakeClient.Create(context.Background(), route, &client.CreateOptions{}) @@ -389,7 +389,7 @@ func Test_updateExtProcConfigMap(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "apple", Namespace: "ns"}, Spec: aigv1a1.LLMBackendSpec{ APISchema: aigv1a1.LLMAPISchema{ - Schema: aigv1a1.APISchemaAWSBedrock, + Name: aigv1a1.APISchemaAWSBedrock, }, BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend1", Namespace: ptr.To[gwapiv1.Namespace]("ns")}, @@ -425,7 +425,7 @@ func Test_updateExtProcConfigMap(t *testing.T) { route: &aigv1a1.LLMRoute{ ObjectMeta: metav1.ObjectMeta{Name: "myroute", Namespace: "ns"}, Spec: aigv1a1.LLMRouteSpec{ - APISchema: aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v123"}, + APISchema: aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v123"}, Rules: []aigv1a1.LLMRouteRule{ { BackendRefs: []aigv1a1.LLMRouteRuleBackendRef{ @@ -446,13 +446,13 @@ func Test_updateExtProcConfigMap(t *testing.T) { }, }, exp: &filterconfig.Config{ - Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, + Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI, Version: "v123"}, ModelNameHeaderKey: aigv1a1.LLMModelHeaderKey, SelectedBackendHeaderKey: selectedBackendHeaderKey, Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "apple.ns", Weight: 1, Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, + {Name: "apple.ns", Weight: 1, Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, }, Headers: []filterconfig.HeaderMatch{{Name: aigv1a1.LLMModelHeaderKey, Value: "some-ai"}}, }, diff --git a/internal/extproc/processor_test.go b/internal/extproc/processor_test.go index 5363797b..f21f7586 100644 --- a/internal/extproc/processor_test.go +++ b/internal/extproc/processor_test.go @@ -95,7 +95,7 @@ func TestProcessor_ProcessRequestBody(t *testing.T) { rbp := mockRequestBodyParser{t: t, retModelName: "some-model", expPath: "/foo"} rt := mockRouter{ t: t, expHeaders: headers, retBackendName: "some-backend", - retVersionedAPISchema: filterconfig.VersionedAPISchema{Schema: "some-schema", Version: "v10.0"}, + retVersionedAPISchema: filterconfig.VersionedAPISchema{Name: "some-schema", Version: "v10.0"}, } p := &Processor{config: &processorConfig{ bodyParser: rbp.impl, router: rt, @@ -109,13 +109,13 @@ func TestProcessor_ProcessRequestBody(t *testing.T) { rbp := mockRequestBodyParser{t: t, retModelName: "some-model", expPath: "/foo"} rt := mockRouter{ t: t, expHeaders: headers, retBackendName: "some-backend", - retVersionedAPISchema: filterconfig.VersionedAPISchema{Schema: "some-schema", Version: "v10.0"}, + retVersionedAPISchema: filterconfig.VersionedAPISchema{Name: "some-schema", Version: "v10.0"}, } factory := mockTranslatorFactory{t: t, retErr: errors.New("test error"), expPath: "/foo"} p := &Processor{config: &processorConfig{ bodyParser: rbp.impl, router: rt, factories: map[filterconfig.VersionedAPISchema]translator.Factory{ - {Schema: "some-schema", Version: "v10.0"}: factory.impl, + {Name: "some-schema", Version: "v10.0"}: factory.impl, }, }, requestHeaders: headers} _, err := p.ProcessRequestBody(context.Background(), &extprocv3.HttpBody{}) @@ -126,13 +126,13 @@ func TestProcessor_ProcessRequestBody(t *testing.T) { rbp := mockRequestBodyParser{t: t, retModelName: "some-model", expPath: "/foo"} rt := mockRouter{ t: t, expHeaders: headers, retBackendName: "some-backend", - retVersionedAPISchema: filterconfig.VersionedAPISchema{Schema: "some-schema", Version: "v10.0"}, + retVersionedAPISchema: filterconfig.VersionedAPISchema{Name: "some-schema", Version: "v10.0"}, } factory := mockTranslatorFactory{t: t, retTranslator: mockTranslator{t: t, retErr: errors.New("test error")}, expPath: "/foo"} p := &Processor{config: &processorConfig{ bodyParser: rbp.impl, router: rt, factories: map[filterconfig.VersionedAPISchema]translator.Factory{ - {Schema: "some-schema", Version: "v10.0"}: factory.impl, + {Name: "some-schema", Version: "v10.0"}: factory.impl, }, }, requestHeaders: headers} _, err := p.ProcessRequestBody(context.Background(), &extprocv3.HttpBody{}) @@ -144,7 +144,7 @@ func TestProcessor_ProcessRequestBody(t *testing.T) { rbp := mockRequestBodyParser{t: t, retModelName: "some-model", expPath: "/foo", retRb: someBody} rt := mockRouter{ t: t, expHeaders: headers, retBackendName: "some-backend", - retVersionedAPISchema: filterconfig.VersionedAPISchema{Schema: "some-schema", Version: "v10.0"}, + retVersionedAPISchema: filterconfig.VersionedAPISchema{Name: "some-schema", Version: "v10.0"}, } headerMut := &extprocv3.HeaderMutation{} bodyMut := &extprocv3.BodyMutation{} @@ -153,7 +153,7 @@ func TestProcessor_ProcessRequestBody(t *testing.T) { p := &Processor{config: &processorConfig{ bodyParser: rbp.impl, router: rt, factories: map[filterconfig.VersionedAPISchema]translator.Factory{ - {Schema: "some-schema", Version: "v10.0"}: factory.impl, + {Name: "some-schema", Version: "v10.0"}: factory.impl, }, selectedBackendHeaderKey: "x-ai-gateway-backend-key", ModelNameHeaderKey: "x-ai-gateway-model-key", diff --git a/internal/extproc/router/request_body.go b/internal/extproc/router/request_body.go index 91661420..d1361fac 100644 --- a/internal/extproc/router/request_body.go +++ b/internal/extproc/router/request_body.go @@ -15,7 +15,7 @@ type RequestBodyParser func(path string, body *extprocv3.HttpBody) (modelName st // NewRequestBodyParser creates a new RequestBodyParser based on the schema. func NewRequestBodyParser(schema filterconfig.VersionedAPISchema) (RequestBodyParser, error) { - if schema.Schema == filterconfig.APISchemaOpenAI { + if schema.Name == filterconfig.APISchemaOpenAI { return openAIParseBody, nil } return nil, fmt.Errorf("unsupported API schema: %s", schema) diff --git a/internal/extproc/router/request_body_test.go b/internal/extproc/router/request_body_test.go index d5e4049e..e42469e9 100644 --- a/internal/extproc/router/request_body_test.go +++ b/internal/extproc/router/request_body_test.go @@ -13,12 +13,12 @@ import ( func TestNewRequestBodyParser(t *testing.T) { t.Run("ok", func(t *testing.T) { - res, err := NewRequestBodyParser(filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}) + res, err := NewRequestBodyParser(filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}) require.NotNil(t, res) require.NoError(t, err) }) t.Run("error", func(t *testing.T) { - res, err := NewRequestBodyParser(filterconfig.VersionedAPISchema{Schema: "foo"}) + res, err := NewRequestBodyParser(filterconfig.VersionedAPISchema{Name: "foo"}) require.Nil(t, res) require.Error(t, err) }) diff --git a/internal/extproc/router/router_test.go b/internal/extproc/router/router_test.go index cc56a0b9..5d0095bd 100644 --- a/internal/extproc/router/router_test.go +++ b/internal/extproc/router/router_test.go @@ -9,7 +9,7 @@ import ( ) func TestRouter_Calculate(t *testing.T) { - outSchema := filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI} + outSchema := filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI} _r, err := NewRouter(&filterconfig.Config{ Rules: []filterconfig.RouteRule{ { @@ -68,7 +68,7 @@ func TestRouter_selectBackendFromRule(t *testing.T) { r, ok := _r.(*router) require.True(t, ok) - outSchema := filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI} + outSchema := filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI} rule := &filterconfig.RouteRule{ Backends: []filterconfig.Backend{ diff --git a/internal/extproc/server_test.go b/internal/extproc/server_test.go index bbcaac56..4c352ca0 100644 --- a/internal/extproc/server_test.go +++ b/internal/extproc/server_test.go @@ -29,7 +29,7 @@ func TestServer_LoadConfig(t *testing.T) { t.Run("invalid input schema", func(t *testing.T) { s := requireNewServerWithMockProcessor(t) err := s.LoadConfig(&filterconfig.Config{ - Schema: filterconfig.VersionedAPISchema{Schema: "some-invalid-schema"}, + Schema: filterconfig.VersionedAPISchema{Name: "some-invalid-schema"}, }) require.Error(t, err) require.ErrorContains(t, err, "cannot create request body parser") @@ -37,14 +37,14 @@ func TestServer_LoadConfig(t *testing.T) { t.Run("ok", func(t *testing.T) { config := &filterconfig.Config{ TokenUsageMetadata: &filterconfig.TokenUsageMetadata{Namespace: "ns", Key: "key"}, - Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, + Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}, SelectedBackendHeaderKey: "x-envoy-ai-gateway-selected-backend", ModelNameHeaderKey: "x-model-name", Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "kserve", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, - {Name: "awsbedrock", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, + {Name: "kserve", Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}}, + {Name: "awsbedrock", Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock}}, }, Headers: []filterconfig.HeaderMatch{ { @@ -55,7 +55,7 @@ func TestServer_LoadConfig(t *testing.T) { }, { Backends: []filterconfig.Backend{ - {Name: "openai", Schema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}}, + {Name: "openai", Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}}, }, Headers: []filterconfig.HeaderMatch{ { @@ -79,8 +79,8 @@ func TestServer_LoadConfig(t *testing.T) { require.Equal(t, "x-envoy-ai-gateway-selected-backend", s.config.selectedBackendHeaderKey) require.Equal(t, "x-model-name", s.config.ModelNameHeaderKey) require.Len(t, s.config.factories, 2) - require.NotNil(t, s.config.factories[filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}]) - require.NotNil(t, s.config.factories[filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}]) + require.NotNil(t, s.config.factories[filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}]) + require.NotNil(t, s.config.factories[filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock}]) }) } diff --git a/internal/extproc/translator/translator.go b/internal/extproc/translator/translator.go index 98b8095d..84265dbc 100644 --- a/internal/extproc/translator/translator.go +++ b/internal/extproc/translator/translator.go @@ -20,9 +20,9 @@ type Factory func(path string) (Translator, error) // NewFactory returns a callback function that creates a translator for the given API schema combination. func NewFactory(in, out filterconfig.VersionedAPISchema) (Factory, error) { - if in.Schema == filterconfig.APISchemaOpenAI { + if in.Name == filterconfig.APISchemaOpenAI { // TODO: currently, we ignore the LLMAPISchema."Version" field. - switch out.Schema { + switch out.Name { case filterconfig.APISchemaOpenAI: return newOpenAIToOpenAITranslator, nil case filterconfig.APISchemaAWSBedrock: diff --git a/internal/extproc/translator/translator_test.go b/internal/extproc/translator/translator_test.go index f14b8f71..64c42023 100644 --- a/internal/extproc/translator/translator_test.go +++ b/internal/extproc/translator/translator_test.go @@ -11,15 +11,15 @@ import ( func TestNewFactory(t *testing.T) { t.Run("error", func(t *testing.T) { _, err := NewFactory( - filterconfig.VersionedAPISchema{Schema: "Foo", Version: "v100"}, - filterconfig.VersionedAPISchema{Schema: "Bar", Version: "v123"}, + filterconfig.VersionedAPISchema{Name: "Foo", Version: "v100"}, + filterconfig.VersionedAPISchema{Name: "Bar", Version: "v123"}, ) require.ErrorContains(t, err, "unsupported API schema combination: client={Foo v100}, backend={Bar v123}") }) t.Run("openai to openai", func(t *testing.T) { f, err := NewFactory( - filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, - filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, + filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}, + filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}, ) require.NoError(t, err) require.NotNil(t, f) @@ -32,8 +32,8 @@ func TestNewFactory(t *testing.T) { }) t.Run("openai to aws bedrock", func(t *testing.T) { f, err := NewFactory( - filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI}, - filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}, + filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI}, + filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock}, ) require.NoError(t, err) require.NotNil(t, f) diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml index 2cb24f0d..02d9aca4 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmbackends.yaml @@ -169,8 +169,9 @@ spec: This is required to be set. properties: - schema: - description: Schema is the API schema of the LLMRoute or LLMBackend. + name: + description: Name is the name of the API schema of the LLMRoute + or LLMBackend. enum: - OpenAI - AWSBedrock @@ -179,7 +180,7 @@ spec: description: Version is the version of the API schema. type: string required: - - schema + - name type: object required: - backendRef diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml index 28870e63..0ddea5da 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_llmroutes.yaml @@ -183,8 +183,9 @@ spec: Currently, the only supported schema is OpenAI as the input schema. properties: - schema: - description: Schema is the API schema of the LLMRoute or LLMBackend. + name: + description: Name is the name of the API schema of the LLMRoute + or LLMBackend. enum: - OpenAI - AWSBedrock @@ -193,10 +194,10 @@ spec: description: Version is the version of the API schema. type: string required: - - schema + - name type: object x-kubernetes-validations: - - rule: self.schema == 'OpenAI' + - rule: self.name == 'OpenAI' targetRefs: description: TargetRefs are the names of the Gateway resources this LLMRoute is being attached to. diff --git a/tests/cel-validation/main_test.go b/tests/cel-validation/main_test.go index 0720caa8..5938be71 100644 --- a/tests/cel-validation/main_test.go +++ b/tests/cel-validation/main_test.go @@ -31,11 +31,11 @@ func TestLLMRoutes(t *testing.T) { {name: "basic.yaml"}, { name: "non_openai_schema.yaml", - expErr: `spec.schema: Invalid value: "object": failed rule: self.schema == 'OpenAI'`, + expErr: `spec.schema: Invalid value: "object": failed rule: self.name == 'OpenAI'`, }, { name: "unknown_schema.yaml", - expErr: "spec.schema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", + expErr: "spec.schema.name: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", }, { name: "unsupported_match.yaml", @@ -74,7 +74,7 @@ func TestLLMBackends(t *testing.T) { {name: "basic-eg-backend.yaml"}, { name: "unknown_schema.yaml", - expErr: "spec.schema.schema: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", + expErr: "spec.schema.name: Unsupported value: \"SomeRandomVendor\": supported values: \"OpenAI\", \"AWSBedrock\"", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml b/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml index c2ece99b..3899ae86 100644 --- a/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml +++ b/tests/cel-validation/testdata/llmbackends/basic-eg-backend.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: schema: - schema: AWSBedrock + name: AWSBedrock backendRef: name: eg-backend kind: Backend diff --git a/tests/cel-validation/testdata/llmbackends/basic.yaml b/tests/cel-validation/testdata/llmbackends/basic.yaml index a305ea08..2eb84a72 100644 --- a/tests/cel-validation/testdata/llmbackends/basic.yaml +++ b/tests/cel-validation/testdata/llmbackends/basic.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: schema: - schema: AWSBedrock + name: AWSBedrock backendRef: name: dog-service kind: Service diff --git a/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml b/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml index f7ddb28a..ff10bc3f 100644 --- a/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml +++ b/tests/cel-validation/testdata/llmbackends/unknown_schema.yaml @@ -5,5 +5,5 @@ metadata: namespace: default spec: schema: - # Schema must be one of the known schemas, so this is invalid. - schema: SomeRandomVendor + # Name must be one of the known schemas, so this is invalid. + name: SomeRandomVendor diff --git a/tests/cel-validation/testdata/llmroutes/basic.yaml b/tests/cel-validation/testdata/llmroutes/basic.yaml index c85a24e8..776f8dac 100644 --- a/tests/cel-validation/testdata/llmroutes/basic.yaml +++ b/tests/cel-validation/testdata/llmroutes/basic.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: schema: - schema: OpenAI + name: OpenAI rules: - matches: - headers: diff --git a/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml b/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml index 570ba11a..95823d80 100644 --- a/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml +++ b/tests/cel-validation/testdata/llmroutes/non_openai_schema.yaml @@ -5,8 +5,8 @@ metadata: namespace: default spec: schema: - # Input must be OpenAI schema at the moment, so this is invalid. - schema: AWSBedrock + # Schema name must be OpenAI schema at the moment, so this is invalid. + name: AWSBedrock rules: - matches: - headers: diff --git a/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml b/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml index 2a360b91..6b0d9cf5 100644 --- a/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml +++ b/tests/cel-validation/testdata/llmroutes/unknown_schema.yaml @@ -5,8 +5,8 @@ metadata: namespace: default spec: schema: - # Schema must be OpenAI schema at the moment, so this is invalid. - schema: SomeRandomVendor + # Schema name must be OpenAI schema at the moment, so this is invalid. + name: SomeRandomVendor rules: - matches: - headers: diff --git a/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml b/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml index 5b4e77f7..e93abbfc 100644 --- a/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml +++ b/tests/cel-validation/testdata/llmroutes/unsupported_match.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: schema: - schema: OpenAI + name: OpenAI rules: - matches: - headers: diff --git a/tests/controller/controller_test.go b/tests/controller/controller_test.go index 9072ef9f..72f30c51 100644 --- a/tests/controller/controller_test.go +++ b/tests/controller/controller_test.go @@ -17,7 +17,7 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" + klog "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,7 +30,7 @@ import ( "github.com/envoyproxy/ai-gateway/tests" ) -var defaultSchema = aigv1a1.LLMAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v1"} +var defaultSchema = aigv1a1.LLMAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v1"} func extProcName(llmRouteName string) string { return fmt.Sprintf("ai-gateway-llm-route-extproc-%s", llmRouteName) diff --git a/tests/extproc/extproc_test.go b/tests/extproc/extproc_test.go index 05c3e971..673205d8 100644 --- a/tests/extproc/extproc_test.go +++ b/tests/extproc/extproc_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/openai/openai-go" + openai "github.com/openai/openai-go" "github.com/openai/openai-go/option" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" @@ -29,8 +29,8 @@ import ( var envoyYamlBase string var ( - openAISchema = filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI} - awsBedrockSchema = filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock} + openAISchema = filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI} + awsBedrockSchema = filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock} ) // TestE2E tests the end-to-end flow of the external processor with Envoy. From 6071621f5a574572b9d503c3ff842f5536dfcdcc Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Wed, 15 Jan 2025 18:39:20 -0500 Subject: [PATCH 06/20] revert bedrock udpate Signed-off-by: Alexa Griffith --- internal/apischema/awsbedrock/awsbedrock.go | 4 ++-- internal/extproc/translator/openai_awsbedrock.go | 2 +- internal/extproc/translator/openai_awsbedrock_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/apischema/awsbedrock/awsbedrock.go b/internal/apischema/awsbedrock/awsbedrock.go index 203fcdfc..3f42465c 100644 --- a/internal/apischema/awsbedrock/awsbedrock.go +++ b/internal/apischema/awsbedrock/awsbedrock.go @@ -460,8 +460,8 @@ type ToolSpecification struct { // The schema for the tool in JSON format. // - // Schema is a required field - Schema *ToolSchema `json:"schema"` + // InputSchema is a required field + InputSchema *ToolSchema `json:"inputSchema"` // The name for the tool. // diff --git a/internal/extproc/translator/openai_awsbedrock.go b/internal/extproc/translator/openai_awsbedrock.go index 5bc7704e..440a39c4 100644 --- a/internal/extproc/translator/openai_awsbedrock.go +++ b/internal/extproc/translator/openai_awsbedrock.go @@ -116,7 +116,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIToolsToBedrockToolC ToolSpec: &awsbedrock.ToolSpecification{ Name: &toolType, Description: &toolDefinition.Function.Description, - Schema: &awsbedrock.ToolSchema{ + InputSchema: &awsbedrock.ToolSchema{ JSON: toolDefinition.Function.Parameters, }, }, diff --git a/internal/extproc/translator/openai_awsbedrock_test.go b/internal/extproc/translator/openai_awsbedrock_test.go index 70c8badb..3d56d423 100644 --- a/internal/extproc/translator/openai_awsbedrock_test.go +++ b/internal/extproc/translator/openai_awsbedrock_test.go @@ -350,7 +350,7 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T) ToolSpec: &awsbedrock.ToolSpecification{ Name: ptr.To("function"), Description: ptr.To("Get the current weather in a given location"), - Schema: &awsbedrock.ToolSchema{ + InputSchema: &awsbedrock.ToolSchema{ JSON: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ From 297b3ddeb2d15de2e4559c657d483cc410654cb6 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 10:23:04 -0500 Subject: [PATCH 07/20] revert tool input schema naming Signed-off-by: Alexa Griffith --- internal/apischema/awsbedrock/awsbedrock.go | 6 +++--- internal/extproc/translator/openai_awsbedrock.go | 2 +- internal/extproc/translator/openai_awsbedrock_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/apischema/awsbedrock/awsbedrock.go b/internal/apischema/awsbedrock/awsbedrock.go index 3f42465c..affe2845 100644 --- a/internal/apischema/awsbedrock/awsbedrock.go +++ b/internal/apischema/awsbedrock/awsbedrock.go @@ -448,8 +448,8 @@ type Tool struct { ToolSpec *ToolSpecification `json:"toolSpec"` } -// ToolSchema The schema for the tool. The top level schema type must be an object. -type ToolSchema struct { +// ToolInputSchema The schema for the tool. The top level schema type must be an object. +type ToolInputSchema struct { JSON any `json:"json"` } @@ -461,7 +461,7 @@ type ToolSpecification struct { // The schema for the tool in JSON format. // // InputSchema is a required field - InputSchema *ToolSchema `json:"inputSchema"` + InputSchema *ToolInputSchema `json:"inputSchema"` // The name for the tool. // diff --git a/internal/extproc/translator/openai_awsbedrock.go b/internal/extproc/translator/openai_awsbedrock.go index 440a39c4..ce2cc70b 100644 --- a/internal/extproc/translator/openai_awsbedrock.go +++ b/internal/extproc/translator/openai_awsbedrock.go @@ -116,7 +116,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIToolsToBedrockToolC ToolSpec: &awsbedrock.ToolSpecification{ Name: &toolType, Description: &toolDefinition.Function.Description, - InputSchema: &awsbedrock.ToolSchema{ + InputSchema: &awsbedrock.ToolInputSchema{ JSON: toolDefinition.Function.Parameters, }, }, diff --git a/internal/extproc/translator/openai_awsbedrock_test.go b/internal/extproc/translator/openai_awsbedrock_test.go index 3d56d423..f4bcc25f 100644 --- a/internal/extproc/translator/openai_awsbedrock_test.go +++ b/internal/extproc/translator/openai_awsbedrock_test.go @@ -350,7 +350,7 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T) ToolSpec: &awsbedrock.ToolSpecification{ Name: ptr.To("function"), Description: ptr.To("Get the current weather in a given location"), - InputSchema: &awsbedrock.ToolSchema{ + InputSchema: &awsbedrock.ToolInputSchema{ JSON: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ From 8f8b7ca4d269924d49e1f4bb0b8b0dff053b50ca Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 17:58:04 -0500 Subject: [PATCH 08/20] fix merge issues Signed-off-by: Alexa Griffith --- internal/controller/sink.go | 8 ++++---- internal/controller/sink_test.go | 4 ++-- internal/extproc/mocks_test.go | 2 +- internal/extproc/processor.go | 13 +++++++------ 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/controller/sink.go b/internal/controller/sink.go index 4d85b918..86a3acc6 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -157,8 +157,8 @@ func (c *configSink) updateExtProcConfigMap(aiGatewayRoute *aigv1a1.AIGatewayRou ec := &filterconfig.Config{} spec := &aiGatewayRoute.Spec - ec.InputSchema.Schema = filterconfig.APISchema(spec.APISchema.Schema) - ec.InputSchema.Version = spec.APISchema.Version + ec.Schema.Name = filterconfig.APISchemaName(spec.APISchema.Schema) + ec.Schema.Version = spec.APISchema.Version ec.ModelNameHeaderKey = aigv1a1.AIModelHeaderKey ec.SelectedBackendHeaderKey = selectedBackendHeaderKey ec.Rules = make([]filterconfig.RouteRule, len(spec.Rules)) @@ -172,8 +172,8 @@ func (c *configSink) updateExtProcConfigMap(aiGatewayRoute *aigv1a1.AIGatewayRou if err != nil { return fmt.Errorf("failed to get AIServiceBackend %s: %w", key, err) } else { - ec.Rules[i].Backends[j].OutputSchema.Schema = filterconfig.APISchema(backendObj.Spec.APISchema.Schema) - ec.Rules[i].Backends[j].OutputSchema.Version = backendObj.Spec.APISchema.Version + ec.Rules[i].Backends[j].Schema.Name = filterconfig.APISchemaName(backendObj.Spec.APISchema.Schema) + ec.Rules[i].Backends[j].Schema.Version = backendObj.Spec.APISchema.Version } } ec.Rules[i].Headers = make([]filterconfig.HeaderMatch, len(rule.Matches)) diff --git a/internal/controller/sink_test.go b/internal/controller/sink_test.go index ceb322f1..568e559a 100644 --- a/internal/controller/sink_test.go +++ b/internal/controller/sink_test.go @@ -276,13 +276,13 @@ func Test_updateExtProcConfigMap(t *testing.T) { }, }, exp: &filterconfig.Config{ - InputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaOpenAI, Version: "v123"}, + Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaOpenAI, Version: "v123"}, ModelNameHeaderKey: aigv1a1.AIModelHeaderKey, SelectedBackendHeaderKey: selectedBackendHeaderKey, Rules: []filterconfig.RouteRule{ { Backends: []filterconfig.Backend{ - {Name: "apple.ns", Weight: 1, OutputSchema: filterconfig.VersionedAPISchema{Schema: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, + {Name: "apple.ns", Weight: 1, Schema: filterconfig.VersionedAPISchema{Name: filterconfig.APISchemaAWSBedrock}}, {Name: "pineapple.ns", Weight: 2}, }, Headers: []filterconfig.HeaderMatch{{Name: aigv1a1.AIModelHeaderKey, Value: "some-ai"}}, }, diff --git a/internal/extproc/mocks_test.go b/internal/extproc/mocks_test.go index 7c3666f7..7936c827 100644 --- a/internal/extproc/mocks_test.go +++ b/internal/extproc/mocks_test.go @@ -107,7 +107,7 @@ type mockRouter struct { // Calculate implements [router.Router.Calculate]. func (m mockRouter) Calculate(headers map[string]string) (*filterconfig.Backend, error) { require.Equal(m.t, m.expHeaders, headers) - b := &filterconfig.Backend{Name: m.retBackendName, OutputSchema: m.retVersionedAPISchema} + b := &filterconfig.Backend{Name: m.retBackendName, Schema: m.retVersionedAPISchema} return b, m.retErr } diff --git a/internal/extproc/processor.go b/internal/extproc/processor.go index 0320a2a2..faa3daa3 100644 --- a/internal/extproc/processor.go +++ b/internal/extproc/processor.go @@ -12,6 +12,7 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" "google.golang.org/protobuf/types/known/structpb" + "github.com/envoyproxy/ai-gateway/extprocapi" "github.com/envoyproxy/ai-gateway/filterconfig" "github.com/envoyproxy/ai-gateway/internal/extproc/backendauth" "github.com/envoyproxy/ai-gateway/internal/extproc/router" @@ -22,7 +23,7 @@ import ( // This will be created by the server and passed to the processor when it detects a new configuration. type processorConfig struct { bodyParser router.RequestBodyParser - router router.Router + router extprocapi.Router ModelNameHeaderKey, selectedBackendHeaderKey string factories map[filterconfig.VersionedAPISchema]translator.Factory backendAuthHandlers map[string]backendauth.Handler @@ -73,14 +74,14 @@ func (p *Processor) ProcessRequestBody(_ context.Context, rawBody *extprocv3.Htt } p.requestHeaders[p.config.ModelNameHeaderKey] = model - backendName, schema, err := p.config.router.Calculate(p.requestHeaders) + b, err := p.config.router.Calculate(p.requestHeaders) if err != nil { return nil, fmt.Errorf("failed to calculate route: %w", err) } - factory, ok := p.config.factories[schema] + factory, ok := p.config.factories[b.Schema] if !ok { - return nil, fmt.Errorf("failed to find factory for output schema %q", schema) + return nil, fmt.Errorf("failed to find factory for output schema %q", b.Schema) } t, err := factory(path) @@ -101,10 +102,10 @@ func (p *Processor) ProcessRequestBody(_ context.Context, rawBody *extprocv3.Htt headerMutation.SetHeaders = append(headerMutation.SetHeaders, &corev3.HeaderValueOption{ Header: &corev3.HeaderValue{Key: p.config.ModelNameHeaderKey, RawValue: []byte(model)}, }, &corev3.HeaderValueOption{ - Header: &corev3.HeaderValue{Key: p.config.selectedBackendHeaderKey, RawValue: []byte(backendName)}, + Header: &corev3.HeaderValue{Key: p.config.selectedBackendHeaderKey, RawValue: []byte(b.Name)}, }) - if authHandler, ok := p.config.backendAuthHandlers[backendName]; ok { + if authHandler, ok := p.config.backendAuthHandlers[b.Name]; ok { if err := authHandler.Do(p.requestHeaders, headerMutation, bodyMutation); err != nil { return nil, fmt.Errorf("failed to do auth request: %w", err) } From c61a6fc341eb984a57863d1623f0df9a31bb8f51 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:02:39 -0500 Subject: [PATCH 09/20] merge fix Signed-off-by: Alexa Griffith --- api/v1alpha1/api.go | 6 +++--- tests/extproc/custom_extproc_test.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go index 9e05b36d..2cd81fa8 100644 --- a/api/v1alpha1/api.go +++ b/api/v1alpha1/api.go @@ -15,7 +15,7 @@ import ( // This serves as a way to define a "unified" AI API for a Gateway which allows downstream // clients to use a single schema API to interact with multiple AI backends. // -// The inputSchema field is used to determine the structure of the requests that the Gateway will +// The schema field is used to determine the structure of the requests that the Gateway will // receive. And then the Gateway will route the traffic to the appropriate AIServiceBackend based // on the output schema of the AIServiceBackend while doing the other necessary jobs like // upstream authentication, rate limit, etc. @@ -53,7 +53,7 @@ type AIGatewayRouteSpec struct { // // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self.schema == 'OpenAI'" - APISchema VersionedAPISchema `json:"inputSchema"` + APISchema VersionedAPISchema `json:"schema"` // Rules is the list of AIGatewayRouteRule that this AIGatewayRoute will match the traffic to. // Each rule is a subset of the HTTPRoute in the Gateway API (https://gateway-api.sigs.k8s.io/api-types/httproute/). // @@ -215,7 +215,7 @@ type AIServiceBackendSpec struct { // This is required to be set. // // +kubebuilder:validation:Required - APISchema VersionedAPISchema `json:"outputSchema"` + APISchema VersionedAPISchema `json:"schema"` // BackendRef is the reference to the Backend resource that this AIServiceBackend corresponds to. // // A backend can be of either k8s Service or Backend resource of Envoy Gateway. diff --git a/tests/extproc/custom_extproc_test.go b/tests/extproc/custom_extproc_test.go index dc187069..b9dc2779 100644 --- a/tests/extproc/custom_extproc_test.go +++ b/tests/extproc/custom_extproc_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/openai/openai-go" "github.com/openai/openai-go/option" "github.com/stretchr/testify/require" @@ -25,13 +24,13 @@ func TestExtProcCustomRouter(t *testing.T) { requireTestUpstream(t) configPath := t.TempDir() + "/extproc-config.yaml" requireWriteExtProcConfig(t, configPath, &filterconfig.Config{ - InputSchema: openAISchema, + schema: openAISchema, // This can be any header key, but it must match the envoy.yaml routing configuration. SelectedBackendHeaderKey: "x-selected-backend-name", ModelNameHeaderKey: "x-model-name", Rules: []filterconfig.RouteRule{ { - Backends: []filterconfig.Backend{{Name: "testupstream", OutputSchema: openAISchema}}, + Backends: []filterconfig.Backend{{Name: "testupstream", Schema: openAISchema}}, Headers: []filterconfig.HeaderMatch{{Name: "x-model-name", Value: "something-cool"}}, }, }, From 31cfbeb3936b03d7dfd6303bf2b37fd6405c85f7 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:06:25 -0500 Subject: [PATCH 10/20] fix self.schema to self.name Signed-off-by: Alexa Griffith --- api/v1alpha1/api.go | 2 +- .../crds/aigateway.envoyproxy.io_aigatewayroutes.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go index 2cd81fa8..a7301d28 100644 --- a/api/v1alpha1/api.go +++ b/api/v1alpha1/api.go @@ -52,7 +52,7 @@ type AIGatewayRouteSpec struct { // Currently, the only supported schema is OpenAI as the input schema. // // +kubebuilder:validation:Required - // +kubebuilder:validation:XValidation:rule="self.schema == 'OpenAI'" + // +kubebuilder:validation:XValidation:rule="self.name == 'OpenAI'" APISchema VersionedAPISchema `json:"schema"` // Rules is the list of AIGatewayRouteRule that this AIGatewayRoute will match the traffic to. // Each rule is a subset of the HTTPRoute in the Gateway API (https://gateway-api.sigs.k8s.io/api-types/httproute/). diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml index 1b804eb5..90a89832 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml @@ -175,7 +175,7 @@ spec: - schema type: object x-kubernetes-validations: - - rule: self.schema == 'OpenAI' + - rule: self.name == 'OpenAI' rules: description: |- Rules is the list of AIGatewayRouteRule that this AIGatewayRoute will match the traffic to. From 24975bd148df57d48f3de44e66fa4cfdafc57531 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:08:38 -0500 Subject: [PATCH 11/20] fix merge Signed-off-by: Alexa Griffith --- tests/extproc/custom_extproc_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/extproc/custom_extproc_test.go b/tests/extproc/custom_extproc_test.go index b9dc2779..a22e174c 100644 --- a/tests/extproc/custom_extproc_test.go +++ b/tests/extproc/custom_extproc_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + openai "github.com/openai/openai-go" "github.com/openai/openai-go/option" "github.com/stretchr/testify/require" From ea404665ceb297b079badedf538eb59b8a3c2087 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:25:51 -0500 Subject: [PATCH 12/20] fix ext proc and cel test Signed-off-by: Alexa Griffith --- api/v1alpha1/api.go | 4 +- internal/controller/sink.go | 4 +- ...gateway.envoyproxy.io_aigatewayroutes.yaml | 48 +++++++++---------- ...teway.envoyproxy.io_aiservicebackends.yaml | 4 +- tests/extproc/custom_extproc_test.go | 2 +- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go index a7301d28..3411a8d8 100644 --- a/api/v1alpha1/api.go +++ b/api/v1alpha1/api.go @@ -240,10 +240,10 @@ type AIServiceBackendSpec struct { // Note that this is vendor specific, and the stability of the API schema is not guaranteed by // the ai-gateway, but by the vendor via proper versioning. type VersionedAPISchema struct { - // Schema is the API schema of the AIGatewayRoute or AIServiceBackend. + // Name is the name of the API schema of the AIGatewayRoute or AIServiceBackend. // // +kubebuilder:validation:Enum=OpenAI;AWSBedrock - Schema APISchema `json:"schema"` + Name APISchema `json:"name"` // Version is the version of the API schema. Version string `json:"version,omitempty"` diff --git a/internal/controller/sink.go b/internal/controller/sink.go index 86a3acc6..3bd9bd44 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -157,7 +157,7 @@ func (c *configSink) updateExtProcConfigMap(aiGatewayRoute *aigv1a1.AIGatewayRou ec := &filterconfig.Config{} spec := &aiGatewayRoute.Spec - ec.Schema.Name = filterconfig.APISchemaName(spec.APISchema.Schema) + ec.Schema.Name = filterconfig.APISchemaName(spec.APISchema.Name) ec.Schema.Version = spec.APISchema.Version ec.ModelNameHeaderKey = aigv1a1.AIModelHeaderKey ec.SelectedBackendHeaderKey = selectedBackendHeaderKey @@ -172,7 +172,7 @@ func (c *configSink) updateExtProcConfigMap(aiGatewayRoute *aigv1a1.AIGatewayRou if err != nil { return fmt.Errorf("failed to get AIServiceBackend %s: %w", key, err) } else { - ec.Rules[i].Backends[j].Schema.Name = filterconfig.APISchemaName(backendObj.Spec.APISchema.Schema) + ec.Rules[i].Backends[j].Schema.Name = filterconfig.APISchemaName(backendObj.Spec.APISchema.Name) ec.Rules[i].Backends[j].Schema.Version = backendObj.Spec.APISchema.Version } } diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml index 90a89832..e7e13980 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml @@ -153,29 +153,6 @@ spec: required: - type type: object - schema: - description: |- - APISchema specifies the API schema of the input that the target Gateway(s) will receive. - Based on this schema, the ai-gateway will perform the necessary transformation to the - output schema specified in the selected AIServiceBackend during the routing process. - - Currently, the only supported schema is OpenAI as the input schema. - properties: - name: - description: Name is the API schema of the AIGatewayRoute or - AIServiceBackend. - enum: - - OpenAI - - AWSBedrock - type: string - version: - description: Version is the version of the API schema. - type: string - required: - - schema - type: object - x-kubernetes-validations: - - rule: self.name == 'OpenAI' rules: description: |- Rules is the list of AIGatewayRouteRule that this AIGatewayRoute will match the traffic to. @@ -301,6 +278,29 @@ spec: type: object maxItems: 128 type: array + schema: + description: |- + APISchema specifies the API schema of the input that the target Gateway(s) will receive. + Based on this schema, the ai-gateway will perform the necessary transformation to the + output schema specified in the selected AIServiceBackend during the routing process. + + Currently, the only supported schema is OpenAI as the input schema. + properties: + name: + description: Name is the name of the API schema of the AIGatewayRoute + or AIServiceBackend. + enum: + - OpenAI + - AWSBedrock + type: string + version: + description: Version is the version of the API schema. + type: string + required: + - name + type: object + x-kubernetes-validations: + - rule: self.name == 'OpenAI' targetRefs: description: TargetRefs are the names of the Gateway resources this AIGatewayRoute is being attached to. @@ -357,8 +357,8 @@ spec: maxItems: 128 type: array required: - - schema - rules + - schema type: object type: object served: true diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aiservicebackends.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aiservicebackends.yaml index befc65b7..4273af2c 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aiservicebackends.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aiservicebackends.yaml @@ -170,8 +170,8 @@ spec: This is required to be set. properties: name: - description: Name is the name of the API schema of the LLMRoute - or LLMBackend. + description: Name is the name of the API schema of the AIGatewayRoute + or AIServiceBackend. enum: - OpenAI - AWSBedrock diff --git a/tests/extproc/custom_extproc_test.go b/tests/extproc/custom_extproc_test.go index a22e174c..0f397ef7 100644 --- a/tests/extproc/custom_extproc_test.go +++ b/tests/extproc/custom_extproc_test.go @@ -25,7 +25,7 @@ func TestExtProcCustomRouter(t *testing.T) { requireTestUpstream(t) configPath := t.TempDir() + "/extproc-config.yaml" requireWriteExtProcConfig(t, configPath, &filterconfig.Config{ - schema: openAISchema, + Schema: openAISchema, // This can be any header key, but it must match the envoy.yaml routing configuration. SelectedBackendHeaderKey: "x-selected-backend-name", ModelNameHeaderKey: "x-model-name", From 4ca71b9a5ec921a72409e435e9cfa1af872a75cc Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:28:39 -0500 Subject: [PATCH 13/20] fix unit test Signed-off-by: Alexa Griffith --- internal/controller/sink_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/sink_test.go b/internal/controller/sink_test.go index 568e559a..33d894e0 100644 --- a/internal/controller/sink_test.go +++ b/internal/controller/sink_test.go @@ -61,7 +61,7 @@ func TestConfigSink_syncAIGatewayRoute(t *testing.T) { BackendRefs: []aigv1a1.AIGatewayRouteRuleBackendRef{{Name: "apple", Weight: 1}, {Name: "orange", Weight: 1}}, }, }, - APISchema: aigv1a1.VersionedAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v123"}, + APISchema: aigv1a1.VersionedAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v123"}, }, } err := fakeClient.Create(context.Background(), route, &client.CreateOptions{}) @@ -216,7 +216,7 @@ func Test_updateExtProcConfigMap(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "apple", Namespace: "ns"}, Spec: aigv1a1.AIServiceBackendSpec{ APISchema: aigv1a1.VersionedAPISchema{ - Schema: aigv1a1.APISchemaAWSBedrock, + Name: aigv1a1.APISchemaAWSBedrock, }, BackendRef: egv1a1.BackendRef{ BackendObjectReference: gwapiv1.BackendObjectReference{Name: "some-backend1", Namespace: ptr.To[gwapiv1.Namespace]("ns")}, @@ -255,7 +255,7 @@ func Test_updateExtProcConfigMap(t *testing.T) { route: &aigv1a1.AIGatewayRoute{ ObjectMeta: metav1.ObjectMeta{Name: "myroute", Namespace: "ns"}, Spec: aigv1a1.AIGatewayRouteSpec{ - APISchema: aigv1a1.VersionedAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v123"}, + APISchema: aigv1a1.VersionedAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v123"}, Rules: []aigv1a1.AIGatewayRouteRule{ { BackendRefs: []aigv1a1.AIGatewayRouteRuleBackendRef{ From 1d3f19b5872c9b304f59076bbceed63cc92ed1bb Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:34:22 -0500 Subject: [PATCH 14/20] fix test Signed-off-by: Alexa Griffith --- tests/controller/controller_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/controller/controller_test.go b/tests/controller/controller_test.go index 8e9fee4e..eaa2a0a0 100644 --- a/tests/controller/controller_test.go +++ b/tests/controller/controller_test.go @@ -19,7 +19,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -32,7 +31,7 @@ import ( "github.com/envoyproxy/ai-gateway/tests" ) -var defaultSchema = aigv1a1.VersionedAPISchema{Schema: aigv1a1.APISchemaOpenAI, Version: "v1"} +var defaultSchema = aigv1a1.VersionedAPISchema{Name: aigv1a1.APISchemaOpenAI, Version: "v1"} func extProcName(aiGatewayRouteName string) string { return fmt.Sprintf("ai-gateway-ai-gateway-route-extproc-%s", aiGatewayRouteName) From 7b4bc56ae97d8a389956164e39485d254ef37cd9 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Thu, 16 Jan 2025 18:36:14 -0500 Subject: [PATCH 15/20] add klog Signed-off-by: Alexa Griffith --- tests/controller/controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/controller/controller_test.go b/tests/controller/controller_test.go index eaa2a0a0..0bc2517a 100644 --- a/tests/controller/controller_test.go +++ b/tests/controller/controller_test.go @@ -19,6 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + klog "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" From fc7d066e65a4bfd29cad968158a18dc0a1d7a331 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Fri, 17 Jan 2025 14:29:37 -0500 Subject: [PATCH 16/20] fix cel Signed-off-by: Alexa Griffith --- .../crds/aigateway.envoyproxy.io_aigatewayroutes.yaml | 2 +- .../testdata/aigatewayroutes/no_target_refs.yaml | 4 ++-- tests/e2e/testdata/translation_testupstream.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml index 55770c01..4ead229c 100644 --- a/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml +++ b/manifests/charts/ai-gateway-helm/crds/aigateway.envoyproxy.io_aigatewayroutes.yaml @@ -359,8 +359,8 @@ spec: type: array required: - rules - - targetRefs - schema + - targetRefs type: object type: object served: true diff --git a/tests/cel-validation/testdata/aigatewayroutes/no_target_refs.yaml b/tests/cel-validation/testdata/aigatewayroutes/no_target_refs.yaml index ae7f356b..f8d60b97 100644 --- a/tests/cel-validation/testdata/aigatewayroutes/no_target_refs.yaml +++ b/tests/cel-validation/testdata/aigatewayroutes/no_target_refs.yaml @@ -5,8 +5,8 @@ metadata: namespace: default spec: targetRefs: [] - inputSchema: - schema: OpenAI + schema: + name: OpenAI rules: - matches: - headers: diff --git a/tests/e2e/testdata/translation_testupstream.yaml b/tests/e2e/testdata/translation_testupstream.yaml index 94202820..1afaaf60 100644 --- a/tests/e2e/testdata/translation_testupstream.yaml +++ b/tests/e2e/testdata/translation_testupstream.yaml @@ -23,8 +23,8 @@ metadata: name: translation-testupstream namespace: default spec: - inputSchema: - schema: OpenAI + schema: + name: OpenAI targetRefs: - name: translation-testupstream kind: Gateway From ff98ebbb019c4f0f5b0c0a2fd4be112ac39d1cb3 Mon Sep 17 00:00:00 2001 From: Alexa Griffith Date: Fri, 17 Jan 2025 14:33:01 -0500 Subject: [PATCH 17/20] fix test Signed-off-by: Alexa Griffith --- tests/e2e/testdata/translation_testupstream.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/testdata/translation_testupstream.yaml b/tests/e2e/testdata/translation_testupstream.yaml index 1afaaf60..09d58889 100644 --- a/tests/e2e/testdata/translation_testupstream.yaml +++ b/tests/e2e/testdata/translation_testupstream.yaml @@ -53,8 +53,8 @@ metadata: name: translation-testupstream-cool-model-backend namespace: default spec: - outputSchema: - schema: OpenAI + schema: + name: OpenAI backendRef: name: testupstream kind: Service @@ -66,8 +66,8 @@ metadata: name: translation-testupstream-another-cool-model-backend namespace: default spec: - outputSchema: - schema: AWSBedrock + schema: + name: AWSBedrock backendRef: name: testupstream-canary kind: Service From 17c34767f45bf15191fee6e0369dde3cbd97bc36 Mon Sep 17 00:00:00 2001 From: Dan Sun Date: Fri, 17 Jan 2025 22:03:46 -0500 Subject: [PATCH 18/20] Apply suggestions from code review Signed-off-by: Dan Sun --- internal/extproc/watcher_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/extproc/watcher_test.go b/internal/extproc/watcher_test.go index 3628ca7e..9ca9e537 100644 --- a/internal/extproc/watcher_test.go +++ b/internal/extproc/watcher_test.go @@ -51,18 +51,18 @@ rules: - name: kserve weight: 1 schema: - schema: OpenAI + name: OpenAI - name: awsbedrock weight: 10 schema: - schema: AWSBedrock + name: AWSBedrock headers: - name: x-model-name value: llama3.3333 - backends: - name: openai schema: - schema: OpenAI + name: OpenAI headers: - name: x-model-name value: gpt4.4444 From 818885ccbc0b9d48a54d5bb1802975b6bc12efa4 Mon Sep 17 00:00:00 2001 From: Dan Sun Date: Fri, 17 Jan 2025 22:13:40 -0500 Subject: [PATCH 19/20] Apply suggestions from code review Signed-off-by: Dan Sun --- internal/extproc/watcher_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/extproc/watcher_test.go b/internal/extproc/watcher_test.go index 9ca9e537..bb652dcc 100644 --- a/internal/extproc/watcher_test.go +++ b/internal/extproc/watcher_test.go @@ -43,7 +43,7 @@ func TestStartConfigWatcher(t *testing.T) { // Create the initial config file. cfg := ` schema: - schema: OpenAI + name: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-model-name rules: @@ -83,14 +83,14 @@ rules: // Update the config file. cfg = ` schema: - schema: OpenAI + name: OpenAI selectedBackendHeaderKey: x-envoy-ai-gateway-selected-backend modelNameHeaderKey: x-model-name rules: - backends: - name: openai schema: - schema: OpenAI + name: OpenAI headers: - name: x-model-name value: gpt4.4444 From a649b79413e1db97d4a06811fa86d9599d3d8dc0 Mon Sep 17 00:00:00 2001 From: Dan Sun Date: Fri, 17 Jan 2025 22:20:36 -0500 Subject: [PATCH 20/20] Update .gitignore Signed-off-by: Dan Sun --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48c1a064..0c5fd309 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ out/ .terraform .idea *for_tests.yaml -.idea/ # Files and directories to ignore in the site directory # dependencies