more updates

Author: adilhafeez

.gitignore

@@ -101,9 +101,6 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-# VSCode stuff:
-.vscode/
-
 # MacOS Metadata
 *.DS_Store
 

arch/arch_config_schema.yaml

@@ -72,20 +72,23 @@ properties:
           type: string
         default:
           type: boolean
-        # endpoint field is deprecated, use base_url instead
-        endpoint:
-          type: string
         base_url:
           type: string
-        protocol:
-          type: string
-          enum:
-            - http
-            - https
         http_host:
           type: string
-        usage:
-          type: string
+        routing_preferences:
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                type: string
+              description:
+                type: string
+          additionalProperties: false
+          required:
+            - name
+            - description
       additionalProperties: false
       required:
         - model

arch/tools/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "cSpell.words": [
+    "BRIGHTSTAFF"
+  ]
+}

arch/tools/cli/config_generator.py

@@ -95,6 +95,8 @@ def validate_and_render_schema():
     updated_llm_providers = []
     llm_provider_name_set = set()
     llms_with_usage = []
+    model_name_keys = set()
+    model_usage_name_keys = set()
     for llm_provider in config_yaml["llm_providers"]:
         if llm_provider.get("usage", None):
             llms_with_usage.append(llm_provider["name"])
@@ -104,6 +106,11 @@ def validate_and_render_schema():
             )
 
         model_name = llm_provider.get("model")
+        if model_name in model_name_keys:
+            raise Exception(
+                f"Duplicate model name {model_name}, please provide unique model name for each llm_provider"
+            )
+        model_name_keys.add(model_name)
         if llm_provider.get("name") is None:
             llm_provider["name"] = model_name
 
@@ -119,6 +126,20 @@ def validate_and_render_schema():
                 f"Unsupported provider {provider} for model {model_name}. Supported providers are: {', '.join(SUPPORTED_PROVIDERS)}"
             )
 
+        if model_id in model_name_keys:
+            raise Exception(
+                f"Duplicate model_id {model_id}, please provide unique model_id for each llm_provider"
+            )
+        model_name_keys.add(model_id)
+
+        for routing_preference in llm_provider.get("routing_preferences", []):
+            if routing_preference.get("name") in model_usage_name_keys:
+                raise Exception(
+                    f"Duplicate routing preference name \"{routing_preference.get('name')}\", please provide unique name for each routing preference"
+                )
+            model_usage_name_keys.add(routing_preference.get("name"))
+
+        llm_provider["model"] = model_id
         llm_provider["provider_interface"] = provider
         llm_provider_name_set.add(llm_provider.get("name"))
         provider = None
@@ -132,21 +153,14 @@ def validate_and_render_schema():
             del llm_provider["provider"]
         updated_llm_providers.append(llm_provider)
 
-        if llm_provider.get("endpoint") and llm_provider.get("base_url"):
-            raise Exception("Please provide either endpoint or base_url, not both")
-
-        if llm_provider.get("endpoint", None):
-            endpoint = llm_provider["endpoint"]
-            protocol = llm_provider.get("protocol", "http")
-            llm_provider["endpoint"], llm_provider["port"] = get_endpoint_and_port(
-                endpoint, protocol
-            )
-            llms_with_endpoint.append(llm_provider)
-        elif llm_provider.get("base_url", None):
+        if llm_provider.get("base_url", None):
             base_url = llm_provider["base_url"]
             urlparse_result = urlparse(base_url)
-            if llm_provider.get("port"):
-                raise Exception("Please provider port in base_url")
+            url_path = urlparse_result.path
+            if url_path and url_path != "/":
+                raise Exception(
+                    f"Please provide base_url without path, got {base_url}. Use base_url like 'http://example.com' instead of 'http://example.com/path'."
+                )
             if urlparse_result.scheme == "" or urlparse_result.scheme not in [
                 "http",
                 "https",

crates/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug Brightstaff",
+      "type": "lldb",
+      "request": "launch",
+      "program": "${workspaceFolder}/target/debug/brightstaff",
+      "args": [],
+      "cwd": "${workspaceFolder}",
+      "stopOnEntry": false,
+      "sourceLanguages": ["rust"],
+      "env": {
+        "RUST_LOG": "debug",
+        "RUST_BACKTRACE": "1",
+        "ARCH_CONFIG_PATH_RENDERED": "../demos/use_cases/preference_based_routing/arch_config_rendered.yaml"
+      },
+      "preLaunchTask": "rust: cargo build"
+    }
+  ]
+}

crates/.vscode/tasks.json

@@ -0,0 +1,21 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+    {
+      "type": "cargo",
+      "command": "build",
+      "args": [
+        "--bin",
+        "brightstaff"
+      ],
+      "problemMatcher": [
+        "$rustc"
+      ],
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      },
+      "label": "rust: cargo build"
+    }
+  ]
+}

crates/brightstaff/src/handlers/chat_completions.rs

@@ -12,7 +12,7 @@ use hyper::{Request, Response, StatusCode};
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 use tokio_stream::StreamExt;
-use tracing::{debug, info, trace, warn};
+use tracing::{debug, info, warn};
 
 use crate::router::llm_router::RouterService;
 
@@ -81,8 +81,8 @@ pub async fn chat_completions(
         }
     }
 
-    trace!(
-        "arch-router request body: {}",
+    debug!(
+        "arch-router request received: {}",
         &serde_json::to_string(&chat_completion_request).unwrap()
     );
 
@@ -102,7 +102,7 @@ pub async fn chat_completions(
         .as_ref()
         .and_then(|s| serde_yaml::from_str(s).ok());
 
-    debug!("usage preferences: {:?}", usage_preferences);
+    debug!("usage preferences from request: {:?}", usage_preferences);
 
     let mut determined_route = match router_service
         .determine_route(

crates/brightstaff/src/main.rs

@@ -44,6 +44,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let _tracer_provider = init_tracer();
     let bind_address = env::var("BIND_ADDRESS").unwrap_or_else(|_| BIND_ADDRESS.to_string());
 
+    info!(
+        "current working directory: {}",
+        env::current_dir().unwrap().display()
+    );
     // loading arch_config.yaml file
     let arch_config_path = env::var("ARCH_CONFIG_PATH_RENDERED")
         .unwrap_or_else(|_| "./arch_config_rendered.yaml".to_string());

crates/brightstaff/src/router/llm_router.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use common::{
-    configuration::{LlmProvider, LlmRoute, ModelUsagePreference},
+    configuration::{LlmProvider, ModelUsagePreference, RoutingPreference},
     consts::ARCH_PROVIDER_HINT_HEADER,
 };
 use hermesllm::providers::openai::types::{ChatCompletionsResponse, ContentType, Message};
@@ -44,11 +44,14 @@ impl RouterService {
     ) -> Self {
         let providers_with_usage = providers
             .iter()
-            .filter(|provider| provider.usage.is_some())
+            .filter(|provider| provider.routing_preferences.is_some())
             .cloned()
             .collect::<Vec<LlmProvider>>();
 
-        let llm_routes: Vec<LlmRoute> = providers_with_usage.iter().map(LlmRoute::from).collect();
+        let llm_routes: Vec<RoutingPreference> = providers_with_usage
+            .iter()
+            .flat_map(|provider| provider.routing_preferences.clone().unwrap_or_default())
+            .collect();
 
         let router_model = Arc::new(router_model_v1::RouterModelV1::new(
             llm_routes,
@@ -156,6 +159,12 @@ impl RouterService {
                 router_response_time.as_millis()
             );
 
+            if let Some(ref route) = route_name {
+                if route == "other" {
+                    return Ok(None);
+                }
+            }
+
             Ok(route_name)
         } else {
             Ok(None)

crates/brightstaff/src/router/router_model_v1.rs

@@ -1,5 +1,5 @@
 use common::{
-    configuration::{LlmRoute, ModelUsagePreference},
+    configuration::{ModelUsagePreference, RoutingPreference},
     consts::{SYSTEM_ROLE, TOOL_ROLE, USER_ROLE},
 };
 use hermesllm::providers::openai::types::{ChatCompletionsRequest, ContentType, Message};
@@ -36,7 +36,11 @@ pub struct RouterModelV1 {
     max_token_length: usize,
 }
 impl RouterModelV1 {
-    pub fn new(llm_routes: Vec<LlmRoute>, routing_model: String, max_token_length: usize) -> Self {
+    pub fn new(
+        llm_routes: Vec<RoutingPreference>,
+        routing_model: String,
+        max_token_length: usize,
+    ) -> Self {
         let llm_route_json_str =
             serde_json::to_string(&llm_routes).unwrap_or_else(|_| "[]".to_string());
         RouterModelV1 {
@@ -138,9 +142,9 @@ impl RouterModel for RouterModelV1 {
         let llm_route_json = usage_preferences
             .as_ref()
             .map(|prefs| {
-                let llm_route: Vec<LlmRoute> = prefs
+                let llm_route: Vec<RoutingPreference> = prefs
                     .iter()
-                    .map(|pref| LlmRoute {
+                    .map(|pref| RoutingPreference {
                         name: pref.name.clone(),
                         description: pref.usage.clone().unwrap_or_default(),
                     })
@@ -255,7 +259,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), usize::MAX);
 
@@ -314,7 +318,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), usize::MAX);
 
@@ -379,7 +383,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), 235);
 
@@ -440,7 +444,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), 200);
 
@@ -501,7 +505,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), 230);
 
@@ -569,7 +573,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), usize::MAX);
 
@@ -639,7 +643,7 @@ Based on your analysis, provide your response in the following JSON formats if y
               {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
           ]
         "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
         let routing_model = "test-model".to_string();
         let router = RouterModelV1::new(llm_routes, routing_model.clone(), usize::MAX);
 
@@ -716,7 +720,7 @@ Based on your analysis, provide your response in the following JSON formats if y
     {"name": "Speech Recognition", "description": "Converting spoken language into written text"}
 ]
 "#;
-        let llm_routes = serde_json::from_str::<Vec<LlmRoute>>(routes_str).unwrap();
+        let llm_routes = serde_json::from_str::<Vec<RoutingPreference>>(routes_str).unwrap();
 
         let router = RouterModelV1::new(llm_routes, "test-model".to_string(), 2000);