feat: revamp UX

packages/backend/package.json

@@ -4,6 +4,7 @@
   "scripts": {
     "start": "npm run migrate:db && tsx src/index.ts",
     "start:radar": "tsx src/radar.ts",
+    "dev:realtime-evaluators": "tsx --env-file=.env --watch src/realtime-evaluators.ts",
     "migrate:db": "tsx src/migrate.ts",
     "build": "tsup src/index.ts --format esm",
     "dev": "tsx --env-file=.env --watch src/index.ts",
@@ -13,15 +14,16 @@
   },
   "dependencies": {
     "@authenio/samlify-node-xmllint": "^2.0.0",
+    "@aws-sdk/client-comprehend": "^3.609.0",
     "@json2csv/plainjs": "^7.0.5",
     "@koa/cors": "^5.0.0",
-    "@sentry/node": "^7.99.0",
-    "@sentry/profiling-node": "^1.3.5",
-    "@sentry/utils": "^7.99.0",
+    "@sentry/node": "^8.15.0",
+    "@sentry/profiling-node": "^8.15.0",
+    "@sentry/utils": "^8.15.0",
     "@xenova/transformers": "^2.14.2",
-    "argon2": "^0.31.2",
+    "argon2": "^0.40.3",
     "bcrypt": "^5.1.1",
-    "jose": "^5.2.0",
+    "jose": "^5.6.3",
     "js-tiktoken": "1.0.7",
     "jsonrepair": "^3.7.0",
     "koa": "^2.15.3",
@@ -39,7 +41,7 @@
     "rouge": "^1.0.3",
     "samlify": "^2.8.11",
     "shared": "*",
-    "stripe": "^15.5.0",
+    "stripe": "^16.2.0",
     "washyourmouthoutwithsoap": "^1.0.2",
     "zod": "^3.23.6"
   },

packages/backend/src/api/v1/analytics/index.ts

@@ -53,7 +53,7 @@ analytics.get(
         order by
           date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -76,7 +76,7 @@ analytics.get(
         order by d.date;
     `
 
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -122,7 +122,7 @@ analytics.get(
         order by
           date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -145,7 +145,7 @@ analytics.get(
         order by d.date;
     `
 
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -191,7 +191,7 @@ analytics.get(
         order by
           date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -213,7 +213,7 @@ analytics.get(
         order by d.date;
     `
 
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -272,7 +272,7 @@ analytics.get(
           date
 
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -299,7 +299,7 @@ analytics.get(
         order by
           d.date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -572,7 +572,7 @@ analytics.get(
         order by
           date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -595,7 +595,7 @@ analytics.get(
         order by d.date;
     `
 
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -741,7 +741,7 @@ analytics.get(
       order by
         date;
       `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     } else {
       const res = await sql`
@@ -776,7 +776,7 @@ analytics.get(
         order by
           d.date;
         `
-      ctx.body = res
+      ctx.body = { data: res }
       return
     }
   },
@@ -840,8 +840,6 @@ analytics.get(
       limit 5
     `
 
-    console.log(topModels)
-
     ctx.body = topModels
   },
 )

packages/backend/src/api/v1/evaluations/index.ts

@@ -73,6 +73,7 @@ evaluations.post(
           count++
           queue.enqueue(async () => {
             await runEval({
+              projectId,
               evaluationId: evaluation.id,
               promptId: prompt.id,
               variation,
@@ -219,7 +220,7 @@ evaluations.post(
       cost: 0,
     }
 
-    const cost = calcRunCost(virtualRun)
+    const cost = await calcRunCost(virtualRun)
     virtualRun.cost = cost
     virtualRun.duration = virtualRun.duration / 1000 // needs to be in ms in calcRunCost, but needs to be in seconds in the checks
 

packages/backend/src/api/v1/evaluations/utils.ts

@@ -5,6 +5,7 @@ import { compilePrompt, runAImodel } from "@/src/utils/playground"
 import { Provider } from "shared"
 
 interface RunEvalParams {
+  projectId: string
   evaluationId: string
   promptId: string
   checklistId: string
@@ -15,6 +16,7 @@ interface RunEvalParams {
 }
 
 export async function runEval({
+  projectId,
   evaluationId,
   promptId,
   checklistId,
@@ -92,12 +94,12 @@ export async function runEval({
         idealOutput,
         // So the SQL queries don't fail:
         id: "00000000-0000-4000-8000-000000000000",
-        projectId: "00000000-0000-4000-8000-000000000000",
+        projectId,
         isPublic: false,
         cost: 0,
       }
 
-      const cost = calcRunCost(virtualRun)
+      const cost = await calcRunCost(virtualRun)
       virtualRun.cost = cost || 0
       virtualRun.duration = virtualRun.duration / 1000 // needs to be in ms in calcRunCost, but needs to be in seconds in the checks
 

packages/backend/src/api/v1/evaluator.ts

@@ -44,6 +44,7 @@ evaluators.get("/:id", async (ctx: Context) => {
 })
 
 evaluators.post("/", async (ctx: Context) => {
+  console.log(ctx.request.body)
   const requestBody = z.object({
     ownerId: z.string().optional(),
     name: z.string(),

packages/backend/src/api/v1/external-users.ts

@@ -38,39 +38,52 @@ users.get("/", checkAccess("users", "list"), async (ctx: Context) => {
     `
   }
 
-  const users = await sql`
-    with user_costs as (
+  const [users, total] = await Promise.all([
+    sql`
+      with user_costs as (
+        select
+          external_user_id,
+          coalesce(sum(cost), 0) as cost
+        from
+          run r
+        where
+          project_id = ${projectId} 
+          ${createAtQuery}
+        group by
+          external_user_id
+      )
       select
-        external_user_id,
-        coalesce(sum(cost), 0) as cost
+        eu.id,
+        eu.created_at,
+        eu.external_id,
+        eu.last_seen,
+        eu.props,
+        uc.cost
       from
-        run r
+        public.external_user eu
+        left join user_costs uc on eu.id = uc.external_user_id
       where
-        project_id = ${projectId} 
-        ${createAtQuery}
-      group by
-        external_user_id
-    )
-    select
-      eu.id,
-      eu.created_at,
-      eu.external_id,
-      eu.last_seen,
-      eu.props,
-      uc.cost
-    from
-      public.external_user eu
-      left join user_costs uc on eu.id = uc.external_user_id
-    where
-      eu.project_id = ${projectId} 
-      ${searchQuery} 
-    order by
-      cost desc nulls last
-    limit ${limit}
-    offset ${page * limit}
-  `
-
-  ctx.body = users
+        eu.project_id = ${projectId} 
+        ${searchQuery} 
+      order by
+        cost desc nulls last
+      limit ${limit}
+      offset ${page * limit}
+    `,
+    sql`
+      select count(*) as total
+      from public.external_user eu
+      where eu.project_id = ${projectId} 
+      ${searchQuery}
+    `,
+  ])
+
+  ctx.body = {
+    total: +total[0].total,
+    page,
+    limit,
+    data: users,
+  }
 })
 
 // TODO: deprecated?

packages/backend/src/api/v1/index.ts

@@ -16,6 +16,8 @@ import templates from "./templates"
 import users from "./users"
 import evaluators from "./evaluator"
 import analytics from "./analytics"
+import views from "./views"
+import models from "./models"
 
 const v1 = new Router({
   prefix: "/v1",
@@ -48,5 +50,7 @@ v1.use(evaluations.routes())
 v1.use(projectUsers.routes())
 v1.use(checklists.routes())
 v1.use(analytics.routes())
+v1.use(views.routes())
+v1.use(models.routes())
 
 export default v1

packages/backend/src/api/v1/models.ts

@@ -0,0 +1,77 @@
+import { checkAccess } from "@/src/utils/authorization"
+import sql from "@/src/utils/db"
+import { clearUndefined } from "@/src/utils/ingest"
+import Context from "@/src/utils/koa"
+import Router from "koa-router"
+import { z } from "zod"
+
+const models = new Router({
+  prefix: "/models",
+})
+
+const ModelSchema = z.object({
+  name: z.string().min(1),
+  pattern: z.string().min(1),
+  unit: z.enum(["TOKENS", "CHARACTERS", "MILLISECONDS"]),
+  inputCost: z.number().min(0),
+  outputCost: z.number().min(0),
+  tokenizer: z.string().optional(),
+  startDate: z.coerce.date().optional(),
+})
+
+models.get("/", checkAccess("logs", "list"), async (ctx: Context) => {
+  const { orgId } = ctx.state
+
+  ctx.body = await sql`select * from model_mapping
+        where org_id = ${orgId} or org_id is null
+        order by updated_at desc`
+})
+
+models.post("/", async (ctx: Context) => {
+  const { orgId } = ctx.state
+
+  const validatedData = ModelSchema.parse(ctx.request.body)
+
+  const [insertedModel] = await sql`
+    insert into model_mapping ${sql(
+      clearUndefined({
+        ...validatedData,
+        orgId,
+      }),
+    )}
+    returning *
+  `
+  ctx.body = insertedModel
+})
+
+models.patch("/:id", async (ctx: Context) => {
+  const { orgId } = ctx.state
+  const { id } = ctx.params
+
+  const validatedData = ModelSchema.partial().parse(ctx.request.body)
+
+  const [updatedModel] = await sql`
+    update model_mapping
+    set ${sql(clearUndefined({ ...validatedData, updatedAt: new Date() }))}
+    where org_id = ${orgId}
+    and id = ${id}
+    returning *
+  `
+  ctx.body = updatedModel
+})
+
+models.delete("/:id", checkAccess("logs", "delete"), async (ctx: Context) => {
+  const { orgId } = ctx.state
+  const { id } = ctx.params
+
+  await sql`
+    delete from model_mapping
+    where org_id = ${orgId}
+    and id = ${id}
+    returning *
+  `
+
+  ctx.status = 200
+})
+
+export default models