Skip to content

Evaluation: Show cost #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AkhileshNegi wants to merge 4 commits into
base: main
Choose a base branch
from
+118 −3
Open

Evaluation: Show cost #113

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bbadd89
added support for showing cost in UI
AkhileshNegi Apr 9, 2026
c0ca999
cleanup UI
AkhileshNegi Apr 10, 2026
35771f7
cleanup UI
AkhileshNegi Apr 10, 2026
27c788c
coderabbit
AkhileshNegi Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions app/(main)/evaluations/[id]/page.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import ConfigModal from "@/app/components/ConfigModal";
import Sidebar from "@/app/components/Sidebar";
import DetailedResultsTable from "@/app/components/DetailedResultsTable";
import { colors } from "@/app/lib/colors";
import { formatCostUSD, formatTokenCount } from "@/app/components/utils";
import { useToast } from "@/app/components/Toast";
import Loader from "@/app/components/Loader";
import {
Expand Down Expand Up @@ -692,6 +693,104 @@ export default function EvaluationReport() {
</p>
</div>
)}

{/* Cost Breakdown */}
{job.cost && (
<div className="mt-6">
<h3
className="text-sm font-semibold mb-3"
style={{ color: colors.text.secondary }}
>
Cost Breakdown
</h3>
<div className="flex gap-4 flex-wrap">
{/* Total Cost */}
<div
className="rounded-lg px-6 py-5 text-center min-w-[180px]"
style={{
backgroundColor: colors.bg.primary,
boxShadow: "0 1px 3px rgba(0, 0, 0, 0.06)",
}}
>
<div
className="text-xs font-medium mb-2"
style={{ color: colors.text.secondary }}
>
Total Cost
</div>
<div
className="text-2xl font-bold"
style={{ color: colors.text.primary }}
>
{formatCostUSD(job.cost.total_cost_usd)}
</div>
</div>

{/* Response Cost */}
{job.cost.response && (
<div
className="rounded-lg px-6 py-5 text-center flex-1 min-w-[180px]"
style={{
backgroundColor: colors.bg.primary,
boxShadow: "0 1px 3px rgba(0, 0, 0, 0.06)",
}}
>
<div
className="text-xs font-medium mb-2"
style={{ color: colors.text.secondary }}
>
Response ({job.cost.response.model})
</div>
<div
className="text-2xl font-bold"
style={{ color: colors.text.primary }}
>
{formatCostUSD(job.cost.response.cost_usd)}
</div>
<div
className="text-xs mt-1"
style={{ color: colors.text.secondary }}
>
{formatTokenCount(job.cost.response.total_tokens)} tokens
{job.cost.response.input_tokens != null && job.cost.response.output_tokens != null && (
<span> · {formatTokenCount(job.cost.response.input_tokens)} in / {formatTokenCount(job.cost.response.output_tokens)} out</span>
)}
</div>
</div>
)}

{/* Embedding Cost */}
{job.cost.embedding && (
<div
className="rounded-lg px-6 py-5 text-center flex-1 min-w-[180px]"
style={{
backgroundColor: colors.bg.primary,
boxShadow: "0 1px 3px rgba(0, 0, 0, 0.06)",
}}
>
<div
className="text-xs font-medium mb-2"
style={{ color: colors.text.secondary }}
>
Embedding ({job.cost.embedding.model})
</div>
<div
className="text-2xl font-bold"
style={{ color: colors.text.primary }}
>
{formatCostUSD(job.cost.embedding.cost_usd)}
</div>
<div
className="text-xs mt-1"
style={{ color: colors.text.secondary }}
>
{formatTokenCount(job.cost.embedding.total_tokens)} tokens
</div>
</div>
)}
</div>
</div>
)}
</div>
) : (
<div
Expand Down
22 changes: 20 additions & 2 deletions app/components/evaluations/EvalRunCard.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
AssistantConfig,
getScoreObject,
} from "@/app/components/types";
import { getStatusColor } from "@/app/components/utils";
import { getStatusColor, formatCostUSD } from "@/app/components/utils";
import { timeAgo } from "@/app/lib/utils";
import ConfigModal from "@/app/components/ConfigModal";
import ScoreDisplay from "@/app/components/ScoreDisplay";
Expand Down Expand Up @@ -81,7 +81,7 @@ export default function EvalRunCard({
</div>
)}

{/* Row 3: Dataset + Config (left) | Actions (right) */}
{/* Row 3: Dataset + Config + Cost (left) | Actions (right) */}
<div className="flex items-center justify-between gap-4 mt-3">
<div
className="flex items-center gap-3 text-xs"
Expand Down Expand Up @@ -113,6 +113,24 @@ export default function EvalRunCard({
{assistantConfig.name}
</span>
)}
{job.cost?.total_cost_usd != null && (
<span className="flex items-center gap-1">
<svg
className="w-3.5 h-3.5 flex-shrink-0"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
strokeWidth={2}
>
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M12 6v12m-3-2.818.879.659c1.171.879 3.07.879 4.242 0 1.172-.879 1.172-2.303 0-3.182C13.536 12.219 12.768 12 12 12c-.725 0-1.45-.22-2.003-.659-1.106-.879-1.106-2.303 0-3.182s2.9-.879 4.006 0l.415.33M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Z"
/>
</svg>
{formatCostUSD(job.cost.total_cost_usd)}
</span>
)}
</div>
<div className="flex items-center gap-3 flex-shrink-0">
<button
Expand Down
16 changes: 16 additions & 0 deletions app/components/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,21 @@ export interface AssistantConfig {
is_deleted: boolean;
}

export interface EvalCostEntry {
model: string;
cost_usd: number;
input_tokens?: number;
output_tokens?: number;
prompt_tokens?: number;
total_tokens: number;
}

export interface EvalCost {
response?: EvalCostEntry;
embedding?: EvalCostEntry;
total_cost_usd: number;
}

export interface EvalJob {
id: number;
run_name: string;
Expand All @@ -130,6 +145,7 @@ export interface EvalJob {
assistant_id?: string;
organization_id: number;
project_id: number;
cost?: EvalCost | null;
inserted_at: string;
updated_at: string;
}
Expand Down
19 changes: 19 additions & 0 deletions app/components/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,25 @@ export const getStatusColor = (
}
};

/**
* Formats a USD cost value for display
* @param cost - Cost in USD
* @returns Formatted cost string (e.g., "$0.0013", "$1.25")
*/
export const formatCostUSD = (cost: number): string => {
if (cost < 0.01) {
return `$${cost.toFixed(4)}`;
}
return `$${cost.toFixed(2)}`;
};

/**
* Formats a token count for display (e.g., 1234 → "1,234")
*/
export const formatTokenCount = (count: number): string => {
return count.toLocaleString();
};

/**
* Calculates dynamic thresholds for color coding based on score distribution
* @param scores - Array of similarity scores
Expand Down
Loading