improv(webapp): Add new table for optimized logs search

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -50,7 +50,6 @@ export type LogsListOptions = {
   retentionLimitDays?: number;
   // search
   search?: string;
-  includeDebugLogs?: boolean;
   // pagination
   direction?: Direction;
   cursor?: string;
@@ -69,7 +68,6 @@ export const LogsListOptionsSchema = z.object({
   defaultPeriod: z.string().optional(),
   retentionLimitDays: z.number().int().positive().optional(),
   search: z.string().max(1000).optional(),
-  includeDebugLogs: z.boolean().optional(),
   direction: z.enum(["forward", "backward"]).optional(),
   cursor: z.string().optional(),
   pageSize: z.number().int().positive().max(1000).optional(),
@@ -83,15 +81,17 @@ export type LogsListAppliedFilters = LogsList["filters"];
 
 // Cursor is a base64 encoded JSON of the pagination keys
 type LogCursor = {
+  organizationId: string;
   environmentId: string;
-  unixTimestamp: number;
-  traceId: string;
+  triggeredTimestamp: string; // DateTime64(9) string
+  spanId: string;
 };
 
 const LogCursorSchema = z.object({
+  organizationId: z.string(),
   environmentId: z.string(),
-  unixTimestamp: z.number(),
-  traceId: z.string(),
+  triggeredTimestamp: z.string(),
+  spanId: z.string(),
 });
 
 function encodeCursor(cursor: LogCursor): string {
@@ -166,7 +166,6 @@ export class LogsListPresenter extends BasePresenter {
       to,
       cursor,
       pageSize = DEFAULT_PAGE_SIZE,
-      includeDebugLogs = true,
       defaultPeriod,
       retentionLimitDays,
     }: LogsListOptions
@@ -252,7 +251,7 @@ export class LogsListPresenter extends BasePresenter {
       );
     }
 
-    const queryBuilder = this.clickhouse.taskEventsV2.logsListQueryBuilder();
+    const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder();
 
     queryBuilder.where("environment_id = {environmentId: String}", {
       environmentId,
@@ -349,39 +348,22 @@ export class LogsListPresenter extends BasePresenter {
       }
     }
 
-    // Debug logs are available only to admins
-    if (includeDebugLogs === false) {
-      queryBuilder.where("kind NOT IN {debugKinds: Array(String)}", {
-        debugKinds: ["DEBUG_EVENT"],
-      });
-
-      queryBuilder.where("NOT ((kind = 'LOG_INFO') AND (attributes_text = '{}'))");
-    }
-
-    queryBuilder.where("kind NOT IN {debugSpans: Array(String)}", {
-      debugSpans: ["SPAN", "ANCESTOR_OVERRIDE", "SPAN_EVENT"],
-    });
-
-    // kindCondition += ` `;
-    // params["excluded_statuses"] = ["SPAN", "ANCESTOR_OVERRIDE", "SPAN_EVENT"];
-
-
-    queryBuilder.where("NOT (kind = 'SPAN' AND status = 'PARTIAL')");
-
-    // Cursor pagination
+    // Cursor pagination using explicit lexicographic comparison
+    // Must mirror the ORDER BY columns: (organization_id DESC, environment_id DESC, triggered_timestamp DESC, span_id DESC)
     const decodedCursor = cursor ? decodeCursor(cursor) : null;
     if (decodedCursor) {
       queryBuilder.where(
-        "(environment_id, toUnixTimestamp(start_time), trace_id) < ({cursorEnvId: String}, {cursorUnixTimestamp: Int64}, {cursorTraceId: String})",
+        `((organization_id = {cursorOrgId: String} AND environment_id = {cursorEnvId: String} AND triggered_timestamp = {cursorTriggeredTimestamp: String} AND span_id < {cursorSpanId: String}) OR (organization_id = {cursorOrgId: String} AND environment_id = {cursorEnvId: String} AND triggered_timestamp < {cursorTriggeredTimestamp: String}) OR (organization_id = {cursorOrgId: String} AND environment_id < {cursorEnvId: String}) OR (organization_id < {cursorOrgId: String}))`,
         {
+          cursorOrgId: decodedCursor.organizationId,
           cursorEnvId: decodedCursor.environmentId,
-          cursorUnixTimestamp: decodedCursor.unixTimestamp,
-          cursorTraceId: decodedCursor.traceId,
+          cursorTriggeredTimestamp: decodedCursor.triggeredTimestamp,
+          cursorSpanId: decodedCursor.spanId,
         }
       );
     }
 
-    queryBuilder.orderBy("environment_id DESC, toUnixTimestamp(start_time) DESC, trace_id DESC");
+    queryBuilder.orderBy("organization_id DESC, environment_id DESC, triggered_timestamp DESC, span_id DESC");
     // Limit + 1 to check if there are more results
     queryBuilder.limit(pageSize + 1);
 
@@ -399,11 +381,11 @@ export class LogsListPresenter extends BasePresenter {
     let nextCursor: string | undefined;
     if (hasMore && logs.length > 0) {
       const lastLog = logs[logs.length - 1];
-      const unixTimestamp = Math.floor(new Date(lastLog.start_time).getTime() / 1000);
       nextCursor = encodeCursor({
+        organizationId,
         environmentId,
-        unixTimestamp,
-        traceId: lastLog.trace_id,
+        triggeredTimestamp: lastLog.triggered_timestamp,
+        spanId: lastLog.span_id,
       });
     }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx

@@ -36,7 +36,6 @@ import {
   ResizablePanel,
   ResizablePanelGroup,
 } from "~/components/primitives/Resizable";
-import { Switch } from "~/components/primitives/Switch";
 import { Button } from "~/components/primitives/Buttons";
 import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server";
 
@@ -95,7 +94,6 @@ async function hasLogsPageAccess(
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const user = await requireUser(request);
   const userId = user.id;
-  const isAdmin = user.admin || user.isImpersonating;
 
   const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
 
@@ -126,7 +124,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const runId = url.searchParams.get("runId") ?? undefined;
   const search = url.searchParams.get("search") ?? undefined;
   const levels = parseLevelsFromUrl(url);
-  const showDebug = url.searchParams.get("showDebug") === "true";
   const period = url.searchParams.get("period") ?? undefined;
   const fromStr = url.searchParams.get("from");
   const toStr = url.searchParams.get("to");
@@ -150,7 +147,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       period,
       from,
       to,
-      includeDebugLogs: isAdmin && showDebug,
       defaultPeriod: "1h",
       retentionLimitDays
     })
@@ -163,15 +159,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   return typeddefer({
     data: listPromise,
-    isAdmin,
-    showDebug,
     defaultPeriod: "1h",
     retentionLimitDays,
   });
 };
 
 export default function Page() {
-  const { data, isAdmin, showDebug, defaultPeriod, retentionLimitDays } =
+  const { data, defaultPeriod, retentionLimitDays } =
     useTypedLoaderData<typeof loader>();
 
   return (
@@ -199,8 +193,6 @@ export default function Page() {
             errorElement={
               <div className="grid h-full max-h-full grid-rows-[2.5rem_auto_1fr] overflow-hidden">
                 <FiltersBar
-                  isAdmin={isAdmin}
-                  showDebug={showDebug}
                   defaultPeriod={defaultPeriod}
                   retentionLimitDays={retentionLimitDays}
                 />
@@ -218,8 +210,6 @@ export default function Page() {
                 return (
                   <div className="grid h-full max-h-full grid-rows-[2.5rem_auto_1fr] overflow-hidden">
                     <FiltersBar
-                      isAdmin={isAdmin}
-                      showDebug={showDebug}
                       defaultPeriod={defaultPeriod}
                       retentionLimitDays={retentionLimitDays}
                     />
@@ -235,15 +225,11 @@ export default function Page() {
                 <div className="grid h-full max-h-full grid-rows-[2.5rem_1fr] overflow-hidden">
                   <FiltersBar
                     list={result}
-                    isAdmin={isAdmin}
-                    showDebug={showDebug}
                     defaultPeriod={defaultPeriod}
                     retentionLimitDays={retentionLimitDays}
                   />
                   <LogsList
                     list={result}
-                    isAdmin={isAdmin}
-                    showDebug={showDebug}
                     defaultPeriod={defaultPeriod}
                   />
                 </div>
@@ -258,14 +244,10 @@ export default function Page() {
 
 function FiltersBar({
   list,
-  isAdmin,
-  showDebug,
   defaultPeriod,
   retentionLimitDays,
 }: {
   list?: Exclude<Awaited<UseDataFunctionReturn<typeof loader>["data"]>, { error: string }>;
-  isAdmin: boolean;
-  showDebug: boolean;
   defaultPeriod?: string;
   retentionLimitDays: number;
 }) {
@@ -280,16 +262,6 @@ function FiltersBar({
     searchParams.has("from") ||
     searchParams.has("to");
 
-  const handleDebugToggle = useCallback((checked: boolean) => {
-    const url = new URL(window.location.href);
-    if (checked) {
-      url.searchParams.set("showDebug", "true");
-    } else {
-      url.searchParams.delete("showDebug");
-    }
-    window.location.href = url.toString();
-  }, []);
-
   return (
     <div className="flex items-start justify-between gap-x-2 border-b border-grid-bright p-2">
       <div className="flex flex-row flex-wrap items-center gap-1">
@@ -329,16 +301,6 @@ function FiltersBar({
           </>
         )}
       </div>
-      <div className="flex items-center gap-2">
-        {isAdmin && (
-          <Switch
-            variant="small"
-            label="Debug"
-            checked={showDebug}
-            onCheckedChange={handleDebugToggle}
-          />
-        )}
-      </div>
     </div>
   );
 }
@@ -347,8 +309,6 @@ function LogsList({
   list,
 }: {
   list: Exclude<Awaited<UseDataFunctionReturn<typeof loader>["data"]>, { error: string }>; //exclude error, it is handled
-  isAdmin: boolean;
-  showDebug: boolean;
   defaultPeriod?: string;
 }) {
   const navigation = useNavigation();

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.spans.tsx

@@ -63,7 +63,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   }
 
   // Query ClickHouse for related spans in the same trace
-  const queryBuilder = clickhouseClient.taskEventsV2.logsListQueryBuilder();
+  const queryBuilder = clickhouseClient.taskEventsSearch.logsListQueryBuilder();
 
   queryBuilder.where("environment_id = {environmentId: String}", {
     environmentId: environment.id,

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts

@@ -21,7 +21,6 @@ function parseLevelsFromUrl(url: URL): LogLevel[] | undefined {
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const user = await requireUser(request);
   const userId = user.id;
-  const isAdmin = user?.admin || user?.isImpersonating;
 
   const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
 
@@ -46,7 +45,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const search = url.searchParams.get("search") ?? undefined;
   const cursor = url.searchParams.get("cursor") ?? undefined;
   const levels = parseLevelsFromUrl(url);
-  const showDebug = url.searchParams.get("showDebug") === "true";
   const period = url.searchParams.get("period") ?? undefined;
   const fromStr = url.searchParams.get("from");
   const toStr = url.searchParams.get("to");
@@ -67,7 +65,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     from,
     to,
     levels,
-    includeDebugLogs: isAdmin && showDebug,
     defaultPeriod: "1h",
     retentionLimitDays,
   }) as any; // Validated by LogsListOptionsSchema at runtime

internal-packages/clickhouse/schema/016_add_task_events_search_v1.sql

@@ -0,0 +1,63 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS trigger_dev.task_events_search_v1
+(
+  environment_id String,
+  organization_id String,
+  project_id String,
+  triggered_timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+  trace_id String CODEC(ZSTD(1)),
+  span_id String CODEC(ZSTD(1)),
+  run_id String CODEC(ZSTD(1)),
+  task_identifier String CODEC(ZSTD(1)),
+  start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+  inserted_at DateTime64(3),
+  message String CODEC(ZSTD(1)),
+  kind LowCardinality(String) CODEC(ZSTD(1)),
+  status LowCardinality(String) CODEC(ZSTD(1)),
+  duration UInt64 CODEC(ZSTD(1)),
+  parent_span_id String CODEC(ZSTD(1)),
+  attributes_text String CODEC(ZSTD(1)),
+
+  INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
+  INDEX idx_message_text_search lower(message) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1,
+  INDEX idx_attributes_text_search lower(attributes_text) TYPE ngrambf_v1(3, 32768, 2, 0) GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(triggered_timestamp)
+ORDER BY (organization_id, environment_id, triggered_timestamp, span_id)
+--Right now we have maximum retention of up to 30 days based on plan.
+--We put a logical limit for now, the 90 DAY TTL is just a backup
+--This might need to be updated for longer retention periods
+TTL toDateTime(triggered_timestamp) + INTERVAL 90 DAY
+SETTINGS ttl_only_drop_parts = 1;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1
+TO trigger_dev.task_events_search_v1 AS
+SELECT
+  environment_id,
+  organization_id,
+  project_id,
+  trace_id,
+  span_id,
+  run_id,
+  task_identifier,
+  start_time,
+  inserted_at,
+  message,
+  kind,
+  status,
+  duration,
+  parent_span_id,
+  toJSONString(attributes) AS attributes_text,
+  fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp
+FROM trigger_dev.task_events_v2
+WHERE
+    kind != 'DEBUG_EVENT'
+    AND status != 'PARTIAL'
+    AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
+    AND kind != 'ANCESTOR_OVERRIDE'
+    AND message != 'trigger.dev/start';
+
+-- +goose Down
+DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1;
+DROP TABLE IF EXISTS trigger_dev.task_events_search_v1;

internal-packages/clickhouse/src/index.ts

@@ -23,8 +23,8 @@ import {
   getTraceSummaryQueryBuilderV2,
   insertTaskEvents,
   insertTaskEventsV2,
-  getLogsListQueryBuilderV2,
   getLogDetailQueryBuilderV2,
+  getLogsSearchListQueryBuilder,
 } from "./taskEvents.js";
 import { Logger, type LogLevel } from "@trigger.dev/core/logger";
 import type { Agent as HttpAgent } from "http";
@@ -220,8 +220,13 @@ export class ClickHouse {
       traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader),
       traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader),
       spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader),
-      logsListQueryBuilder: getLogsListQueryBuilderV2(this.reader, this.logsQuerySettings?.list),
       logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader, this.logsQuerySettings?.detail),
     };
   }
+
+  get taskEventsSearch() {
+    return {
+      logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader, this.logsQuerySettings?.list),
+    };
+  }
 }