Tags listing now uses ClickHouse (#2576)

* WIP using ClickHouse for the tags filter list

* WIP on tags listing

* Webapp: exclude test files when typechecking

* Tags filtering working with CH

* Remove unused import

* The AI filter should only look at the last past 30d of tags

* Do the text query in ClickHouse

* Deal with encoded characters better

* More encoding fixes

* Fix for wrong items being checked

* Put applied tags back

* Add the env.id to the dependencies array

Author: matt-aitken

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -58,10 +58,10 @@ import { useProject } from "~/hooks/useProject";
 import { useSearchParams } from "~/hooks/useSearchParam";
 import { type loader as queuesLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues";
 import { type loader as versionsLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions";
-import { type loader as tagsLoader } from "~/routes/resources.projects.$projectParam.runs.tags";
+import { type loader as tagsLoader } from "~/routes/resources.environments.$envId.runs.tags";
 import { Button } from "../../primitives/Buttons";
 import { BulkActionTypeCombo } from "./BulkAction";
-import { appliedSummary, FilterMenuProvider, TimeFilter } from "./SharedFilters";
+import { appliedSummary, FilterMenuProvider, TimeFilter, timeFilters } from "./SharedFilters";
 import { AIFilterInput } from "./AIFilterInput";
 import {
   allTaskRunStatuses,
@@ -280,7 +280,7 @@ export function getRunFiltersFromSearchParams(
     bulkId: searchParams.get("bulkId") ?? undefined,
     tags:
       searchParams.getAll("tags").filter((v) => v.length > 0).length > 0
-        ? searchParams.getAll("tags").map((t) => decodeURIComponent(t))
+        ? searchParams.getAll("tags")
         : undefined,
     from: searchParams.get("from") ?? undefined,
     to: searchParams.get("to") ?? undefined,
@@ -810,8 +810,8 @@ function TagsDropdown({
   searchValue: string;
   onClose?: () => void;
 }) {
-  const project = useProject();
-  const { values, replace } = useSearchParams();
+  const environment = useEnvironment();
+  const { values, value, replace } = useSearchParams();
 
   const handleChange = (values: string[]) => {
     clearSearchValue();
@@ -822,6 +822,12 @@ function TagsDropdown({
     });
   };
 
+  const { period, from, to } = timeFilters({
+    period: value("period"),
+    from: value("from"),
+    to: value("to"),
+  });
+
   const tagValues = values("tags").filter((v) => v !== "");
   const selected = tagValues.length > 0 ? tagValues : undefined;
 
@@ -830,25 +836,34 @@ function TagsDropdown({
   useEffect(() => {
     const searchParams = new URLSearchParams();
     if (searchValue) {
-      searchParams.set("name", encodeURIComponent(searchValue));
+      searchParams.set("name", searchValue);
     }
-    fetcher.load(`/resources/projects/${project.slug}/runs/tags?${searchParams}`);
-  }, [searchValue]);
+    if (period) {
+      searchParams.set("period", period);
+    }
+    if (from) {
+      searchParams.set("from", from.getTime().toString());
+    }
+    if (to) {
+      searchParams.set("to", to.getTime().toString());
+    }
+    fetcher.load(`/resources/environments/${environment.id}/runs/tags?${searchParams}`);
+  }, [environment.id, searchValue, period, from?.getTime(), to?.getTime()]);
 
   const filtered = useMemo(() => {
     let items: string[] = [];
     if (searchValue === "") {
-      items = selected ?? [];
+      items = [...(selected ?? [])];
     }
 
     if (fetcher.data === undefined) {
       return matchSorter(items, searchValue);
     }
 
-    items.push(...fetcher.data.tags.map((t) => t.name));
+    items.push(...fetcher.data.tags);
 
     return matchSorter(Array.from(new Set(items)), searchValue);
-  }, [searchValue, fetcher.data]);
+  }, [searchValue, fetcher.data, selected]);
 
   return (
     <SelectProvider value={selected ?? []} setValue={handleChange} virtualFocus={true}>
@@ -958,7 +973,7 @@ function QueuesDropdown({
       const searchParams = new URLSearchParams();
       searchParams.set("per_page", "25");
       if (searchValue) {
-        searchParams.set("query", encodeURIComponent(s));
+        searchParams.set("query", s);
       }
       fetcher.load(
         `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${
@@ -1220,7 +1235,7 @@ function VersionsDropdown({
     (s) => {
       const searchParams = new URLSearchParams();
       if (searchValue) {
-        searchParams.set("query", encodeURIComponent(s));
+        searchParams.set("query", s);
       }
       fetcher.load(
         `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${

apps/webapp/app/components/runs/v3/WaitpointTokenFilters.tsx

@@ -330,7 +330,7 @@ function TagsDropdown({
   useEffect(() => {
     const searchParams = new URLSearchParams();
     if (searchValue) {
-      searchParams.set("name", encodeURIComponent(searchValue));
+      searchParams.set("name", searchValue);
     }
     fetcher.load(
       `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/waitpoints/tags?${searchParams}`

apps/webapp/app/hooks/useSearchParam.ts

@@ -33,21 +33,15 @@ export function useSearchParams() {
   const value = useCallback(
     (param: string) => {
       const search = new URLSearchParams(location.search);
-      const val = search.get(param) ?? undefined;
-      if (val === undefined) {
-        return val;
-      }
-
-      return decodeURIComponent(val);
+      return search.get(param) ?? undefined;
     },
     [location]
   );
 
   const values = useCallback(
     (param: string) => {
       const search = new URLSearchParams(location.search);
-      const all = search.getAll(param);
-      return all.map((v) => decodeURIComponent(v));
+      return search.getAll(param);
     },
     [location]
   );

apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts

@@ -1,8 +1,16 @@
+import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
 import { BasePresenter } from "./basePresenter.server";
+import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { type PrismaClient } from "@trigger.dev/database";
+import { timeFilters } from "~/components/runs/v3/SharedFilters";
 
 export type TagListOptions = {
-  userId?: string;
+  organizationId: string;
+  environmentId: string;
   projectId: string;
+  period?: string;
+  from?: Date;
+  to?: Date;
   //filters
   name?: string;
   //pagination
@@ -17,40 +25,39 @@ export type TagListItem = TagList["tags"][number];
 
 export class RunTagListPresenter extends BasePresenter {
   public async call({
-    userId,
+    organizationId,
+    environmentId,
     projectId,
     name,
+    period,
+    from,
+    to,
     page = 1,
     pageSize = DEFAULT_PAGE_SIZE,
   }: TagListOptions) {
     const hasFilters = Boolean(name?.trim());
 
-    const tags = await this._replica.taskRunTag.findMany({
-      where: {
-        projectId,
-        name: name
-          ? {
-              startsWith: name,
-              mode: "insensitive",
-            }
-          : undefined,
-      },
-      orderBy: {
-        id: "desc",
-      },
-      take: pageSize + 1,
-      skip: (page - 1) * pageSize,
+    const runsRepository = new RunsRepository({
+      clickhouse: clickhouseClient,
+      prisma: this._replica as PrismaClient,
+    });
+
+    const tags = await runsRepository.listTags({
+      organizationId,
+      projectId,
+      environmentId,
+      query: name,
+      period,
+      from: from ? from.getTime() : undefined,
+      to: to ? to.getTime() : undefined,
+      offset: (page - 1) * pageSize,
+      limit: pageSize + 1,
     });
 
     return {
-      tags: tags
-        .map((tag) => ({
-          id: tag.friendlyId,
-          name: tag.name,
-        }))
-        .slice(0, pageSize),
+      tags: tags.tags,
       currentPage: page,
-      hasMore: tags.length > pageSize,
+      hasMore: tags.tags.length > pageSize,
       hasFilters,
     };
   }

apps/webapp/app/routes/resources.environments.$envId.runs.tags.tsx

@@ -0,0 +1,66 @@
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { timeFilters } from "~/components/runs/v3/SharedFilters";
+import { $replica } from "~/db.server";
+import { RunTagListPresenter } from "~/presenters/v3/RunTagListPresenter.server";
+import { requireUserId } from "~/services/session.server";
+
+const Params = z.object({
+  envId: z.string(),
+});
+
+const SearchParams = z.object({
+  name: z.string().optional(),
+  period: z.preprocess((value) => (value === "all" ? undefined : value), z.string().optional()),
+  from: z.coerce.number().optional(),
+  to: z.coerce.number().optional(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const { envId } = Params.parse(params);
+
+  const environment = await $replica.runtimeEnvironment.findFirst({
+    select: {
+      id: true,
+      projectId: true,
+      organizationId: true,
+    },
+    where: { id: envId, organization: { members: { some: { userId } } } },
+  });
+
+  if (!environment) {
+    throw new Response("Not Found", { status: 404 });
+  }
+
+  const search = new URL(request.url).searchParams;
+
+  const parsedSearchParams = SearchParams.safeParse({
+    name: search.get("name") ?? undefined,
+    period: search.get("period") ?? undefined,
+    from: search.get("from") ?? undefined,
+    to: search.get("to") ?? undefined,
+  });
+
+  if (!parsedSearchParams.success) {
+    throw new Response("Invalid search params", { status: 400 });
+  }
+
+  const { period, from, to } = timeFilters({
+    period: parsedSearchParams.data.period,
+    from: parsedSearchParams.data.from,
+    to: parsedSearchParams.data.to,
+  });
+
+  const presenter = new RunTagListPresenter();
+  const result = await presenter.call({
+    environmentId: environment.id,
+    projectId: environment.projectId,
+    organizationId: environment.organizationId,
+    name: parsedSearchParams.data.name,
+    period,
+    from,
+    to,
+  });
+  return result;
+}

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.ai-filter.tsx

@@ -65,13 +65,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
     query: async (search) => {
       const tagPresenter = new RunTagListPresenter();
       const tags = await tagPresenter.call({
+        organizationId: environment.organizationId,
         projectId: environment.projectId,
+        environmentId: environment.id,
         name: search,
         page: 1,
         pageSize: 50,
+        period: "30d",
       });
       return {
-        tags: tags.tags.map((t) => t.name),
+        tags: tags.tags,
       };
     },
   };