From 1a3204d76a676965e6a5daa80e4dd59d10b35d59 Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 21 Jan 2025 09:06:52 +0100 Subject: [PATCH] Refactor schema loading (#545) --- packages/zudoku/package.json | 6 - packages/zudoku/src/app/demo.tsx | 1 - packages/zudoku/src/app/standalone.tsx | 1 - packages/zudoku/src/lib/oas/graphql/index.ts | 34 +- .../zudoku/src/lib/plugins/openapi-worker.ts | 11 - .../plugins/openapi/client/GraphQLClient.tsx | 148 ++------- .../plugins/openapi/client/createServer.ts | 8 +- .../plugins/openapi/client/useCreateQuery.ts | 19 +- .../src/lib/plugins/openapi/client/worker.ts | 44 --- .../zudoku/src/lib/plugins/openapi/index.tsx | 3 +- packages/zudoku/src/lib/util/traverse.ts | 20 +- .../src/vite/api/schema-codegen.test.ts | 263 ++++++++++++++++ .../zudoku/src/vite/api/schema-codegen.ts | 86 +++++ packages/zudoku/src/vite/config.ts | 7 - packages/zudoku/src/vite/plugin-api.ts | 293 ++++++++++-------- packages/zudoku/src/vite/plugin-component.ts | 1 - packages/zudoku/tsconfig.json | 1 - packages/zudoku/vite.config.ts | 27 +- packages/zudoku/vite.standalone.config.ts | 3 - 19 files changed, 594 insertions(+), 382 deletions(-) delete mode 100644 packages/zudoku/src/lib/plugins/openapi-worker.ts delete mode 100644 packages/zudoku/src/lib/plugins/openapi/client/worker.ts create mode 100644 packages/zudoku/src/vite/api/schema-codegen.test.ts create mode 100644 packages/zudoku/src/vite/api/schema-codegen.ts diff --git a/packages/zudoku/package.json b/packages/zudoku/package.json index 004b18e3..6ca60bf4 100644 --- a/packages/zudoku/package.json +++ b/packages/zudoku/package.json @@ -35,7 +35,6 @@ "./plugins/custom-pages": "./src/lib/plugins/custom-pages/index.ts", "./plugins/search-inkeep": "./src/lib/plugins/search-inkeep/index.ts", "./plugins/api-catalog": "./src/lib/plugins/api-catalog/index.ts", - "./openapi-worker": "./src/lib/plugins/openapi-worker.ts", "./components": "./src/lib/components/index.ts", "./icons": "./src/lib/icons.ts", "./vite": "./src/vite/plugin.ts", @@ -103,10 +102,6 @@ "import": "./lib/zudoku.plugin-search-inkeep.js", "types": "./dist/lib/plugins/search-inkeep/index.d.ts" }, - "./openapi-worker": { - "import": "./lib/zudoku.openapi-worker.js", - "types": "./dist/lib/plugins/openapi-worker.d.ts" - }, "./components": { "import": "./lib/zudoku.components.js", "types": "./dist/lib/components/index.d.ts" @@ -146,7 +141,6 @@ "generate:icon-types": "tsx ./scripts/generate-icon-types.ts", "build:standalone:vite": "vite build --mode standalone --config vite.standalone.config.ts", "build:standalone:html": "cp ./src/app/standalone.html ./standalone/standalone.html && cp ./src/app/demo.html ./standalone/demo.html && cp ./src/app/demo-cdn.html ./standalone/index.html", - "hack:fix-worker-paths": "node ./scripts/hack-worker.mjs", "clean": "tsc --build --clean", "codegen": "graphql-codegen --config ./src/codegen.ts", "test": "vitest run" diff --git a/packages/zudoku/src/app/demo.tsx b/packages/zudoku/src/app/demo.tsx index b2c03213..4ea11456 100644 --- a/packages/zudoku/src/app/demo.tsx +++ b/packages/zudoku/src/app/demo.tsx @@ -48,7 +48,6 @@ const config = { type: "url", input: apiUrl, navigationId: "/", - inMemory: true, }), ], } satisfies ZudokuConfig; diff --git a/packages/zudoku/src/app/standalone.tsx b/packages/zudoku/src/app/standalone.tsx index 924eb82a..3fda8a3f 100644 --- a/packages/zudoku/src/app/standalone.tsx +++ b/packages/zudoku/src/app/standalone.tsx @@ -43,7 +43,6 @@ const config = { type: "url", input: apiUrl!, navigationId: "/", - inMemory: true, }), ], } satisfies ZudokuConfig; diff --git a/packages/zudoku/src/lib/oas/graphql/index.ts b/packages/zudoku/src/lib/oas/graphql/index.ts index 1d0f28f0..262b2f6e 100644 --- a/packages/zudoku/src/lib/oas/graphql/index.ts +++ b/packages/zudoku/src/lib/oas/graphql/index.ts @@ -6,8 +6,6 @@ import { } from "@sindresorhus/slugify"; import { GraphQLJSON, GraphQLJSONObject } from "graphql-type-json"; import { createYoga, type YogaServerOptions } from "graphql-yoga"; -import { LRUCache } from "lru-cache"; -import hashit from "object-hash"; import { HttpMethods, validate, @@ -58,11 +56,10 @@ export const createOperationSlug = ( ); }; -const cache = new LRUCache({ - ttl: 60 * 10 * 1000, - ttlAutopurge: true, - fetchMethod: (_key, _oldValue, { context }) => validate(context as string), -}); +export type SchemaImports = Record< + string, + () => Promise<{ schema: OpenAPIDocument }> +>; const builder = new SchemaBuilder<{ Scalars: { @@ -71,6 +68,7 @@ const builder = new SchemaBuilder<{ }; Context: { schema: OpenAPIDocument; + schemaImports?: SchemaImports; }; }>({}); @@ -441,11 +439,6 @@ const Schema = builder.objectRef("Schema").implement({ }), }); -const loadOpenAPISchema = async (input: NonNullable) => { - const hash = hashit(input); - return await cache.forceFetch(hash, { context: input }); -}; - const SchemaSource = builder.enumType("SchemaType", { values: ["url", "file", "raw"] as const, }); @@ -459,10 +452,21 @@ builder.queryType({ input: t.arg({ type: JSONScalar, required: true }), }, resolve: async (_, args, ctx) => { - const schema = await loadOpenAPISchema(args.input!); - // for easier access of the whole schema in children resolvers - ctx.schema = schema; + let schema: OpenAPIDocument; + if (args.type === "file" && typeof args.input === "string") { + const loadSchema = ctx.schemaImports?.[args.input]; + + if (!loadSchema) { + throw new Error(`No schema loader found for path: ${args.input}`); + } + const module = await loadSchema(); + schema = module.schema; + } else { + schema = await validate(args.input as string); + } + + ctx.schema = schema; return schema; }, }), diff --git a/packages/zudoku/src/lib/plugins/openapi-worker.ts b/packages/zudoku/src/lib/plugins/openapi-worker.ts deleted file mode 100644 index 72a3aa2e..00000000 --- a/packages/zudoku/src/lib/plugins/openapi-worker.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const initializeWorker = () => { - const worker = new SharedWorker( - new URL("./openapi/client/worker.ts", import.meta.url), - { type: "module" }, - ); - // eslint-disable-next-line no-console - worker.onerror = (e) => console.error(e); - worker.port.start(); - - return worker; -}; diff --git a/packages/zudoku/src/lib/plugins/openapi/client/GraphQLClient.tsx b/packages/zudoku/src/lib/plugins/openapi/client/GraphQLClient.tsx index e4f4a8e7..1811e3e9 100644 --- a/packages/zudoku/src/lib/plugins/openapi/client/GraphQLClient.tsx +++ b/packages/zudoku/src/lib/plugins/openapi/client/GraphQLClient.tsx @@ -1,34 +1,16 @@ import type { GraphQLError } from "graphql/error/index.js"; -import { ulid } from "ulidx"; -import { initializeWorker } from "zudoku/openapi-worker"; import { ZudokuError } from "../../../util/invariant.js"; import type { TypedDocumentString } from "../graphql/graphql.js"; import type { OpenApiPluginOptions } from "../index.js"; import type { LocalServer } from "./createServer.js"; -import type { WorkerGraphQLMessage } from "./worker.js"; let localServerPromise: Promise | undefined; -let worker: SharedWorker | undefined; type GraphQLResponse = { errors?: GraphQLError[]; data: TResult; }; -const resolveVariables = async (variables?: unknown) => { - if (!variables) return; - - if ( - typeof variables === "object" && - "type" in variables && - variables.type === "file" && - "input" in variables && - typeof variables.input === "function" - ) { - variables.input = await variables.input(); - } -}; - const throwIfError = (response: GraphQLResponse) => { if (!response.errors?.[0]) return; @@ -39,119 +21,45 @@ const throwIfError = (response: GraphQLResponse) => { }; export class GraphQLClient { - readonly #mode: "remote" | "in-memory" | "worker"; - #pendingRequests = new Map void>(); - #port: MessagePort | undefined; + constructor(private readonly config: OpenApiPluginOptions) {} - constructor(private config: OpenApiPluginOptions) { - if (config.server) { - this.#mode = "remote"; - } else if (config.inMemory || typeof SharedWorker === "undefined") { - this.#mode = "in-memory"; - } else { - this.#mode = "worker"; + #getLocalServer = async () => { + if (!localServerPromise) { + localServerPromise = import("./createServer.js").then((m) => + m.createServer(this.config), + ); } - } + return localServerPromise; + }; - #initializeLocalServer = () => - import("./createServer.js").then((m) => m.createServer()); + #executeFetch = async (init: RequestInit): Promise => { + if (this.config.server) { + return fetch(this.config.server, init); + } + + const localServer = await this.#getLocalServer(); + return localServer.fetch("http://localhost/graphql", init); + }; fetch = async ( query: TypedDocumentString, ...[variables]: TVariables extends Record ? [] : [TVariables] - ) => { + ): Promise => { const operationName = query.match(/query (\w+)/)?.[1]; - await resolveVariables(variables); - - const body = JSON.stringify({ query, variables, operationName }); - - switch (this.#mode) { - case "remote": { - const response = await fetch(this.config.server!, { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - }); - - if (!response.ok) { - throw new Error("Network response was not ok"); - } - - const result = (await response.json()) as GraphQLResponse; - throwIfError(result); - - return result.data; - } - - case "in-memory": { - if (!localServerPromise) { - localServerPromise = this.#initializeLocalServer(); - } - - const localServer = await localServerPromise; - if (!localServer) throw new Error("Local server not initialized"); + const response = await this.#executeFetch({ + method: "POST", + body: JSON.stringify({ query, variables, operationName }), + headers: { "Content-Type": "application/json" }, + }); - const response = await localServer.fetch( - new Request("http://localhost/graphql", { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - }), - ); - - if (!response.ok) { - throw new Error("Network response was not ok"); - } - - const result = (await response.json()) as GraphQLResponse; - throwIfError(result); - - return result.data; - } - - case "worker": { - if (!worker) { - worker = initializeWorker(); - } - - if (!this.#port) { - const channel = new MessageChannel(); - - worker.port.postMessage({ port: channel.port2 }, [channel.port2]); - - this.#port = channel.port1; - - this.#port.onmessage = (e: MessageEvent) => { - const { id, body } = e.data; - const resolve = this.#pendingRequests.get(id); - if (resolve) { - const result = JSON.parse(body); - resolve(result); - this.#pendingRequests.delete(id); - } else { - // eslint-disable-next-line no-console - console.error(`No pending request found for id: ${id}`); - } - }; - - this.#port.start(); - } - - const id = ulid(); - - const resultPromise = new Promise>( - (resolve) => { - this.#pendingRequests.set(id, resolve); - this.#port!.postMessage({ id, body } as WorkerGraphQLMessage); - }, - ); + if (!response.ok) { + throw new Error("Network response was not ok"); + } - const result = await resultPromise; - throwIfError(result); + const result = (await response.json()) as GraphQLResponse; + throwIfError(result); - return result.data; - } - } + return result.data; }; } diff --git a/packages/zudoku/src/lib/plugins/openapi/client/createServer.ts b/packages/zudoku/src/lib/plugins/openapi/client/createServer.ts index 52543bfd..d313f0b9 100644 --- a/packages/zudoku/src/lib/plugins/openapi/client/createServer.ts +++ b/packages/zudoku/src/lib/plugins/openapi/client/createServer.ts @@ -1,13 +1,17 @@ import { useLogger } from "@envelop/core"; import { createGraphQLServer } from "../../../oas/graphql/index.js"; +import type { OpenApiPluginOptions } from "../index.js"; const map = new Map(); /** * Creates the GraphQL server */ -export const createServer = () => +export const createServer = (config: OpenApiPluginOptions) => createGraphQLServer({ + context: { + schemaImports: config.schemaImports, + }, plugins: [ // eslint-disable-next-line react-hooks/rules-of-hooks useLogger({ @@ -22,7 +26,7 @@ export const createServer = () => if (start) { // eslint-disable-next-line no-console console.log( - `${args.operationName} query took ${performance.now() - start}ms`, + `[zudoku:debug] ${args.operationName} query took ${performance.now() - start}ms`, ); map.delete(`${startEvent}-${args.operationName}`); } diff --git a/packages/zudoku/src/lib/plugins/openapi/client/useCreateQuery.ts b/packages/zudoku/src/lib/plugins/openapi/client/useCreateQuery.ts index 9ac87c29..0bb57cea 100644 --- a/packages/zudoku/src/lib/plugins/openapi/client/useCreateQuery.ts +++ b/packages/zudoku/src/lib/plugins/openapi/client/useCreateQuery.ts @@ -1,5 +1,4 @@ -import hashit from "object-hash"; -import { useContext, useMemo } from "react"; +import { useContext } from "react"; import type { TypedDocumentString } from "../graphql/graphql.js"; import { GraphQLContext } from "./GraphQLContext.js"; @@ -12,22 +11,8 @@ export const useCreateQuery = ( throw new Error("useGraphQL must be used within a GraphQLProvider"); } - const hash = useMemo(() => { - if ( - typeof variables[0] === "object" && - variables[0] != null && - "input" in variables[0] && - typeof variables[0].input === "function" - ) { - // This is a pre-hashed name to ensure that the query key is consistent across server and client - return variables[0].input.name; - } - - return hashit(variables[0] ?? {}); - }, [variables]); - return { queryFn: () => graphQLClient.fetch(query, ...variables), - queryKey: [query, hash], + queryKey: [query, variables[0]], } as const; }; diff --git a/packages/zudoku/src/lib/plugins/openapi/client/worker.ts b/packages/zudoku/src/lib/plugins/openapi/client/worker.ts deleted file mode 100644 index 98427afd..00000000 --- a/packages/zudoku/src/lib/plugins/openapi/client/worker.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { createServer } from "./createServer.js"; - -export type WorkerGraphQLMessage = { id: string; body: string }; - -const localServer = createServer(); - -const worker = self as unknown as SharedWorkerGlobalScope; - -worker.addEventListener("connect", (event) => { - const mainPort = event.ports[0]; - - mainPort!.onmessage = (e) => { - if (e.data.port) { - const clientPort = e.data.port as MessagePort; - - clientPort.onmessage = async ( - event: MessageEvent<{ id: string; body: string }>, - ) => { - const { id, body } = event.data; - - const response = await localServer.fetch( - new Request("/__z/graphql", { - method: "POST", - body, - headers: { - "Content-Type": "application/json", - }, - }), - ); - - const responseBody = await response.text(); - - clientPort.postMessage({ - id, - body: responseBody, - } as WorkerGraphQLMessage); - }; - - clientPort.start(); - } - }; - - mainPort!.start(); -}); diff --git a/packages/zudoku/src/lib/plugins/openapi/index.tsx b/packages/zudoku/src/lib/plugins/openapi/index.tsx index d99f0f0e..a6d8b480 100644 --- a/packages/zudoku/src/lib/plugins/openapi/index.tsx +++ b/packages/zudoku/src/lib/plugins/openapi/index.tsx @@ -7,6 +7,7 @@ import { CirclePlayIcon, LogInIcon } from "lucide-react"; import type { SidebarItem } from "../../../config/validators/SidebarSchema.js"; import { useAuth } from "../../authentication/hook.js"; import { ColorMap } from "../../components/navigation/SidebarBadge.js"; +import type { SchemaImports } from "../../oas/graphql/index.js"; import { Button } from "../../ui/Button.js"; import { joinPath } from "../../util/joinPath.js"; import { GraphQLClient } from "./client/GraphQLClient.js"; @@ -35,7 +36,7 @@ const GetCategoriesQuery = graphql(` } `); -type InternalOasPluginConfig = { inMemory?: boolean }; +type InternalOasPluginConfig = { schemaImports?: SchemaImports }; const MethodColorMap: Record = { get: "green", diff --git a/packages/zudoku/src/lib/util/traverse.ts b/packages/zudoku/src/lib/util/traverse.ts index 359b0427..f2f3a6c2 100644 --- a/packages/zudoku/src/lib/util/traverse.ts +++ b/packages/zudoku/src/lib/util/traverse.ts @@ -1,13 +1,23 @@ // eslint-disable-next-line @typescript-eslint/no-explicit-any export type RecordAny = Record; -export const traverse = ( +type JsonPrimitive = string | number | boolean | null; +type JsonArray = JsonValue[]; +type JsonObject = { [key: string]: JsonValue }; +type JsonValue = JsonPrimitive | JsonArray | JsonObject; + +export const traverse = ( specification: RecordAny, - transform: (specification: RecordAny) => RecordAny, + transform: (specification: RecordAny) => T, ) => { - const result: RecordAny = {}; + const transformed = transform(specification); + if (typeof transformed !== "object" || transformed === null) { + return transformed; + } + + const result: RecordAny = Array.isArray(transformed) ? [] : {}; - for (const [key, value] of Object.entries(specification)) { + for (const [key, value] of Object.entries(transformed)) { if (Array.isArray(value)) { result[key] = value.map((item) => typeof item === "object" && item !== null @@ -21,5 +31,5 @@ export const traverse = ( } } - return transform(result); + return result; }; diff --git a/packages/zudoku/src/vite/api/schema-codegen.test.ts b/packages/zudoku/src/vite/api/schema-codegen.test.ts new file mode 100644 index 00000000..89f2394b --- /dev/null +++ b/packages/zudoku/src/vite/api/schema-codegen.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from "vitest"; +import { generateCode } from "./schema-codegen.js"; + +const executeCode = (code: string) => { + const encodedCode = encodeURIComponent(code); + return import(`data:text/javascript,${encodedCode}`); +}; + +describe("Generate OpenAPI schema module", () => { + it("should handle basic schema refs", async () => { + const input = { + components: { + schemas: { + Pet: { + type: "object", + properties: { + name: { type: "string" }, + }, + }, + Error: { + type: "object", + properties: { + code: { type: "integer" }, + message: { type: "string" }, + }, + }, + }, + }, + paths: { + "/pets": { + get: { + responses: { + "200": { + content: { + "application/json": { + schema: { $ref: "#/components/schemas/Pet" }, + }, + }, + }, + default: { + content: { + "application/json": { + schema: { $ref: "#/components/schemas/Error" }, + }, + }, + }, + }, + }, + }, + }, + }; + + const { schema } = await executeCode(await generateCode(input)); + const successSchema = + schema.paths["/pets"].get.responses["200"].content["application/json"] + .schema; + const errorSchema = + schema.paths["/pets"].get.responses["default"].content["application/json"] + .schema; + + expect(successSchema).toStrictEqual(input.components.schemas.Pet); + expect(errorSchema).toStrictEqual(input.components.schemas.Error); + }); + + it("should handle circular refs", async () => { + const input = { + components: { + schemas: { + Person: { + type: "object", + properties: { + name: { type: "string" }, + bestFriend: { $ref: "#/components/schemas/Person" }, + pets: { + type: "array", + items: { $ref: "#/components/schemas/Pet" }, + }, + }, + }, + Pet: { + type: "object", + properties: { + name: { type: "string" }, + owner: { $ref: "#/components/schemas/Person" }, + }, + }, + }, + }, + }; + + const code = await generateCode(input); + const { schema } = await executeCode(code); + const person = schema.components.schemas.Person; + const pet = schema.components.schemas.Pet; + + expect(person.properties.bestFriend).toStrictEqual(person); + expect(person.properties.pets.items).toStrictEqual(pet); + expect(pet.properties.owner).toStrictEqual(person); + }); + + it("should handle composition through refs", async () => { + const input = { + components: { + schemas: { + Pet: { + type: "object", + allOf: [ + { $ref: "#/components/schemas/Animal" }, + { $ref: "#/components/schemas/Named" }, + ], + }, + Animal: { + type: "object", + properties: { + species: { type: "string" }, + }, + }, + Named: { + type: "object", + properties: { + name: { type: "string" }, + }, + }, + }, + }, + }; + + const { schema } = await executeCode(await generateCode(input)); + const pet = schema.components.schemas.Pet; + + expect(pet.allOf[0]).toStrictEqual(input.components.schemas.Animal); + expect(pet.allOf[1]).toStrictEqual(input.components.schemas.Named); + }); + + it("should handle discriminated unions with refs", async () => { + const input = { + components: { + schemas: { + Pet: { + oneOf: [ + { $ref: "#/components/schemas/Cat" }, + { $ref: "#/components/schemas/Dog" }, + ], + discriminator: { propertyName: "petType" }, + }, + Cat: { + type: "object", + properties: { + petType: { type: "string", enum: ["cat"] }, + purrs: { type: "boolean" }, + }, + }, + Dog: { + type: "object", + properties: { + petType: { type: "string", enum: ["dog"] }, + barks: { type: "boolean" }, + }, + }, + }, + }, + }; + + const { schema } = await executeCode(await generateCode(input)); + const pet = schema.components.schemas.Pet; + + expect(pet.oneOf[0]).toStrictEqual(input.components.schemas.Cat); + expect(pet.oneOf[1]).toStrictEqual(input.components.schemas.Dog); + }); + + it("should generate correct code for circular refs", async () => { + const input = { + definitions: { + child: { + title: "child", + type: "object", + properties: { + name: { type: "string" }, + pet: { $ref: "#/definitions/pet" }, + }, + }, + // self-reference + thing: { $ref: "#/definitions/thing" }, + pet: { + title: "pet", + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + species: { type: "string", enum: ["cat", "dog", "bird", "fish"] }, + }, + }, + }, + }; + + const code = await generateCode(input); + expect(code).toMatchInlineSnapshot(` + "const __refs = Array.from({ length: 2 }, () => ({})); + const __refMap = { + "#/definitions/pet": __refs[0], + "#/definitions/thing": __refs[1] + }; + Object.assign(__refs[0], { + "title": "pet", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "age": { + "type": "number" + }, + "species": { + "type": "string", + "enum": [ + "cat", + "dog", + "bird", + "fish" + ] + } + } + }); + Object.assign(__refs[1], __refMap["#/definitions/thing"]); + export const schema = { + "definitions": { + "child": { + "title": "child", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "pet": __refMap["#/definitions/pet"] + } + }, + "thing": __refMap["#/definitions/thing"], + "pet": { + "title": "pet", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "age": { + "type": "number" + }, + "species": { + "type": "string", + "enum": [ + "cat", + "dog", + "bird", + "fish" + ] + } + } + } + } + };" + `); + }); +}); diff --git a/packages/zudoku/src/vite/api/schema-codegen.ts b/packages/zudoku/src/vite/api/schema-codegen.ts new file mode 100644 index 00000000..3d59d101 --- /dev/null +++ b/packages/zudoku/src/vite/api/schema-codegen.ts @@ -0,0 +1,86 @@ +import { type RecordAny, traverse } from "../../lib/util/traverse.js"; + +// Find all $ref occurrences in the schema and assign them unique variable names +const createLocalRefMap = (obj: RecordAny) => { + const refMap = new Map(); + let refCounter = 0; + + traverse(obj, (node) => { + if (typeof node.$ref === "string" && node.$ref.startsWith("#/")) { + if (!refMap.has(node.$ref)) { + refMap.set(node.$ref, refCounter++); + } + } + return node; + }); + + return refMap; +}; + +// Replace all $ref occurrences with a special marker that will be transformed into a reference to the __refMap lookup +const setRefMarkers = (obj: RecordAny, refMap: Map) => + traverse(obj, (node) => { + if (node.$ref && typeof node.$ref === "string" && refMap.has(node.$ref)) { + return `__refMap:${node.$ref}`; + } + return node; + }); + +// Replace the marker strings with actual __refMap lookups in the generated code +const replaceRefMarkers = (code: string) => + code.replace(/"__refMap:(.*?)"/g, '__refMap["$1"]'); + +const lookup = (schema: RecordAny, path: string) => { + const parts = path.split("/").slice(1); + let value = schema; + + for (const part of parts) { + value = value[part]; + } + + return value; +}; + +/** + * Generate JavaScript code that exports the schema with all references resolved as variables. + * + * Handles circular references by: + * 1. Creating empty objects first to establish the references + * 2. Adding these objects to the reference map + * 3. Using Object.assign to populate their properties later + * + * This ensures object identity throughout the circular references. + */ +export const generateCode = async (schema: RecordAny) => { + const refMap = createLocalRefMap(schema); + const lines: string[] = []; + + const str = (obj: unknown) => JSON.stringify(obj, null, 2); + + lines.push( + `const __refs = Array.from({ length: ${refMap.size} }, () => ({}));`, + ); + + lines.push( + "const __refMap = {", + Array.from(refMap) + .map(([refPath, index]) => ` "${refPath}": __refs[${index}]`) + .join(",\n"), + "};", + ); + + for (const [refPath, index] of refMap) { + const value = lookup(schema, refPath); + const transformedValue = setRefMarkers(value, refMap); + + lines.push( + // Use assign so that the object identity is maintained and correctly resolves circular references + `Object.assign(__refs[${index}], ${replaceRefMarkers(str(transformedValue))});`, + ); + } + + const transformed = setRefMarkers(schema, refMap); + lines.push(`export const schema = ${replaceRefMarkers(str(transformed))};`); + + return lines.join("\n"); +}; diff --git a/packages/zudoku/src/vite/config.ts b/packages/zudoku/src/vite/config.ts index 6333bd90..56d200ff 100644 --- a/packages/zudoku/src/vite/config.ts +++ b/packages/zudoku/src/vite/config.ts @@ -209,9 +209,6 @@ export async function getViteConfig( logLevel: (process.env.LOG_LEVEL ?? "info") as LogLevel, customLogger: logger, envPrefix, - worker: { - format: "es", - }, resolve: { alias: { "@mdx-js/react": path.resolve( @@ -282,10 +279,6 @@ export async function getViteConfig( : getAppClientEntryPath(), ], include: ["react-dom/client", "@sentry/react"], - exclude: [ - // Vite does not like optimizing the worker dependency - "zudoku/openapi-worker", - ], }, // Workaround for Pre-transform error for "virtual" file: https://github.com/vitejs/vite/issues/15374 assetsInclude: ["/__z/entry.client.tsx"], diff --git a/packages/zudoku/src/vite/plugin-api.ts b/packages/zudoku/src/vite/plugin-api.ts index 3867cc6f..35c3108e 100644 --- a/packages/zudoku/src/vite/plugin-api.ts +++ b/packages/zudoku/src/vite/plugin-api.ts @@ -1,27 +1,111 @@ import fs from "node:fs/promises"; import path from "node:path"; -import hashit from "object-hash"; import { type Plugin } from "vite"; import yaml from "yaml"; import { type ZudokuPluginOptions } from "../config/config.js"; -import { validate } from "../lib/oas/parser/index.js"; +import { upgradeSchema } from "../lib/oas/parser/upgrade/index.js"; import type { ApiCatalogItem, ApiCatalogPluginOptions, } from "../lib/plugins/api-catalog/index.js"; +import { generateCode } from "./api/schema-codegen.js"; + +type ProcessedSchema = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + schema: any; + version: string; + inputPath: string; +}; + +const schemaMap = new Map(); + +async function processSchemas( + config: ZudokuPluginOptions, +): Promise> { + const tmpDir = path.posix.join( + config.rootDir, + "node_modules/.zudoku/processed", + ); + await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.mkdir(tmpDir, { recursive: true }); + + if (!config.apis) return {}; + + const apis = Array.isArray(config.apis) ? config.apis : [config.apis]; + const processedSchemas: Record = {}; + + for (const apiConfig of apis) { + if (apiConfig.type !== "file" || !apiConfig.navigationId) { + continue; + } + + const postProcessors = apiConfig.postProcessors ?? []; + postProcessors.unshift(upgradeSchema); + + const inputs = Array.isArray(apiConfig.input) + ? apiConfig.input + : [apiConfig.input]; + + const inputFiles = await Promise.all( + inputs.map(async (input) => + /\.ya?ml$/.test(input) + ? yaml.parse(await fs.readFile(input, "utf-8")) + : JSON.parse(await fs.readFile(input, "utf-8")), + ), + ); + + const processedInputs = await Promise.all( + inputFiles.map(async (schema, index) => { + const processedSchema = await postProcessors.reduce( + async (acc, postProcessor) => postProcessor(await acc), + schema, + ); + + const inputPath = inputs[index]!; + const processedPath = path.posix.join( + tmpDir, + `${path.basename(inputPath)}.js`, + ); + + const code = await generateCode(processedSchema); + await fs.writeFile(processedPath, code); + schemaMap.set(inputPath, processedPath); + + return { + schema: processedSchema, + version: processedSchema.info.version || "default", + inputPath, + } satisfies ProcessedSchema; + }), + ); + + if (processedInputs.length === 0) { + throw new Error("No schema found"); + } + + processedSchemas[apiConfig.navigationId] = processedInputs; + } + + return processedSchemas; +} const viteApiPlugin = (getConfig: () => ZudokuPluginOptions): Plugin => { const virtualModuleId = "virtual:zudoku-api-plugins"; const resolvedVirtualModuleId = "\0" + virtualModuleId; + let processedSchemas: Awaited>; + return { name: "zudoku-api-plugins", + async buildStart() { + processedSchemas = await processSchemas(getConfig()); + }, resolveId(id) { if (id === virtualModuleId) { return resolvedVirtualModuleId; } }, - async load(id, options) { + async load(id) { if (id === resolvedVirtualModuleId) { const config = getConfig(); @@ -42,148 +126,113 @@ const viteApiPlugin = (getConfig: () => ZudokuPluginOptions): Plugin => { if (config.apis) { const apis = Array.isArray(config.apis) ? config.apis : [config.apis]; - const catalogs = Array.isArray(config.catalogs) - ? config.catalogs - : [config.catalogs]; - - const categories = apis - .flatMap((api) => api.categories ?? []) - .reduce((acc, catalog) => { - if (!acc.has(catalog.label)) { - acc.set(catalog.label, new Set(catalog.tags)); - } - for (const tag of catalog.tags) { - acc.get(catalog.label)?.add(tag); - } - return acc; - }, new Map>()); - - const tmpDir = path.posix.join( - config.rootDir, - "node_modules/.zudoku/processed", - ); - await fs.rm(tmpDir, { recursive: true, force: true }); - await fs.mkdir(tmpDir, { recursive: true }); - const apiMetadata: ApiCatalogItem[] = []; - for (const apiConfig of apis) { - if (apiConfig.type === "file") { - const postProcessors = apiConfig.postProcessors ?? []; - const inputs = Array.isArray(apiConfig.input) - ? apiConfig.input - : [apiConfig.input]; - - const inputFiles = await Promise.all( - inputs.map(async (input) => - /\.ya?ml$/.test(input) - ? yaml.parse(await fs.readFile(input, "utf-8")) - : JSON.parse(await fs.readFile(input, "utf-8")), - ), - ); + const versionMaps: Record> = {}; - const processedSchemas = await Promise.all( - inputFiles - .map((schema) => - postProcessors.reduce( - async (acc, postProcessor) => postProcessor(await acc), - schema, - ), - ) - .map(async (schema) => await validate(schema)), - ); - - const latestSchema = processedSchemas.at(0); + for (const apiConfig of apis) { + if (apiConfig.type === "file" && apiConfig.navigationId) { + const schemas = processedSchemas[apiConfig.navigationId]; + if (!schemas?.length) continue; - if (!latestSchema) { - throw new Error("No schema found"); - } + const latestSchema = schemas[0]?.schema; + if (!latestSchema?.info) continue; - if (apiConfig.navigationId) { - apiMetadata.push({ - path: apiConfig.navigationId, - label: latestSchema.info.title, - description: latestSchema.info.description ?? "", - categories: apiConfig.categories ?? [], - }); - } - - const processedFilePaths = inputs.map((input) => - path.posix.join(tmpDir, `${path.basename(input)}.json`), - ); + apiMetadata.push({ + path: apiConfig.navigationId, + label: latestSchema.info.title, + description: latestSchema.info.description ?? "", + categories: apiConfig.categories ?? [], + }); const versionMap = Object.fromEntries( - processedSchemas.map((schema, index) => [ - schema.info.version || "default", - processedFilePaths[index], + schemas.map((processed) => [ + processed.version, + processed.inputPath, ]), ); - if (Object.keys(versionMap).length === 0) { - throw new Error("No schema versions found"); + if (Object.keys(versionMap).length > 0) { + versionMaps[apiConfig.navigationId] = versionMap; } + } + } - await Promise.all( - processedSchemas.map((schema, i) => { - if (!processedFilePaths[i]) { - throw new Error("No processed file path found"); - } - fs.writeFile(processedFilePaths[i], JSON.stringify(schema)); - }), - ); + // Generate API plugin code + for (const apiConfig of apis) { + if (apiConfig.type === "file") { + if ( + !apiConfig.navigationId || + !versionMaps[apiConfig.navigationId] + ) { + continue; + } code.push( "configuredApiPlugins.push(openApiPlugin({", - ' type: "file",', - ` input: {${Object.entries(versionMap) - .map( - ([version, path]) => - // The function name is a hash of the file name to ensure that each function has a unique and consistent identifier - // We use this hash when creating a GraphQL query to ensure that the query key is consistent across server and client - `"${version}": function _${hashit(path!)}() { return import("${path}"); }`, - ) - .join(",")}},`, - ` navigationId: "${apiConfig.navigationId}",`, + ` type: "file",`, + ` input: ${JSON.stringify(versionMaps[apiConfig.navigationId])},`, + ` navigationId: ${JSON.stringify(apiConfig.navigationId)},`, + ` schemaImports: {`, + ...Array.from(schemaMap.entries()).map( + ([key, schemaPath]) => + ` "${key}": () => import("${schemaPath}"),`, + ), + ` },`, "}));", ); } else { code.push( - `// @ts-ignore`, // To make tests pass - `configuredApiPlugins.push(openApiPlugin(${JSON.stringify({ - ...apiConfig, - inMemory: options?.ssr ?? config.mode === "internal", - })}));`, + `configuredApiPlugins.push(openApiPlugin(${JSON.stringify(apiConfig)}));`, ); } } - const categoryList = Array.from(categories.entries()).map( - ([label, tags]) => ({ - label, - tags: Array.from(tags), - }), - ); - - for (let i = 0; i < catalogs.length; i++) { - const catalog = catalogs[i]; - if (!catalog) { - continue; - } - const apiCatalogConfig: ApiCatalogPluginOptions = { - ...catalog, - items: apiMetadata, - label: catalog.label, - categories: categoryList, - filterCatalogItems: catalog.filterItems, - }; - - code.push( - `configuredApiCatalogPlugins.push(apiCatalogPlugin({`, - ` ...${JSON.stringify(apiCatalogConfig, null, 2)},`, - ` filterCatalogItems: Array.isArray(config.catalogs)`, - ` ? config.catalogs[${i}].filterItems`, - ` : config.catalogs.filterItems,`, - `}));`, + if (config.catalogs) { + const catalogs = Array.isArray(config.catalogs) + ? config.catalogs + : [config.catalogs]; + + const categories = apis + .flatMap((api) => api.categories ?? []) + .reduce((acc, catalog) => { + if (!acc.has(catalog.label)) { + acc.set(catalog.label ?? "", new Set(catalog.tags)); + } + for (const tag of catalog.tags) { + acc.get(catalog.label ?? "")?.add(tag); + } + return acc; + }, new Map>()); + + const categoryList = Array.from(categories.entries()).map( + ([label, tags]) => ({ + label, + tags: Array.from(tags), + }), ); + + for (let i = 0; i < catalogs.length; i++) { + const catalog = catalogs[i]; + if (!catalog) { + continue; + } + const apiCatalogConfig: ApiCatalogPluginOptions = { + ...catalog, + items: apiMetadata, + label: catalog.label, + categories: categoryList, + filterCatalogItems: catalog.filterItems, + }; + + code.push( + `configuredApiCatalogPlugins.push(apiCatalogPlugin({`, + ` ...${JSON.stringify(apiCatalogConfig, null, 2)},`, + ` filterCatalogItems: Array.isArray(config.catalogs)`, + ` ? config.catalogs[${i}].filterItems`, + ` : config.catalogs.filterItems,`, + `}));`, + ); + } } } diff --git a/packages/zudoku/src/vite/plugin-component.ts b/packages/zudoku/src/vite/plugin-component.ts index 708147e8..d1394228 100644 --- a/packages/zudoku/src/vite/plugin-component.ts +++ b/packages/zudoku/src/vite/plugin-component.ts @@ -9,7 +9,6 @@ const viteAliasPlugin = (getConfig: () => ZudokuPluginOptions): Plugin => { const config = getConfig(); const replacements = [ - ["zudoku/openapi-worker", "src/lib/plugins/openapi-worker.ts"], ["zudoku/components", "src/lib/components/index.ts"], ["zudoku/plugins/openapi", "src/lib/plugins/openapi/index.tsx"], ["zudoku/plugins/api-catalog", "src/lib/plugins/api-catalog/index.tsx"], diff --git a/packages/zudoku/tsconfig.json b/packages/zudoku/tsconfig.json index dd762eea..df3f8781 100644 --- a/packages/zudoku/tsconfig.json +++ b/packages/zudoku/tsconfig.json @@ -22,7 +22,6 @@ "jsx": "react-jsx", "types": ["vite/client"], "paths": { - "zudoku/openapi-worker": ["./src/lib/plugins/openapi-worker.ts"], "zudoku/components": ["./src/lib/components/index.ts"], "zudoku/ui/*": ["./src/lib/ui/*"], "zudoku/plugins/openapi": ["./src/lib/plugins/openapi/index.ts"], diff --git a/packages/zudoku/vite.config.ts b/packages/zudoku/vite.config.ts index cee4845f..805956e8 100644 --- a/packages/zudoku/vite.config.ts +++ b/packages/zudoku/vite.config.ts @@ -1,7 +1,7 @@ import { glob } from "glob"; import path from "path"; import { visualizer } from "rollup-plugin-visualizer"; -import { defineConfig, type Plugin } from "vite"; +import { defineConfig } from "vite"; import pkgJson from "./package.json"; const entries: Record = { @@ -10,7 +10,6 @@ const entries: Record = { "auth-clerk": "./src/lib/authentication/providers/clerk.tsx", "auth-auth0": "./src/lib/authentication/providers/auth0.tsx", "auth-openid": "./src/lib/authentication/providers/openid.tsx", - "openapi-worker": "./src/lib/plugins/openapi-worker.ts", "plugin-api-keys": "./src/lib/plugins/api-keys/index.tsx", "plugin-markdown": "./src/lib/plugins/markdown/index.tsx", "plugin-openapi": "./src/lib/plugins/openapi/index.tsx", @@ -26,9 +25,6 @@ const entries: Record = { }; export default defineConfig({ - worker: { - format: "es", - }, resolve: { alias: [ { find: /^zudoku\/ui\/(.*)\.js/, replacement: `./src/lib/ui/$1.tsx` }, @@ -61,12 +57,8 @@ export default defineConfig({ // want to bundle these in the library. Users will install these // themselves and they will be bundled in their app ...Object.keys(pkgJson.optionalDependencies), - - // This is here because otherwise it tries to resolve at build time - // we only want this to be resolved when the end app gets built - "zudoku/openapi-worker", ], - plugins: [visualizer(), fixWorkerPathsPlugin()], + plugins: [visualizer()], onwarn(warning, warn) { // Suppress "Module level directives cause errors when bundled" warnings if ( @@ -81,21 +73,6 @@ export default defineConfig({ }, }); -// Fixes the worker import paths -// See: https://github.com/vitejs/vite/issues/15618 -function fixWorkerPathsPlugin(): Plugin { - return { - name: "fix-worker-paths", - apply: "build", - generateBundle(_, bundle) { - Object.values(bundle).forEach((chunk) => { - if (chunk.type === "chunk" && chunk.fileName.endsWith(".js")) { - chunk.code = chunk.code.replaceAll('"/assets/', '"./assets/'); - } - }); - }, - }; -} // Globs files and returns all entries without file extension in a given folder function globEntries(globString: string, distSubFolder = "") { return Object.fromEntries( diff --git a/packages/zudoku/vite.standalone.config.ts b/packages/zudoku/vite.standalone.config.ts index d98f9b69..fec6b68d 100644 --- a/packages/zudoku/vite.standalone.config.ts +++ b/packages/zudoku/vite.standalone.config.ts @@ -20,9 +20,6 @@ const config = { }; export default defineConfig({ - worker: { - format: "es", - }, define: { "process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV), },