Skip to content

Commit 73a1a90

Browse files
teamchongteamchong
committed
feat(typescript): generate SDKs across worker shards
1 parent c1c64e3 commit 73a1a90

14 files changed

Lines changed: 1078 additions & 7 deletions

File tree

‎generators/typescript/sdk/cli/package.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
"compile:debug": "tsc --sourceMap",
2828
"depcheck": "knip --config ../../../../knip.json --no-config-hints",
2929
"dist:cli": "node build.mjs",
30+
"test": "vitest --run",
31+
"test:debug": "pnpm run test --inspect --no-file-parallelism",
3032
"dockerTagLatest": "docker build -f ./Dockerfile -t fernapi/fern-typescript-node-sdk:latest -t fernapi/fern-typescript-sdk:latest ../../../..",
3133
"podmanTagLatest": "podman build -f ./Dockerfile -t fernapi/fern-typescript-node-sdk:latest -t fernapi/fern-typescript-sdk:latest ../../../.."
3234
},
@@ -45,6 +47,8 @@
4547
"@fern-typescript/contexts": "workspace:*",
4648
"@fern-typescript/sdk-generator": "workspace:*",
4749
"@types/node": "catalog:",
48-
"typescript": "catalog:"
50+
"ts-morph": "catalog:",
51+
"typescript": "catalog:",
52+
"vitest": "catalog:"
4953
}
5054
}

‎generators/typescript/sdk/cli/src/SdkGeneratorCli.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,26 @@ import {
1515
writeTemplateFiles
1616
} from "@fern-typescript/commons";
1717
import { GeneratorContext } from "@fern-typescript/contexts";
18-
import { SdkGenerator } from "@fern-typescript/sdk-generator";
18+
import { GenerationShard, SdkGenerator } from "@fern-typescript/sdk-generator";
1919
import { copyFile } from "fs/promises";
2020
import path from "path";
2121
import { SdkCustomConfig } from "./custom-config/SdkCustomConfig.js";
2222
import { SdkCustomConfigSchema } from "./custom-config/schema/SdkCustomConfigSchema.js";
2323
export declare namespace SdkGeneratorCli {
2424
export interface Init {
2525
configOverrides?: Partial<SdkCustomConfig>;
26+
generationShard?: GenerationShard;
2627
}
2728
}
2829

2930
export class SdkGeneratorCli extends AbstractGeneratorCli<SdkCustomConfig> {
3031
private configOverrides: Partial<SdkCustomConfig>;
32+
private generationShard: GenerationShard | undefined;
3133

32-
constructor({ configOverrides }: SdkGeneratorCli.Init = {}) {
34+
constructor({ configOverrides, generationShard }: SdkGeneratorCli.Init = {}) {
3335
super();
3436
this.configOverrides = configOverrides ?? {};
37+
this.generationShard = generationShard;
3538
}
3639

3740
protected parseCustomConfig(customConfig: unknown, logger: Logger): SdkCustomConfig {
@@ -274,7 +277,8 @@ export class SdkGeneratorCli extends AbstractGeneratorCli<SdkCustomConfig> {
274277
maxRetries: customConfig.maxRetries,
275278
alwaysSendAuth: customConfig.alwaysSendAuth,
276279
generateReactQueryHooks: customConfig.generateReactQueryHooks
277-
}
280+
},
281+
generationShard: this.generationShard
278282
});
279283
const typescriptProject = await sdkGenerator.generate();
280284
const persistedTypescriptProject = await typescriptProject.persist();
@@ -346,7 +350,7 @@ export class SdkGeneratorCli extends AbstractGeneratorCli<SdkCustomConfig> {
346350
_customConfig: SdkCustomConfig
347351
): Promise<void> {
348352
const customConfig = this.customConfigWithOverrides(_customConfig);
349-
if (customConfig.useLegacyExports === false) {
353+
if (customConfig.useLegacyExports === false && (this.generationShard?.count ?? 1) <= 1) {
350354
await fixImportsForEsm(persistedTypescriptProject.getRootDirectory());
351355
}
352356
if (customConfig.testFramework === "vitest") {

‎generators/typescript/sdk/cli/src/cli.ts‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,43 @@
11
import { SdkGeneratorCli } from "./SdkGeneratorCli.js";
2+
import { mergeShardOutputs } from "./sharding/mergeShardOutputs.js";
3+
import { runShardedGeneration } from "./sharding/runShardedGeneration.js";
24

3-
void new SdkGeneratorCli().runCli();
5+
async function main(): Promise<void> {
6+
if (process.argv[2] === "--internal-sharded-generation") {
7+
const [configPath, countValue, heapValue] = process.argv.slice(3);
8+
if (configPath == null || countValue == null) {
9+
throw new Error("--internal-sharded-generation requires a config path and shard count");
10+
}
11+
await runShardedGeneration({
12+
configPath,
13+
shardCount: Number(countValue),
14+
maxOldSpaceSizeMb: heapValue == null ? undefined : Number(heapValue)
15+
});
16+
} else if (process.argv[2] === "--internal-shard-worker") {
17+
const [configPath, countValue, indexValue] = process.argv.slice(3);
18+
if (configPath == null || countValue == null || indexValue == null) {
19+
throw new Error("--internal-shard-worker requires a config path, shard count, and shard index");
20+
}
21+
await new SdkGeneratorCli({ generationShard: { count: Number(countValue), index: Number(indexValue) } }).run(
22+
configPath,
23+
{
24+
disableNotifications: true,
25+
unzipOutput: true
26+
}
27+
);
28+
} else if (process.argv[2] === "--internal-merge-shards") {
29+
const [outputDirectory, ...shardDirectories] = process.argv.slice(3);
30+
if (outputDirectory == null || shardDirectories.length === 0) {
31+
throw new Error("--internal-merge-shards requires an output directory and at least one shard directory");
32+
}
33+
await mergeShardOutputs({ outputDirectory, shardDirectories });
34+
} else {
35+
await new SdkGeneratorCli().runCli();
36+
}
37+
}
38+
39+
void main().catch((error) => {
40+
// biome-ignore lint/suspicious/noConsole: report fatal CLI errors
41+
console.error(error);
42+
process.exitCode = 1;
43+
});

‎generators/typescript/sdk/cli/src/sharding/__test__/mergeShardOutputs.test.ts‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
2+
import { tmpdir } from "node:os";
3+
import { dirname, join } from "node:path";
4+
import { afterEach, describe, expect, it } from "vitest";
5+
import { mergeShardOutputs } from "../mergeShardOutputs.js";
6+
7+
const temporaryDirectories: string[] = [];
8+
9+
afterEach(async () => {
10+
await Promise.all(
11+
temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))
12+
);
13+
});
14+
15+
describe("mergeShardOutputs", () => {
16+
it("merges distributed aggregate barrels and fixes ESM imports", async () => {
17+
const { output, shards, write } = await createFixture();
18+
for (const shard of shards) {
19+
await write(shard, "api/resources/index.ts", "");
20+
}
21+
await write(shards[0], "api/index.ts", 'export * from "./errors/index";\n');
22+
await write(shards[1], "api/index.ts", 'export * from "./types/index";\n');
23+
await write(shards[0], "api/errors/index.ts", "");
24+
await write(shards[0], "api/types/index.ts", 'export * from "./Account";\n');
25+
await write(shards[1], "api/types/index.ts", 'export * from "./Zone";\n');
26+
await write(shards[0], "api/types/Account.ts", "export interface Account {}\n");
27+
await write(shards[1], "api/types/Zone.ts", "export interface Zone {}\n");
28+
await write(
29+
shards[0],
30+
"Client.ts",
31+
'export { Account } from "./api/types/Account";\nimport "./api/errors";\nexport const account = import("./api/types/Account.ts");\n'
32+
);
33+
34+
const result = await mergeShardOutputs({
35+
outputDirectory: output,
36+
shardDirectories: shards
37+
});
38+
39+
expect(result.fileCount).toBe(7);
40+
await expect(readFile(join(output, "api/index.ts"), "utf8")).resolves.toBe(
41+
'export * from "./errors/index.js";\nexport * from "./types/index.js";\n'
42+
);
43+
await expect(readFile(join(output, "api/types/index.ts"), "utf8")).resolves.toBe(
44+
'export * from "./Account.js";\nexport * from "./Zone.js";\n'
45+
);
46+
await expect(readFile(join(output, "Client.ts"), "utf8")).resolves.toBe(
47+
'export { Account } from "./api/types/Account.js";\nimport "./api/errors/index.js";\nexport const account = import("./api/types/Account.js");\n'
48+
);
49+
await expect(readFile(join(shards[0], "Client.ts"), "utf8")).resolves.toBe(
50+
'export { Account } from "./api/types/Account";\nimport "./api/errors";\nexport const account = import("./api/types/Account.ts");\n'
51+
);
52+
});
53+
54+
it("rejects case-only path collisions", async () => {
55+
const { output, shards, write } = await createFixture();
56+
await write(shards[0], "api/resources/oAuthClients/exports.ts", "");
57+
await write(shards[1], "api/resources/oauthClients/exports.ts", "");
58+
59+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
60+
"Case-only shard path collision"
61+
);
62+
});
63+
64+
it("rejects conflicting non-aggregate files", async () => {
65+
const { output, shards, write } = await createFixture();
66+
await write(shards[0], "Client.ts", "export const shard = 0;\n");
67+
await write(shards[1], "Client.ts", "export const shard = 1;\n");
68+
69+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
70+
"Conflicting shard file: Client.ts"
71+
);
72+
});
73+
74+
it("accepts byte-identical shared files", async () => {
75+
const { output, shards, write } = await createFixture();
76+
await write(shards[0], "package.json", '{"name":"example"}\n');
77+
await write(shards[1], "package.json", '{"name":"example"}\n');
78+
79+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).resolves.toEqual({
80+
fileCount: 1
81+
});
82+
});
83+
84+
it("produces deterministic aggregate ordering regardless of shard order", async () => {
85+
const { output, shards, write } = await createFixture();
86+
await write(shards[0], "api/resources/index.ts", 'export * as zones from "./zones/index.js";\n');
87+
await write(shards[1], "api/resources/index.ts", 'export * as accounts from "./accounts/index.js";\n');
88+
await write(shards[0], "api/resources/zones/index.ts", "");
89+
await write(shards[1], "api/resources/accounts/index.ts", "");
90+
91+
await mergeShardOutputs({
92+
outputDirectory: output,
93+
shardDirectories: [...shards].reverse()
94+
});
95+
96+
await expect(readFile(join(output, "api/resources/index.ts"), "utf8")).resolves.toBe(
97+
'export * as accounts from "./accounts/index.js";\nexport * as zones from "./zones/index.js";\n'
98+
);
99+
});
100+
101+
it("does not rewrite import-like text in strings or comments", async () => {
102+
const { output, shards, write } = await createFixture();
103+
const source = 'const example = \'import "./Missing"\';\n// export * from "./Missing";\n';
104+
await write(shards[0], "example.ts", source);
105+
106+
await mergeShardOutputs({
107+
outputDirectory: output,
108+
shardDirectories: shards
109+
});
110+
111+
await expect(readFile(join(output, "example.ts"), "utf8")).resolves.toBe(source);
112+
});
113+
114+
it("rejects declarations in distributed barrels", async () => {
115+
const { output, shards, write } = await createFixture();
116+
await write(shards[0], "api/types/index.ts", "export const shard = 0;\n");
117+
await write(shards[1], "api/types/index.ts", "export const shard = 1;\n");
118+
119+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
120+
"Non-export content in shard barrel"
121+
);
122+
});
123+
124+
it("validates and normalizes a barrel from one shard", async () => {
125+
const { output, shards, write } = await createFixture();
126+
await write(shards[0], "api/types/index.ts", 'export * from "./Account";\r\n\r\n');
127+
await write(shards[0], "api/types/Account.ts", "export interface Account {}\n");
128+
129+
await mergeShardOutputs({ outputDirectory: output, shardDirectories: shards });
130+
131+
await expect(readFile(join(output, "api/types/index.ts"), "utf8")).resolves.toBe(
132+
'export * from "./Account.js";\n'
133+
);
134+
});
135+
136+
it("rejects declarations in byte-identical barrels", async () => {
137+
const { output, shards, write } = await createFixture();
138+
for (const shard of shards) {
139+
await write(shard, "api/types/index.ts", "export const invalid = true;\n");
140+
}
141+
142+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
143+
"Non-export content in shard barrel"
144+
);
145+
});
146+
147+
it("rejects declarations in a resource barrel from one shard", async () => {
148+
const { output, shards, write } = await createFixture();
149+
await write(shards[0], "api/resources/index.ts", "export const invalid = true;\n");
150+
151+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
152+
"Non-export content in shard barrel"
153+
);
154+
});
155+
156+
it("rejects unresolved relative imports", async () => {
157+
const { output, shards, write } = await createFixture();
158+
await write(shards[0], "Client.ts", 'export { Missing } from "./Missing.js";\n');
159+
160+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow(
161+
/Unresolved Fern ESM specifiers.*Client\.ts: \.\/Missing\.js/s
162+
);
163+
});
164+
165+
it("preserves existing output when merge validation fails", async () => {
166+
const { output, shards, write } = await createFixture();
167+
await mkdir(output, { recursive: true });
168+
await writeFile(join(output, "existing.txt"), "previous output\n");
169+
await write(shards[0], "Client.ts", "export const shard = 0;\n");
170+
await write(shards[1], "Client.ts", "export const shard = 1;\n");
171+
172+
await expect(mergeShardOutputs({ outputDirectory: output, shardDirectories: shards })).rejects.toThrow();
173+
await expect(readFile(join(output, "existing.txt"), "utf8")).resolves.toBe("previous output\n");
174+
});
175+
176+
it("rejects overlapping output and shard paths", async () => {
177+
const { shards } = await createFixture();
178+
179+
await expect(
180+
mergeShardOutputs({
181+
outputDirectory: join(shards[0], "output"),
182+
shardDirectories: shards
183+
})
184+
).rejects.toThrow("must not overlap");
185+
});
186+
});
187+
188+
async function createFixture(): Promise<{
189+
output: string;
190+
shards: [string, string];
191+
write: (shard: string, path: string, content: string) => Promise<void>;
192+
}> {
193+
const root = await mkdtemp(join(tmpdir(), "fern-shards-"));
194+
temporaryDirectories.push(root);
195+
const shards: [string, string] = [join(root, "shard-0"), join(root, "shard-1")];
196+
await Promise.all(shards.map((shard) => mkdir(shard, { recursive: true })));
197+
return {
198+
output: join(root, "output"),
199+
shards,
200+
write: async (shard, path, content) => {
201+
const destination = join(shard, path);
202+
await mkdir(dirname(destination), { recursive: true });
203+
await writeFile(destination, content);
204+
}
205+
};
206+
}

0 commit comments

Comments
 (0)