feat: add performance-testing package

.prettierignore

@@ -2,4 +2,5 @@
 packages/*/.astro
 packages/*/dist
 packages/*/lib
+packages/performance-testing/cases
 /pnpm-lock.yaml

eslint.config.ts

@@ -44,6 +44,7 @@ const config: ConfigObject[] = defineConfig(
 		"packages/*/dist",
 		"packages/*/lib",
 		"packages/fixtures",
+		"packages/performance-testing/cases",
 		"packages/e2e/tests/**/fixtures/**",
 		"pnpm-lock.yaml",
 		"coverage",

knip.ts

@@ -1,7 +1,7 @@
 import type { KnipConfig } from "knip";
 
 const config: KnipConfig = {
-	ignore: ["packages/e2e/**/*"],
+	ignore: ["packages/e2e/**/*", "packages/performance-testing/**/*"],
 	ignoreExportsUsedInFile: { interface: true, type: true },
 	treatConfigHintsAsErrors: true,
 	workspaces: {

packages/performance-testing/.gitignore

@@ -0,0 +1 @@
+/cases

packages/performance-testing/README.md

@@ -0,0 +1,40 @@
+<h1 align="center"><code>@flint.fyi/performance-testing</code></h1>
+
+<p align="center">
+	Runs performance measurements for Flint and other linters.
+	❤️‍🔥
+</p>
+
+```shell
+pnpm generate
+DEBUG=*runInHyperfine pnpm measure
+```
+
+## Results
+
+Measured with Hyperfine 1.20.0 on an Apple Silicon Mac running Node 24.13.1, against ESLint 10.8.0 and typescript-eslint 8.66.0:
+
+```text
+┌───────────────────────┬───────┬───────────────────────┬───────┐
+│ eslint                │ files │ flint                 │ rules │
+├───────────────────────┼───────┼───────────────────────┼───────┤
+│ '543.0 ms ±  24.1 ms' │ 2     │ '791.5 ms ±  23.3 ms' │ 1     │
+│ '781.0 ms ±  29.1 ms' │ 2     │ '844.7 ms ±  46.8 ms' │ 119   │
+│ '854.7 ms ±  56.7 ms' │ 2     │ '827.4 ms ±  38.6 ms' │ 272   │
+│ '966.5 ms ±  22.7 ms' │ 256   │ '1.060 s ±  0.054 s'  │ 1     │
+│ '1.301 s ±  0.037 s'  │ 256   │ '1.234 s ±  0.043 s'  │ 119   │
+│ '1.458 s ±  0.049 s'  │ 256   │ '1.322 s ±  0.032 s'  │ 272   │
+└───────────────────────┴───────┴───────────────────────┴───────┘
+```
+
+Numbers from any one machine are only meaningful next to the other linter measured on that same machine, so treat the columns as a comparison rather than as absolute costs.
+
+## Comparability
+
+`pnpm generate` writes one test case directory per (files, rules) pair, each containing an `eslint.config.js` and a `flint.config.ts` meant to ask for the same work:
+
+- **Same files.** Both configs lint `src/**/*.ts` and ignore `node_modules` and `*.config.*`, so neither linter is charged for the other's config file.
+- **Same rules.** Every enabled rule comes from a `@flint.fyi/rule-data` entry that maps a Flint rule to an ESLint rule, so the counts match exactly.
+Flint rules that map to several overlapping ESLint rules — typically a core rule and its typescript-eslint extension — are paired with just one of them, preferring the typescript-eslint rule, then the core rule, then the plugin rule that sorts first.
+- **Same type information.** Both run type-aware: ESLint through `projectService`, Flint through the case's `tsconfig.json`.
+- **Same cold start.** Flint runs with `--cache-ignore` so Hyperfine's repeated runs re-lint from scratch, matching ESLint's lack of a cache.

packages/performance-testing/package.json

@@ -0,0 +1,43 @@
+{
+	"name": "@flint.fyi/performance-testing",
+	"version": "0.0.0",
+	"private": true,
+	"description": "Runs performance measurements for Flint and other linters.",
+	"homepage": "https://flint.fyi",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/flint-fyi/flint.git",
+		"directory": "packages/performance-testing"
+	},
+	"license": "MIT",
+	"author": {
+		"name": "Flint Team",
+		"url": "https://flint.fyi/team"
+	},
+	"sideEffects": false,
+	"type": "module",
+	"scripts": {
+		"generate": "node src/generate.ts",
+		"measure": "DEBUG=\"*runInHyperfine\" node src/measure.ts"
+	},
+	"dependencies": {
+		"@flint.fyi/rule-data": "workspace:^",
+		"console-table-without-index": "^0.1.1",
+		"debug-for-file": "^0.4.0",
+		"execa": "catalog:dev",
+		"flint": "workspace:^",
+		"prettier": "catalog:dev"
+	},
+	"devDependencies": {
+		"@flint.fyi/build": "workspace:^",
+		"eslint": "catalog:dev",
+		"eslint-plugin-import": "catalog:dev",
+		"eslint-plugin-regexp": "catalog:dev",
+		"eslint-plugin-unicorn": "catalog:dev",
+		"typescript": "catalog:dev",
+		"typescript-eslint": "catalog:dev"
+	},
+	"engines": {
+		"node": ">=26.1.0"
+	}
+}

packages/performance-testing/src/creators/createCaseFiles.ts

@@ -0,0 +1,89 @@
+import type { TestCase } from "../testCases.ts";
+import type { Structure } from "../writing/writeStructure.ts";
+import { createESLintConfigFile } from "./files/createESLintConfigFile.ts";
+import { createFlintConfigFile } from "./files/createFlintConfigFile.ts";
+import { createStandardTSConfigFile } from "./files/createStandardTSConfigFile.ts";
+import { range } from "./utils.ts";
+
+export function countCaseFiles(testCase: TestCase): number {
+	return countStructureFiles(createSourceFiles(testCase));
+}
+
+export function createCaseFiles(testCase: TestCase): Structure {
+	return {
+		"eslint.config.js": [createESLintConfigFile(testCase.rules), "typescript"],
+		"flint.config.ts": [createFlintConfigFile(testCase.rules), "typescript"],
+		src: createSourceFiles(testCase),
+		"tsconfig.json": [createStandardTSConfigFile(), "json"],
+	};
+}
+
+function countStructureFiles(structure: Structure): number {
+	return Object.values(structure).reduce(
+		(total, value) =>
+			total + (Array.isArray(value) ? 1 : countStructureFiles(value)),
+		0,
+	);
+}
+
+function createExampleDirectory(index: number): Structure {
+	return {
+		"index.ts": [createExampleFile(index), "typescript"],
+		...(index > 2 &&
+			Object.fromEntries(
+				range(1, index).map((i) => [
+					`nested${i}`,
+					createExampleDirectory(i - 1),
+				]),
+			)),
+	};
+}
+
+function createExampleFile(index: number) {
+	return [
+		index > 1 &&
+			range(1, index)
+				.map((i) => `export * as nested${i} from "./nested${i}/index.js";`)
+				.join("\n\t\t"),
+		`
+			export async function example${index}(prefix: string) {
+				await Promise.resolve();
+				return [prefix + "", ${index}];
+			}
+		`,
+	]
+		.filter(Boolean)
+		.join("\n\n");
+}
+
+function createIndexFile(topLevelWidth: number) {
+	const indices = range(0, topLevelWidth);
+
+	return `
+		import { example0 } from "./example0/index.ts";
+
+		export async function root() {
+			// Lint report: ts/forInArrays
+			for (const i in await example0("")) {}
+
+			// No lint report
+			for (const i of await example0("")) {}
+		}
+
+		${indices.map((index) => `export { example${index} } from "./example${index}/index.js";`).join("\n\t\t")}
+	`;
+}
+
+function createSourceFiles(testCase: TestCase): Structure {
+	const topLevelWidth = Math.max(1, Math.floor(Math.log(testCase.files) * 1.7));
+
+	return {
+		"index.ts": [createIndexFile(topLevelWidth), "typescript"],
+		...Object.fromEntries(
+			Array.from({ length: topLevelWidth }, (_, index) => [
+				`example${index}`,
+				createExampleDirectory(index),
+			]),
+		),
+	};
+}

packages/performance-testing/src/creators/files/createESLintConfigFile.ts

@@ -0,0 +1,72 @@
+import type { TestCaseRules } from "../../testCases.ts";
+import { comparedRules } from "./rules.ts";
+
+interface PluginPackage {
+	alias: string;
+	specifier: string;
+}
+
+const preregisteredPlugins = new Set(["@typescript-eslint"]);
+
+const pluginPackages = new Map<string, PluginPackage>([
+	["import", { alias: "importPlugin", specifier: "eslint-plugin-import" }],
+	["regexp", { alias: "regexp", specifier: "eslint-plugin-regexp" }],
+	["unicorn", { alias: "unicorn", specifier: "eslint-plugin-unicorn" }],
+]);
+
+export function createESLintConfigFile(rules: TestCaseRules): string {
+	const enabled = comparedRules[rules];
+	const used = new Map<string, PluginPackage>();
+
+	for (const { eslint } of enabled) {
+		const pluginName = getPluginName(eslint);
+
+		if (pluginName === undefined || preregisteredPlugins.has(pluginName)) {
+			continue;
+		}
+
+		const pluginPackage = pluginPackages.get(pluginName);
+
+		if (!pluginPackage) {
+			throw new Error(`No ESLint plugin package is known for ${eslint}.`);
+		}
+
+		used.set(pluginName, pluginPackage);
+	}
+
+	const plugins = Array.from(used, ([name, pluginPackage]) => ({
+		...pluginPackage,
+		name,
+	})).sort((a, b) => a.name.localeCompare(b.name));
+
+	return `
+import { defineConfig, globalIgnores } from "eslint/config";
+${plugins.map(({ alias, specifier }) => `import ${alias} from "${specifier}";`).join("\n")}
+import tseslint from "typescript-eslint";
+
+export default defineConfig(
+	globalIgnores(["node_modules", "*.config.*"]),
+	tseslint.configs.base,
+	{
+		files: ["src/**/*.ts"],
+		languageOptions: {
+			parserOptions: {
+				projectService: true,
+			},
+		},
+		plugins: {
+			${plugins.map(({ alias, name }) => `"${name}": ${alias}`).join(",\n")}
+		},
+		rules: {
+			${enabled.map(({ eslint }) => `"${eslint}": "error"`).join(",\n")}
+		},
+	},
+);
+`;
+}
+
+function getPluginName(ruleName: string) {
+	const separator = ruleName.indexOf("/");
+
+	return separator === -1 ? undefined : ruleName.slice(0, separator);
+}

packages/performance-testing/src/creators/files/createFlintConfigFile.ts

@@ -0,0 +1,22 @@
+import type { TestCaseRules } from "../../testCases.ts";
+import { comparedRules } from "./rules.ts";
+
+export function createFlintConfigFile(rules: TestCaseRules): string {
+	return `
+import { defineConfig, ts } from "flint";
+
+export default defineConfig({
+	ignore: ["node_modules", "*.config.*"],
+	use: [
+		{
+			files: ["src/**/*.ts"],
+			rules: [
+				ts.rules({
+					${comparedRules[rules].map(({ flint }) => `${flint}: true`).join(",\n")}
+				}),
+			],
+		},
+	],
+});
+`;
+}

packages/performance-testing/src/creators/files/createPackageFile.ts

@@ -0,0 +1,10 @@
+import type { TestCase } from "../../testCases.ts";
+import { createTestCaseSlug } from "../../utils.ts";
+
+export function createPackageFile(data: TestCase): object {
+	return {
+		name: createTestCaseSlug(data),
+		private: true,
+		type: "module",
+	};
+}

packages/performance-testing/src/creators/files/createStandardTSConfigFile.ts

@@ -0,0 +1,13 @@
+export function createStandardTSConfigFile(): object {
+	return {
+		compilerOptions: {
+			allowImportingTsExtensions: true,
+			module: "NodeNext",
+			noEmit: true,
+			skipLibCheck: true,
+			strict: true,
+			target: "ESNext",
+		},
+		include: ["src"],
+	};
+}