feat: add performance-testing package

.prettierignore

@@ -2,4 +2,5 @@
 packages/*/.astro
 packages/*/dist
 packages/*/lib
+packages/performance-testing/cases
 /pnpm-lock.yaml

eslint.config.ts

@@ -44,6 +44,7 @@ const config: ConfigObject[] = defineConfig(
 		"packages/*/dist",
 		"packages/*/lib",
 		"packages/fixtures",
+		"packages/performance-testing/cases",
 		"packages/e2e/tests/**/fixtures/**",
 		"pnpm-lock.yaml",
 		"coverage",

knip.ts

@@ -40,6 +40,16 @@ const config: KnipConfig = {
 		"packages/performance": {
 			project: ["src/**/*.ts!", "!src/rules/ruleTester.ts!"],
 		},
+		"packages/performance-testing": {
+			ignoreDependencies: [
+				"eslint",
+				"eslint-plugin-import",
+				"eslint-plugin-regexp",
+				"eslint-plugin-unicorn",
+				"typescript-eslint",
+			],
+			project: ["src/**/*.ts!"],
+		},
 		"packages/plugin-flint": {
 			ignoreDependencies: [
 				// It's bugging IDK.

packages/performance-testing/.gitignore

@@ -0,0 +1 @@
+/cases

packages/performance-testing/README.md

@@ -0,0 +1,13 @@
+<h1 align="center"><code>@flint.fyi/performance-testing</code></h1>
+
+<p align="center">
+	Runs performance measurements for Flint and other linters.
+	❤️‍🔥
+</p>
+
+> Requires [hyperfine](https://github.com/sharkdp/hyperfine#installation) to be installed globally.
+
+```shell
+pnpm generate
+pnpm measure
+```

packages/performance-testing/package.json

@@ -0,0 +1,45 @@
+{
+	"name": "@flint.fyi/performance-testing",
+	"version": "0.0.0",
+	"private": true,
+	"description": "Runs performance measurements for Flint and other linters.",
+	"homepage": "https://flint.fyi",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/flint-fyi/flint.git",
+		"directory": "packages/performance-testing"
+	},
+	"license": "MIT",
+	"author": {
+		"name": "Flint Team",
+		"url": "https://flint.fyi/team"
+	},
+	"sideEffects": false,
+	"type": "module",
+	"scripts": {
+		"generate": "node src/generate.ts",
+		"measure": "DEBUG=\"*runInHyperfine\" node src/measure.ts",
+		"test": "vitest --project performance-testing"
+	},
+	"dependencies": {
+		"@flint.fyi/rule-data": "workspace:^",
+		"console-table-without-index": "^0.1.1",
+		"debug-for-file": "^0.4.0",
+		"execa": "catalog:dev",
+		"flint": "workspace:^",
+		"prettier": "catalog:dev"
+	},
+	"devDependencies": {
+		"@flint.fyi/build": "workspace:^",
+		"eslint": "catalog:dev",
+		"eslint-plugin-import": "catalog:comparisons",
+		"eslint-plugin-regexp": "catalog:comparisons",
+		"eslint-plugin-unicorn": "catalog:comparisons",
+		"typescript": "catalog:dev",
+		"typescript-eslint": "catalog:dev",
+		"vitest": "catalog:dev"
+	},
+	"engines": {
+		"node": ">=26.1.0"
+	}
+}

packages/performance-testing/src/createTestCaseSlug.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+
+import { createTestCaseSlug } from "./createTestCaseSlug.ts";
+
+describe(createTestCaseSlug, () => {
+	it("joins the entry labels and values when given a numeric rules count", () => {
+		const actual = createTestCaseSlug({ files: 256, rules: 1 });
+
+		expect(actual).toBe("files-256-rules-1");
+	});
+
+	it("lowercases values when given a named rules preset", () => {
+		const actual = createTestCaseSlug({ files: 1024, rules: "common" });
+
+		expect(actual).toBe("files-1024-rules-common");
+	});
+});

packages/performance-testing/src/createTestCaseSlug.ts

@@ -0,0 +1,5 @@
+import type { TestCase } from "./testCases.ts";
+
+export function createTestCaseSlug(testCase: TestCase): string {
+	return Object.entries(testCase).flat().join("-").toLowerCase();
+}

packages/performance-testing/src/creators/createCaseFiles.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+
+import { countCaseFiles, createCaseFiles } from "./createCaseFiles.ts";
+
+describe(countCaseFiles, () => {
+	it.for([2, 256, 1024])(
+		"counts source files matching the requested count when files is %i",
+		(files) => {
+			const actual = countCaseFiles({ files, rules: 1 });
+
+			expect(actual).toBe(files);
+		},
+	);
+});
+
+describe(createCaseFiles, () => {
+	it("creates a config file per linter alongside src and tsconfig.json", () => {
+		const actual = createCaseFiles({ files: 2, rules: 1 });
+
+		expect(Object.keys(actual)).toEqual([
+			"eslint.config.js",
+			"flint.config.ts",
+			"src",
+			"tsconfig.json",
+		]);
+	});
+
+	it("nests example directories under src when files is small", () => {
+		const actual = createCaseFiles({ files: 2, rules: 1 });
+
+		expect(Object.keys(actual.src ?? {})).toEqual(["index.ts", "example0"]);
+	});
+});

packages/performance-testing/src/creators/createCaseFiles.ts

@@ -0,0 +1,89 @@
+import type { TestCase } from "../testCases.ts";
+import type { Structure } from "../writing/writeStructure.ts";
+import { createESLintConfigFile } from "./files/createESLintConfigFile.ts";
+import { createFlintConfigFile } from "./files/createFlintConfigFile.ts";
+import { createStandardTSConfigFile } from "./files/createStandardTSConfigFile.ts";
+import { range } from "./range.ts";
+
+export function countCaseFiles(testCase: TestCase): number {
+	return countStructureFiles(createSourceFiles(testCase));
+}
+
+export function createCaseFiles(testCase: TestCase): Structure {
+	return {
+		"eslint.config.js": [createESLintConfigFile(testCase.rules), "typescript"],
+		"flint.config.ts": [createFlintConfigFile(testCase.rules), "typescript"],
+		src: createSourceFiles(testCase),
+		"tsconfig.json": [createStandardTSConfigFile(), "json"],
+	};
+}
+
+function countStructureFiles(structure: Structure): number {
+	return Object.values(structure).reduce(
+		(total, value) =>
+			total + (Array.isArray(value) ? 1 : countStructureFiles(value)),
+		0,
+	);
+}
+
+function createExampleDirectory(index: number): Structure {
+	return {
+		"index.ts": [createExampleFile(index), "typescript"],
+		...(index > 2 &&
+			Object.fromEntries(
+				range(1, index).map((i) => [
+					`nested${i}`,
+					createExampleDirectory(i - 1),
+				]),
+			)),
+	};
+}
+
+function createExampleFile(index: number) {
+	return [
+		index > 1 &&
+			range(1, index)
+				.map((i) => `export * as nested${i} from "./nested${i}/index.ts";`)
+				.join("\n\t\t"),
+		`
+			export async function example${index}(prefix: string) {
+				await Promise.resolve();
+				return [prefix + "", ${index}];
+			}
+		`,
+	]
+		.filter(Boolean)
+		.join("\n\n");
+}
+
+function createIndexFile(topLevelWidth: number) {
+	const indices = range(0, topLevelWidth);
+
+	return `
+		import { example0 } from "./example0/index.ts";
+
+		export async function root() {
+			// Lint report: ts/forInArrays
+			for (const i in await example0("")) {}
+
+			// No lint report
+			for (const i of await example0("")) {}
+		}
+
+		${indices.map((index) => `export { example${index} } from "./example${index}/index.ts";`).join("\n\t\t")}
+	`;
+}
+
+function createSourceFiles(testCase: TestCase): Structure {
+	const topLevelWidth = Math.max(1, Math.floor(Math.log(testCase.files) * 1.7));
+
+	return {
+		"index.ts": [createIndexFile(topLevelWidth), "typescript"],
+		...Object.fromEntries(
+			Array.from({ length: topLevelWidth }, (_, index) => [
+				`example${index}`,
+				createExampleDirectory(index),
+			]),
+		),
+	};
+}

packages/performance-testing/src/creators/files/createESLintConfigFile.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+
+import { createESLintConfigFile } from "./createESLintConfigFile.ts";
+import { comparedRules } from "./rules.ts";
+
+describe(createESLintConfigFile, () => {
+	it("imports no plugin packages when the only rule is preregistered", () => {
+		const actual = createESLintConfigFile(1);
+
+		expect(actual).toMatchInlineSnapshot(`
+			"
+			import { defineConfig, globalIgnores } from "eslint/config";
+
+			import tseslint from "typescript-eslint";
+
+			export default defineConfig(
+				globalIgnores(["node_modules", "*.config.*"]),
+				tseslint.configs.base,
+				{
+					files: ["src/**/*.ts"],
+					languageOptions: {
+						parserOptions: {
+							projectService: true,
+						},
+					},
+					plugins: {
+
+					},
+					rules: {
+						"@typescript-eslint/no-for-in-array": "error"
+					},
+				},
+			);
+			"
+		`);
+	});
+
+	it("imports each plugin package once when given the many rules preset", () => {
+		const actual = createESLintConfigFile("many");
+
+		expect(Array.from(actual.matchAll(/^import unicorn from/gm))).toHaveLength(
+			1,
+		);
+	});
+
+	it("enables every compared rule when given the many rules preset", () => {
+		const actual = createESLintConfigFile("many");
+
+		expect(
+			comparedRules.many.every(({ eslint }) =>
+				actual.includes(`"${eslint}": "error"`),
+			),
+		).toBe(true);
+	});
+});

packages/performance-testing/src/creators/files/createESLintConfigFile.ts

@@ -0,0 +1,72 @@
+import type { TestCaseRules } from "../../testCases.ts";
+import { comparedRules } from "./rules.ts";
+
+interface PluginPackage {
+	alias: string;
+	specifier: string;
+}
+
+const preregisteredPlugins = new Set(["@typescript-eslint"]);
+
+const pluginPackages = new Map<string, PluginPackage>([
+	["import", { alias: "importPlugin", specifier: "eslint-plugin-import" }],
+	["regexp", { alias: "regexp", specifier: "eslint-plugin-regexp" }],
+	["unicorn", { alias: "unicorn", specifier: "eslint-plugin-unicorn" }],
+]);
+
+export function createESLintConfigFile(rules: TestCaseRules): string {
+	const enabled = comparedRules[rules];
+	const used = new Map<string, PluginPackage>();
+
+	for (const { eslint } of enabled) {
+		const pluginName = getPluginName(eslint);
+
+		if (pluginName === undefined || preregisteredPlugins.has(pluginName)) {
+			continue;
+		}
+
+		const pluginPackage = pluginPackages.get(pluginName);
+
+		if (!pluginPackage) {
+			throw new Error(`No ESLint plugin package is known for ${eslint}.`);
+		}
+
+		used.set(pluginName, pluginPackage);
+	}
+
+	const plugins = Array.from(used, ([name, pluginPackage]) => ({
+		...pluginPackage,
+		name,
+	})).sort((a, b) => a.name.localeCompare(b.name));
+
+	return `
+import { defineConfig, globalIgnores } from "eslint/config";
+${plugins.map(({ alias, specifier }) => `import ${alias} from "${specifier}";`).join("\n")}
+import tseslint from "typescript-eslint";
+
+export default defineConfig(
+	globalIgnores(["node_modules", "*.config.*"]),
+	tseslint.configs.base,
+	{
+		files: ["src/**/*.ts"],
+		languageOptions: {
+			parserOptions: {
+				projectService: true,
+			},
+		},
+		plugins: {
+			${plugins.map(({ alias, name }) => `"${name}": ${alias}`).join(",\n")}
+		},
+		rules: {
+			${enabled.map(({ eslint }) => `"${eslint}": "error"`).join(",\n")}
+		},
+	},
+);
+`;
+}
+
+function getPluginName(ruleName: string) {
+	const separator = ruleName.indexOf("/");
+
+	return separator === -1 ? undefined : ruleName.slice(0, separator);
+}