feat: community plugin list - #3642

.github/workflows/check.yml

@@ -31,3 +31,6 @@ jobs:
 
       - name: Check formatting
         run: pnpm format:check
+
+      - name: Run tests
+        run: pnpm test:community-resources

.github/workflows/syncCommunityResources.yml

@@ -0,0 +1,56 @@
+name: 'Sync Community Resources'
+
+on:
+  schedule:
+    # weekly on Mondays at 00:00 UTC
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  sync-community-resources:
+    name: Sync Community Resources
+    runs-on: ubuntu-slim
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - uses: pnpm/action-setup@v5
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+          cache: pnpm
+
+      # build.ts only needs Node's own APIs, so skip installing the docs site
+      - run: pnpm install --frozen-lockfile --filter community-resources
+
+      - run: pnpm test:community-resources
+
+      - name: sync-community-resources
+        run: pnpm sync:community-resources
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # tauri-docs PR
+      - name: Git config
+        run: |
+          git config --global user.name "tauri-bot"
+          git config --global user.email "tauri-bot@tauri.app"
+
+      - name: Create pull request for updated docs
+        id: cpr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # 8.1.1
+        if: github.event_name != 'pull_request' && github.event_name != 'push'
+        with:
+          token: ${{ secrets.ORG_TAURI_BOT_PAT }}
+          commit-message: 'chore(docs): Update Community Resources'
+          branch: ci/v2/update-community-resources
+          title: Update Community Resources
+          labels: 'bot'
+          sign-commits: true

package.json

@@ -11,11 +11,14 @@
     "dev:setup": "pnpm dev:setup:submodules && pnpm dev:setup:tauri && pnpm dev:setup:plugins-workspace && pnpm build:compatibility-table",
     "sync:sponsors": "pnpm --filter fetch-sponsors run build sponsors",
     "sync:contributors": "pnpm --filter fetch-sponsors run build contributors",
+    "sync:community-resources": "pnpm --filter community-resources run build",
     "dev": "astro dev",
     "format": "prettier -w --cache --plugin prettier-plugin-astro .",
     "format:check": "prettier -c --cache --plugin prettier-plugin-astro .",
     "check": "pnpm build:releases && astro check",
+    "build:community-resources": "pnpm --filter community-resources run build",
     "build:compatibility-table": "pnpm --filter compatibility-table run build",
+    "test:community-resources": "pnpm --filter community-resources run test",
     "build:references": "pnpm --filter js-api-generator run build",
     "build:config": "pnpm --filter config-generator run build",
     "build:cli": "pnpm --filter cli-generator run build",

packages/community-resources/build.ts

@@ -0,0 +1,247 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import {
+  assertNoDataLoss,
+  cleanRepoUrl,
+  githubRepo,
+  isOfficial,
+  mergeRegistries,
+  npmPackageUrl,
+  sortByCreatedDesc,
+  type Resource,
+  type Snapshot,
+} from './transform.ts';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const OUTPUT_FILE = path.resolve(__dirname, '../../src/data/communityResources.json');
+
+const GITHUB_TOKEN = process.env.GITHUB_TOKEN || null;
+const query = 'tauri-plugin-';
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+// GitHub answers 403 (not 429) when a token is exhausted, so status alone
+// can't tell a rate limit from a genuine permission error
+function isRateLimited(res: Response) {
+  return (
+    res.status === 429 || (res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0')
+  );
+}
+
+async function fetchJson(url: string, headers = new Headers()) {
+  if (!headers.has('User-Agent')) {
+    headers.set(
+      'User-Agent',
+      'tauri-docs-plugins-discover (https://github.com/tauri-apps/tauri-docs)'
+    );
+  }
+
+  for (let attempt = 1; ; attempt++) {
+    const res = await fetch(url, { headers, signal: AbortSignal.timeout(30_000) });
+    if (res.ok) {
+      return res.json();
+    }
+    if ((isRateLimited(res) || res.status >= 500) && attempt < 4) {
+      const retryAfter = Number(res.headers.get('retry-after')) || attempt * 15;
+      console.warn(`  ${res.status} for ${url} - retrying in ${retryAfter}s`);
+      await sleep(retryAfter * 1000);
+      continue;
+    }
+    throw new Error(`Failed ${url}: ${res.status} ${res.statusText}`);
+  }
+}
+
+// https://crates.io/data-access
+async function fetchCrates(): Promise<Resource[]> {
+  const results: Resource[] = [];
+  let page = 1;
+  const per_page = 100;
+  while (true) {
+    const url = `https://crates.io/api/v1/crates?page=${page}&per_page=${per_page}&q=${query}`;
+    const j = await fetchJson(url);
+    if (!j.crates || j.crates.length === 0) {
+      break;
+    }
+    for (const c of j.crates) {
+      if (!c.name || !c.name.startsWith(query)) {
+        continue;
+      }
+      results.push({
+        source: 'crates',
+        name: c.name,
+        description: c.description || '',
+        created_at: c.created_at || '',
+        repository: cleanRepoUrl(c.repository || c.homepage || ''),
+        crates_io: `https://crates.io/crates/${c.name}`,
+      });
+    }
+    if (j.meta && j.meta.total <= page * per_page) break;
+    page++;
+    await sleep(1001);
+  }
+  return results;
+}
+
+// https://docs.npmjs.com/policies/open-source-terms
+async function fetchNpm(): Promise<Resource[]> {
+  const results: Resource[] = [];
+  const size = 250;
+  let from = 0;
+  while (true) {
+    const url = `https://registry.npmjs.org/-/v1/search?text=${query}&size=${size}&from=${from}`;
+    const j = await fetchJson(url);
+    const objects = j.objects || [];
+    if (objects.length === 0) {
+      break;
+    }
+    let pageMatches = 0;
+    for (const obj of objects) {
+      const p = obj.package;
+      const name = p.name;
+      if (!name) {
+        continue;
+      }
+      if (!/(^|\/)tauri-plugin-/.test(name)) {
+        continue;
+      }
+      pageMatches++;
+      results.push({
+        source: 'npm',
+        name,
+        description: p.description || '',
+        created_at: p.date || '',
+        // `links` is sparse - many packages expose only `links.npm`
+        repository: cleanRepoUrl(p.links?.repository || p.links?.homepage || ''),
+        npm: npmPackageUrl(name),
+      });
+    }
+    from += size;
+    // `text=` matches fuzzily (descriptions, keywords), so j.total is far
+    // larger than the set of real name matches, and npm caps `from` at 10k
+    // anyway. Results are relevance-sorted, so once a whole page has no
+    // name match the tail is only noise - stop there.
+    if (pageMatches === 0 || from >= Math.min(j.total || 0, 10_000 - size)) {
+      break;
+    }
+    await sleep(1001);
+  }
+  return results;
+}
+
+// the npm search API only exposes the last-publish date; the real creation
+// date lives in the package's registry document under time.created
+async function fetchNpmCreatedDate(name: string) {
+  try {
+    const j = await fetchJson(`https://registry.npmjs.org/${encodeURIComponent(name)}`);
+    return j.time?.created || null;
+  } catch {
+    return null;
+  }
+}
+
+async function fetchGithubStars(ownerRepo: string) {
+  const headers = new Headers({ Accept: 'application/vnd.github+json' });
+  if (GITHUB_TOKEN) {
+    headers.append('Authorization', `token ${GITHUB_TOKEN}`);
+  }
+  try {
+    const j = await fetchJson(`https://api.github.com/repos/${ownerRepo}`, headers);
+    return j.stargazers_count ?? null;
+  } catch (e) {
+    // a 404 here is normal (repo renamed or deleted); anything else is worth seeing
+    console.warn(`  no stars for ${ownerRepo}: ${(e as Error).message}`);
+    return null;
+  }
+}
+
+async function addNpmCreatedDates(items: Resource[]) {
+  const npmOnly = items.filter((item) => item.source === 'npm');
+  console.log(`Fetching creation dates for ${npmOnly.length} npm-only packages...`);
+  let done = 0;
+  for (const item of npmOnly) {
+    const created = await fetchNpmCreatedDate(item.name);
+    if (created) {
+      item.created_at = created;
+    }
+    done++;
+    if (done % 50 === 0) {
+      console.log(`  dates ${done}/${npmOnly.length}`);
+    }
+    await sleep(100);
+  }
+}
+
+async function addGithubStars(items: Resource[]) {
+  if (!GITHUB_TOKEN) {
+    console.warn(
+      'GITHUB_TOKEN not set - skipping star counts (unauthenticated rate limits are too low).'
+    );
+    return;
+  }
+
+  console.log('Fetching GitHub star counts...');
+  const starsCache = new Map<string, number | null>();
+  for (const item of items) {
+    const ownerRepo = githubRepo(item.repository);
+    if (!ownerRepo) {
+      continue;
+    }
+    if (!starsCache.has(ownerRepo)) {
+      starsCache.set(ownerRepo, await fetchGithubStars(ownerRepo));
+      await sleep(100);
+    }
+    item.stars = starsCache.get(ownerRepo) ?? null;
+  }
+}
+
+async function readPrevious(): Promise<Snapshot | null> {
+  try {
+    return JSON.parse(await fs.readFile(OUTPUT_FILE, 'utf8')) as Snapshot;
+  } catch {
+    return null;
+  }
+}
+
+async function run() {
+  const previous = await readPrevious();
+
+  console.log('Fetching crates.io packages...');
+  const crates = await fetchCrates();
+  console.log(`Found ${crates.length} crates matching prefix.`);
+
+  console.log('Fetching npm packages...');
+  const npm = await fetchNpm();
+  console.log(`Found ${npm.length} npm packages matching prefix.`);
+
+  const merged = mergeRegistries(crates, npm);
+  const items = merged.filter((item) => !isOfficial(item.repository));
+  console.log(
+    `Merged to ${merged.length} entries, ${merged.length - items.length} official dropped.`
+  );
+
+  // registry.npmjs.org and api.github.com have independent rate budgets, and
+  // the two passes touch different fields, so they can run side by side
+  await Promise.all([addNpmCreatedDates(items), addGithubStars(items)]);
+
+  const resources = sortByCreatedDesc(items);
+  assertNoDataLoss(previous, resources);
+
+  // the timestamp alone would make every weekly run a diff, and the sync
+  // workflow would open a pull request for it
+  if (previous && JSON.stringify(previous.resources) === JSON.stringify(resources)) {
+    console.log(`No changes to ${resources.length} resources - leaving the file untouched.`);
+    return;
+  }
+
+  const output: Snapshot = { generated: new Date().toISOString(), resources };
+  await fs.mkdir(path.dirname(OUTPUT_FILE), { recursive: true });
+  // trailing newline so the file is already prettier-clean and CI needs no format pass
+  await fs.writeFile(OUTPUT_FILE, JSON.stringify(output, null, 2) + '\n', 'utf8');
+  console.log(`Wrote ${resources.length} resources to ${OUTPUT_FILE}`);
+}
+
+run().catch((e) => {
+  console.error('Error generating resources:', e);
+  process.exit(1);
+});

packages/community-resources/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "community-resources",
+  "version": "1.0.0",
+  "private": "true",
+  "description": "Generates the community plugin directory from the crates.io and npm registries",
+  "type": "module",
+  "scripts": {
+    "build": "node ./build.ts",
+    "test": "node --test"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=24"
+  },
+  "dependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^7.0.2"
+  }
+}