Start implementing ingestion logic

Author: spastorelli

integrations/github-issues/src/components.tsx

@@ -3,7 +3,7 @@ import type { GitHubIssuesRuntimeContext, GitHubIssuesRuntimeEnvironment } from
 import { createGitHubAppSetupState } from './setup';
 
 /**
- * Configuration component for the GitHub Conversations integration.
+ * Configuration component for the GitHub Issues integration.
  */
 export const configComponent = createComponent<
     InstallationConfigurationProps<GitHubIssuesRuntimeEnvironment>,
@@ -26,7 +26,7 @@ export const configComponent = createComponent<
             `https://github.com/apps/${context.environment.secrets.GITHUB_APP_NAME}/installations/new`,
         );
         const githubAppSetupState = await createGitHubAppSetupState(context, {
-            gitBookInstallationId: installation.id,
+            gitbookInstallationId: installation.id,
         });
         githubAppInstallURL.searchParams.append('state', githubAppSetupState);
 

integrations/github-issues/src/github-client.ts

@@ -0,0 +1,125 @@
+import jwt from '@tsndr/cloudflare-worker-jwt';
+import { Octokit } from 'octokit';
+
+import { ExposableError, Logger } from '@gitbook/runtime';
+import { GitHubIssuesRepository, GitHubIssuesRuntimeContext } from './types';
+
+const logger = Logger('github-issues:github-client');
+
+const GITBOOK_INTEGRATION_USER_AGENT = 'GitBook-GitHub-Issues-Integration';
+const GITHUB_API_VERSION = '2022-11-28';
+
+/**
+ * Fetch repositories with issues from a specific GitHub App installation
+ */
+export async function fetchGitHubReposForInstallation(
+    octokit: Octokit,
+    installationId: string,
+): Promise<GitHubIssuesRepository[]> {
+    try {
+        const response = await octokit.request('GET /installation/repositories', {
+            per_page: 100,
+            headers: {
+                'X-GitHub-Api-Version': GITHUB_API_VERSION,
+            },
+        });
+
+        const repositories = response.data.repositories;
+
+        return repositories.filter((repo) => repo.has_issues);
+    } catch (error) {
+        logger.error(
+            `Failed to fetch GitHub repositories with issue for installation ${installationId}: `,
+            error instanceof Error ? error.message : String(error),
+        );
+        return [];
+    }
+}
+
+/**
+ * Get an authenticated Octokit instance for a GitHub app installation.
+ */
+export async function getOctokitClientForInstallation(
+    context: GitHubIssuesRuntimeContext,
+    githubInstallationId: string,
+): Promise<Octokit> {
+    const { installation } = context.environment;
+    if (!installation) {
+        throw new ExposableError(`GitBook installation not found`);
+    }
+
+    const config = getGitHubAppConfig(context);
+    if (!config.appId || !config.privateKey) {
+        throw new ExposableError('GitHub App credentials not configured');
+    }
+
+    const token = await getGitHubInstallationAccessToken({
+        githubInstallationId,
+        appId: config.appId,
+        privateKey: config.privateKey,
+    });
+
+    return new Octokit({
+        auth: token,
+        userAgent: GITBOOK_INTEGRATION_USER_AGENT,
+    });
+}
+/**
+ * Generate a JWT token for GitHub App authentication.
+ */
+async function generateGitHubAppJWT(appId: string, privateKey: string): Promise<string> {
+    const now = Math.floor(Date.now() / 1000);
+
+    const payload = {
+        iat: now - 60, // Issued 60 seconds ago (for clock drift)
+        exp: now + 60 * 10,
+        iss: appId,
+    };
+
+    return await jwt.sign(payload, privateKey, { algorithm: 'RS256' });
+}
+
+/**
+ * Get an access token for a GitHub App installation.
+ */
+async function getGitHubInstallationAccessToken(args: {
+    githubInstallationId: string;
+    appId: string;
+    privateKey: string;
+}): Promise<string> {
+    const { githubInstallationId, appId, privateKey } = args;
+    const jwtToken = await generateGitHubAppJWT(appId, privateKey);
+
+    const octokit = new Octokit({
+        auth: jwtToken,
+        userAgent: GITBOOK_INTEGRATION_USER_AGENT,
+    });
+
+    try {
+        const response = await octokit.request(
+            'POST /app/installations/{installation_id}/access_tokens',
+            {
+                installation_id: parseInt(githubInstallationId),
+                headers: {
+                    'X-GitHub-Api-Version': GITHUB_API_VERSION,
+                },
+            },
+        );
+
+        return response.data.token;
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to get installation access token: ${errorMessage}`);
+    }
+}
+
+/**
+ * Get GitHub App configuration for installation-based authentication.
+ */
+export function getGitHubAppConfig(context: GitHubIssuesRuntimeContext) {
+    return {
+        appId: context.environment.secrets.GITHUB_APP_ID,
+        privateKey: context.environment.secrets.GITHUB_PRIVATE_KEY,
+        installationIds: context.environment.installation?.configuration?.installation_ids,
+    };
+}

integrations/github-issues/src/index.ts

@@ -4,6 +4,9 @@ import type { GitHubIssuesRuntimeContext, GitHubWebhookInstallationEventPayload
 import { configComponent } from './components';
 import { handleGitHubAppSetup } from './setup';
 import { handleGitHubAppInstallationEvent, verifyGitHubWebhookSignature } from './webhook';
+import { triggerInitialIngestionForGitBookInstallation } from './ingestion';
+import { getGitHubInstallationIds } from './utils';
+import { handleGitHubIssuesIntegrationTask } from './tasks';
 
 const logger = Logger('github-issues');
 
@@ -87,6 +90,27 @@ export default createIntegration<GitHubIssuesRuntimeContext>({
     },
     components: [configComponent],
     events: {
-        installation_setup: async (_, context) => {},
+        installation_setup: async (_, context) => {
+            const githubInstallationIds = getGitHubInstallationIds(context);
+            const hasInstallations = githubInstallationIds.length > 0;
+            const gitbookInstallationId = context.environment.installation?.id;
+
+            if (!hasInstallations) {
+                logger.info(
+                    `GitBook installation ${gitbookInstallationId} has no associated GitHub installations. Skipping initial ingestion.`,
+                );
+                return;
+            }
+
+            try {
+                await triggerInitialIngestionForGitBookInstallation(context);
+            } catch (error) {
+                logger.error(
+                    `GitBook installation ${gitbookInstallationId} setup failed: `,
+                    error instanceof Error ? error.message : String(error),
+                );
+            }
+        },
     },
+    task: handleGitHubIssuesIntegrationTask,
 });

integrations/github-issues/src/ingestion.ts

@@ -0,0 +1,80 @@
+import pMap from 'p-map';
+
+import { ExposableError, Logger } from '@gitbook/runtime';
+import { GitHubIssuesRuntimeContext } from './types';
+import { getGitHubInstallationIds } from './utils';
+import { fetchGitHubReposForInstallation, getOctokitClientForInstallation } from './github-client';
+import { queueGitHubIssuesIntegrationTask } from './tasks';
+
+const logger = Logger('github-issues:ingest');
+
+/**
+ * Trigger the GitBook Agent ingestion of all issues from all of the
+ * linked GitHub respositories of a GitBook installation.
+ */
+export async function triggerInitialIngestionForGitBookInstallation(
+    context: GitHubIssuesRuntimeContext,
+) {
+    const { installation: gitbookInstallation } = context.environment;
+    if (!gitbookInstallation) {
+        throw new ExposableError('GitBook installation not found');
+    }
+
+    const githubInstallationIds = getGitHubInstallationIds(context);
+    if (githubInstallationIds.length === 0) {
+        throw new ExposableError('No GitHub App installation IDs found');
+    }
+
+    let totalRepoToProcess = 0;
+    const pendingTasks: Array<Promise<void>> = [];
+    await pMap(
+        githubInstallationIds,
+        async (githubInstallationId) => {
+            try {
+                const octokit = await getOctokitClientForInstallation(
+                    context,
+                    githubInstallationId,
+                );
+                const repos = await fetchGitHubReposForInstallation(octokit, githubInstallationId);
+
+                if (repos.length === 0) {
+                    logger.info(
+                        `No GitHub repositories found for installation ${githubInstallationId}. Skipping.`,
+                    );
+                    return;
+                }
+
+                totalRepoToProcess += repos.length;
+                for (const repo of repos) {
+                    pendingTasks.push(
+                        queueGitHubIssuesIntegrationTask(context, {
+                            type: 'ingest:github-installation:repo-issues',
+                            payload: {
+                                organization: gitbookInstallation.target.organization,
+                                gitbookInstallationId: gitbookInstallation.id,
+                                githubInstallationId: Number(githubInstallationId),
+                                repository: {
+                                    name: repo.name,
+                                    owner: repo.owner.login,
+                                },
+                            },
+                        }),
+                    );
+                }
+            } catch (err) {
+                logger.error(
+                    `Error while fetching repositories for installation ${githubInstallationId}`,
+                    err,
+                );
+                return;
+            }
+        },
+        { concurrency: 5 },
+    );
+
+    logger.info(
+        `Dispatched ${pendingTasks.length} tasks to ingest a total of ${totalRepoToProcess} github repositories`,
+    );
+
+    context.waitUntil(Promise.all(pendingTasks));
+}