Merge pull request #633 from dzcode-io/feat/translate-name-and-title-fields-using-llm

ZibanPirate · web-flow · commit a893a2e3bbbf · 2025-01-13T09:15:05.000+01:00
Feat: translate name and title fields using llm
diff --git a/api/README.md b/api/README.md
@@ -24,8 +24,9 @@ NODE_ENV=development
 Keep in mind that you have limited calls to the GitHub API (60 calls per hour). The [FetchService](./api/src/fetch/service.ts) does a great job of caching these calls so it doesn't unnecessarily consume the GitHub API quota. If you wish to extend the limit from 60 to 5000, simply create a [GitHub Personal Access Token](https://github.com/settings/tokens) (make sure it has `Access public repositories` checked), and set it in `./api/.env` like this:
 
 ```.env
-GITHUB_TOKEN=Paste_Your_Token_Here
+GITHUB_TOKEN=Paste_your_token_here
 NODE_ENV=development
+OPENAI_KEY=Pase_your_key_here
 ```
 
 **Note:** If the README is still unclear, please create a PR with your suggested changes/additions.
diff --git a/api/oracle-cloud/deploy.ts b/api/oracle-cloud/deploy.ts
@@ -52,6 +52,7 @@ const appPath = "~/app";
 const sshPrefix =
   "ssh -o StrictHostKeyChecking=no " + (sshKeyPath ? `-i ${sshKeyPath} ` : "") + sshServer + " ";
 
+// todo-ZM: let docker-compose handle deletion of old containers
 // Check for existing containers
 logs = execSync(sshPrefix + '"sudo docker ps -aq"');
 
diff --git a/api/package.json b/api/package.json
@@ -15,6 +15,7 @@
     "@types/make-fetch-happen": "^10.0.4",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
+    "class-validator-jsonschema": "^5.0.1",
     "cors": "^2.8.5",
     "cron": "^3.1.7",
     "dotenv": "^16.4.5",
diff --git a/api/src/ai/service.ts b/api/src/ai/service.ts
@@ -0,0 +1,66 @@
+import { ConfigService } from "src/config/service";
+import { LoggerService } from "src/logger/service";
+import { Service } from "typedi";
+import { targetConstructorToSchema } from "class-validator-jsonschema";
+import { FetchService } from "src/fetch/service";
+import { ClassConstructor, plainToClass } from "class-transformer";
+import { validateSync } from "class-validator";
+
+type AIChat = { role: "user" | "system"; content: string };
+
+type OpenAIResponse = {
+  choices: Array<{ message: AIChat }>;
+};
+
+@Service()
+export class AIService {
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly logger: LoggerService,
+    private readonly fetchService: FetchService,
+  ) {}
+
+  public query = async <T extends object>(
+    payload: AIChat[],
+    ResponseDto: ClassConstructor<T>,
+  ): Promise<T> => {
+    const schema = targetConstructorToSchema(ResponseDto);
+
+    const payloadWithValidationPrompt: AIChat[] = [
+      {
+        role: "system",
+        content: `system response must strictly follow the schema:\n${JSON.stringify(schema)}`,
+      },
+      ...payload,
+    ];
+
+    const { OPENAI_KEY } = this.configService.env();
+
+    // todo: cache response
+    const res = await this.fetchService.post<OpenAIResponse>(
+      "https://api.openai.com/v1/chat/completions",
+      {
+        headers: { Authorization: `Bearer ${OPENAI_KEY}` },
+        body: {
+          model: "gpt-4o",
+          messages: payloadWithValidationPrompt,
+        },
+      },
+    );
+
+    const chatResponseUnchecked = JSON.parse(res.choices[0].message.content) as T;
+
+    const output = plainToClass(ResponseDto, chatResponseUnchecked);
+    const errors = validateSync(output);
+
+    if (errors.length > 0)
+      throw new Error(
+        `⚠️  Errors in AI response in the following keys:${errors.reduce(
+          (pV, cV) => (pV += "\n" + cV.property + " : " + JSON.stringify(cV.constraints)),
+          "",
+        )}`,
+      );
+
+    return output;
+  };
+}
diff --git a/api/src/config/types.ts b/api/src/config/types.ts
@@ -40,4 +40,7 @@ export class EnvRecord {
   }
 
   MEILISEARCH_MASTER_KEY = "default";
+
+  @IsString()
+  OPENAI_KEY = "no-key";
 }
diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts
@@ -14,6 +14,8 @@ import { RepositoryRepository } from "src/repository/repository";
 import { SearchService } from "src/search/service";
 import { Service } from "typedi";
 import { TagRepository } from "src/tag/repository";
+import { AIService } from "src/ai/service";
+import { AIResponseTranslateNameDto, AIResponseTranslateTitleDto } from "./dto";
 
 @Service()
 export class DigestCron {
@@ -29,7 +31,8 @@ export class DigestCron {
     private readonly contributionsRepository: ContributionRepository,
     private readonly contributorsRepository: ContributorRepository,
     private readonly searchService: SearchService,
-    private readonly tagsRepository: TagRepository,
+    private readonly tagRepository: TagRepository,
+    private readonly aiService: AIService,
   ) {
     const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron");
     new SentryCronJob(
@@ -81,10 +84,29 @@ export class DigestCron {
     // or uncomment to skip the cron
     // if (Math.random()) return;
 
+    const projectTitleSystemPrompt = `user will give you an open-source project name, and you will translate it to Arabic.`;
+    const contributorNameSystemPrompt = `user will give you an open-source contributor name, and you will translate it to Arabic.
+if the name contain both english and arabic only keep the parts related to the language.`;
+    const issueTitleSystemPrompt = `user will give you an open-source issue/PR title, and you will translate it to Arabic.`;
+
     for (const project of projectsFromDataFolder) {
-      // todo: call AIService
-      const name_en = project.name;
-      const name_ar = `ar ${name_en}`;
+      let name_en = project.name;
+      let name_ar = name_en;
+
+      try {
+        const aiRes = await this.aiService.query(
+          [
+            { role: "system", content: projectTitleSystemPrompt },
+            { role: "user", content: name_en },
+          ],
+          AIResponseTranslateNameDto,
+        );
+
+        name_en = aiRes.name_en;
+        name_ar = aiRes.name_ar;
+      } catch (error) {
+        captureException(error, { tags: { type: "CRON" } });
+      }
 
       const projectEntity: ProjectRow = {
         runId,
@@ -94,7 +116,7 @@ export class DigestCron {
       };
       const [{ id: projectId }] = await this.projectsRepository.upsert(projectEntity);
       for (const tagId of project.tags || []) {
-        await this.tagsRepository.upsert({ id: tagId, runId });
+        await this.tagRepository.upsert({ id: tagId, runId });
         await this.projectsRepository.upsertRelationWithTag({ projectId, tagId, runId });
       }
       await this.searchService.upsert("project", projectEntity);
@@ -133,9 +155,22 @@ export class DigestCron {
 
               if (githubUser.type !== "User") continue;
 
-              // todo: call AIService
-              const name_en = githubUser.name || githubUser.login;
-              const name_ar = `ar ${name_en}`;
+              let name_en = githubUser.name || githubUser.login;
+              let name_ar = name_en;
+              try {
+                const aiRes = await this.aiService.query(
+                  [
+                    { role: "system", content: contributorNameSystemPrompt },
+                    { role: "user", content: name_en },
+                  ],
+                  AIResponseTranslateNameDto,
+                );
+
+                name_en = aiRes.name_en;
+                name_ar = aiRes.name_ar;
+              } catch (error) {
+                captureException(error, { tags: { type: "CRON" } });
+              }
 
               const contributorEntity: ContributorRow = {
                 name_en,
@@ -160,9 +195,22 @@ export class DigestCron {
 
               const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE";
 
-              // todo: call AIService
-              const title_en = issue.title;
-              const title_ar = `ar ${title_en}`;
+              let title_en = issue.title;
+              let title_ar = `ar ${title_en}`;
+              try {
+                const aiRes = await this.aiService.query(
+                  [
+                    { role: "system", content: issueTitleSystemPrompt },
+                    { role: "user", content: title_en },
+                  ],
+                  AIResponseTranslateTitleDto,
+                );
+
+                title_en = aiRes.title_en;
+                title_ar = aiRes.title_ar;
+              } catch (error) {
+                captureException(error, { tags: { type: "CRON" } });
+              }
 
               const contributionEntity: ContributionRow = {
                 title_en,
@@ -194,9 +242,22 @@ export class DigestCron {
                 username: repoContributor.login,
               });
 
-              // todo: call AIService
-              const name_en = contributor.name || contributor.login;
-              const name_ar = `ar ${name_en}`;
+              let name_en = contributor.name || contributor.login;
+              let name_ar = `ar ${name_en}`;
+              try {
+                const aiRes = await this.aiService.query(
+                  [
+                    { role: "system", content: contributorNameSystemPrompt },
+                    { role: "user", content: name_en },
+                  ],
+                  AIResponseTranslateNameDto,
+                );
+
+                name_en = aiRes.name_en;
+                name_ar = aiRes.name_ar;
+              } catch (error) {
+                captureException(error, { tags: { type: "CRON" } });
+              }
 
               const contributorEntity: ContributorRow = {
                 name_en,
@@ -244,7 +305,7 @@ export class DigestCron {
       await this.projectsRepository.deleteAllRelationWithTagButWithRunId(runId);
       await this.projectsRepository.deleteAllButWithRunId(runId);
 
-      await this.tagsRepository.deleteAllButWithRunId(runId);
+      await this.tagRepository.deleteAllButWithRunId(runId);
 
       await Promise.all([
         this.searchService.deleteAllButWithRunId("project", runId),
diff --git a/api/src/digest/dto.ts b/api/src/digest/dto.ts
@@ -0,0 +1,17 @@
+import { IsString } from "class-validator";
+
+export class AIResponseTranslateNameDto {
+  @IsString()
+  name_en!: string;
+
+  @IsString()
+  name_ar!: string;
+}
+
+export class AIResponseTranslateTitleDto {
+  @IsString()
+  title_en!: string;
+
+  @IsString()
+  title_ar!: string;
+}
diff --git a/api/src/fetch/service.ts b/api/src/fetch/service.ts
@@ -1,4 +1,4 @@
-import { defaults } from "make-fetch-happen";
+import { defaults, FetchOptions } from "make-fetch-happen";
 import { ConfigService } from "src/config/service";
 import { LoggerService } from "src/logger/service";
 import { Service } from "typedi";
@@ -18,21 +18,36 @@ export class FetchService {
     });
   }
 
+  public post = async <T>(
+    url: string,
+    { headers = {}, body }: FetchConfig = {},
+  ): Promise<Awaited<T>> => {
+    const response = await this.fetch<T>(url, {
+      headers: {
+        "Content-Type": "application/json",
+        ...headers,
+      },
+      method: "POST",
+      body: body ? JSON.stringify(body) : undefined,
+    });
+    return response;
+  };
+
   public get = async <T>(
     url: string,
     { params = {}, headers = {} }: FetchConfig = {},
   ): Promise<Awaited<T>> => {
     const _url = new URL(url);
     Object.keys(params).forEach((key) => _url.searchParams.append(key, String(params[key])));
 
-    const response = await this.fetch<T>(_url.toString(), { headers });
+    const response = await this.fetch<T>(_url.toString(), { headers, method: "GET" });
     return response;
   };
 
   private makeFetchHappenInstance;
-  private async fetch<T>(url: string, { headers }: Omit<FetchConfig, "params"> = {}) {
+  private async fetch<T>(url: string, options: FetchOptions) {
     this.logger.info({ message: `Fetching ${url}` });
-    const response = await this.makeFetchHappenInstance(url, { headers });
+    const response = await this.makeFetchHappenInstance(url, options);
     const jsonResponse = (await response.json()) as T;
     return jsonResponse;
   }
diff --git a/api/src/fetch/types.ts b/api/src/fetch/types.ts
@@ -1,4 +1,5 @@
 export interface FetchConfig {
   params?: Record<string, string | number | boolean>;
   headers?: Record<string, string>;
+  body?: Record<string, unknown>;
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/web/src/components/contribution-card.tsx b/web/src/components/contribution-card.tsx
diff --git a/web/src/components/project-card.tsx b/web/src/components/project-card.tsx
diff --git a/web/src/pages/contribute/contribution/index.tsx b/web/src/pages/contribute/contribution/index.tsx

Original file line number	Diff line number	Diff line change
`@@ -40,4 +40,7 @@ export class EnvRecord {`
`40`	`40`	`}`
`41`	`41`
`42`	`42`	`MEILISEARCH_MASTER_KEY = "default";`
	`43`	`+`
	`44`	`+ @IsString()`
	`45`	`+ OPENAI_KEY = "no-key";`
`43`	`46`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`export interface FetchConfig {`
`2`	`2`	`params?: Record<string, string \| number \| boolean>;`
`3`	`3`	`headers?: Record<string, string>;`
	`4`	`+ body?: Record<string, unknown>;`
`4`	`5`	`}`