version without an specific dojo agent

0xhijo · 0xhijo · commit d131aa02b778 · 2025-12-02T18:32:48.000+01:00
diff --git a/ingesters/__tests__/AsciiDocIngester.test.ts b/ingesters/__tests__/AsciiDocIngester.test.ts
@@ -66,7 +66,7 @@ describe('AsciiDocIngester', () => {
       bookConfig: {
         repoOwner: 'test-owner',
         repoName: 'test-repo',
-        fileExtension: '.adoc',
+        fileExtensions: '.adoc',
         chunkSize: 1000,
         chunkOverlap: 200,
         baseUrl: 'https://example.com',
@@ -300,7 +300,7 @@ This is page 2 content.`,
         bookConfig: {
           repoOwner: 'test-owner',
           repoName: 'test-repo',
-          fileExtension: '.adoc',
+          fileExtensions: '.adoc',
           chunkSize: 1000,
           chunkOverlap: 200,
           baseUrl: '',
diff --git a/ingesters/__tests__/DojoDocsIngester.test.ts b/ingesters/__tests__/DojoDocsIngester.test.ts
@@ -12,7 +12,7 @@ describe('DojoDocsIngester', () => {
     expect(ingester.config).toEqual({
       repoOwner: 'dojoengine',
       repoName: 'book',
-      fileExtension: '.md',
+      fileExtensions: ['.md', '.mdx'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://book.dojoengine.org',
diff --git a/ingesters/__tests__/MarkdownIngester.test.ts b/ingesters/__tests__/MarkdownIngester.test.ts
@@ -15,7 +15,7 @@ const markdownIngester = new TestMarkdownIngester(
     repoOwner: 'test',
     repoName: 'test',
     baseUrl: 'https://test.com',
-    fileExtension: 'md',
+    fileExtensions: 'md',
     urlSuffix: '.html',
     chunkSize: 1000,
     chunkOverlap: 100,
@@ -311,7 +311,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://docs.example.com',
-        fileExtension: 'md',
+        fileExtensions: 'md',
         urlSuffix: '.html',
         chunkSize: 1000,
         chunkOverlap: 100,
@@ -342,7 +342,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://docs.starknet.io',
-        fileExtension: 'md',
+        fileExtensions: 'md',
         urlSuffix: '',
         chunkSize: 1000,
         chunkOverlap: 100,
@@ -379,7 +379,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://book.cairo-lang.org',
-        fileExtension: 'md',
+        fileExtensions: 'md',
         urlSuffix: '.html',
         chunkSize: 1000,
         chunkOverlap: 100,
diff --git a/ingesters/__tests__/StarknetDocsIngester.test.ts b/ingesters/__tests__/StarknetDocsIngester.test.ts
@@ -12,7 +12,7 @@ describe('StarknetDocsIngester', () => {
     expect(ingester.config).toEqual({
       repoOwner: 'starknet-io',
       repoName: 'starknet-docs',
-      fileExtension: '.mdx',
+      fileExtensions: '.mdx',
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://docs.starknet.io',
diff --git a/ingesters/src/ingesters/AsciiDocIngester.ts b/ingesters/src/ingesters/AsciiDocIngester.ts
@@ -161,13 +161,13 @@ export abstract class AsciiDocIngester extends BaseIngester {
    *
    * @param sourceDir - Source directory
    * @param targetDir - Target directory
-   * @param fileExtension - File extension to process
+   * @param fileExtensions - File extension to process
    * @returns Promise<boolean> - Whether relevant files were found
    */
   private async restructureFiles(
     sourceDir: string,
     targetDir: string,
-    fileExtension: string = '.adoc',
+    fileExtensions: string = '.adoc',
   ): Promise<boolean> {
     const entries = await fs.readdir(sourceDir, { withFileTypes: true });
     let hasRelevantFiles = false;
@@ -185,7 +185,7 @@ export abstract class AsciiDocIngester extends BaseIngester {
           const subDirHasRelevantFiles = await this.restructureFiles(
             sourcePath,
             targetDir,
-            fileExtension,
+            fileExtensions,
           );
           if (subDirHasRelevantFiles) {
             hasRelevantFiles = true;
@@ -194,13 +194,13 @@ export abstract class AsciiDocIngester extends BaseIngester {
           const subDirHasRelevantFiles = await this.restructureFiles(
             sourcePath,
             targetPath,
-            fileExtension,
+            fileExtensions,
           );
           if (subDirHasRelevantFiles) {
             hasRelevantFiles = true;
           }
         }
-      } else if (entry.isFile() && path.extname(entry.name) === fileExtension) {
+      } else if (entry.isFile() && path.extname(entry.name) === fileExtensions) {
         if (entry.name.toLowerCase() === 'nav.adoc') {
           continue;
         }
diff --git a/ingesters/src/ingesters/CairoBookIngester.ts b/ingesters/src/ingesters/CairoBookIngester.ts
@@ -29,7 +29,7 @@ export class CairoBookIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'cairo-book',
       repoName: 'cairo-book',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://book.cairo-lang.org',
diff --git a/ingesters/src/ingesters/CairoByExampleIngester.ts b/ingesters/src/ingesters/CairoByExampleIngester.ts
@@ -17,7 +17,7 @@ export class CairoByExampleIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'enitrat',
       repoName: 'cairo-by-example',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://enitrat.github.io/cairo-by-example',
diff --git a/ingesters/src/ingesters/CoreLibDocsIngester.ts b/ingesters/src/ingesters/CoreLibDocsIngester.ts
@@ -29,7 +29,7 @@ export class CoreLibDocsIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'enitrat',
       repoName: 'cairo-docs',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://docs.starknet.io/build/corelib/intro',
diff --git a/ingesters/src/ingesters/DojoDocsIngester.ts b/ingesters/src/ingesters/DojoDocsIngester.ts
@@ -21,7 +21,7 @@ export class DojoDocsIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoName: 'book',
       repoOwner: 'dojoengine',
-      fileExtension: '.md',
+      fileExtensions: ['.md', '.mdx'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: DojoDocsIngester.BASE_URL,
diff --git a/ingesters/src/ingesters/OpenZeppelinDocsIngester.ts b/ingesters/src/ingesters/OpenZeppelinDocsIngester.ts
@@ -28,7 +28,7 @@ export class OpenZeppelinDocsIngester extends AsciiDocIngester {
     const config: BookConfig = {
       repoOwner: 'OpenZeppelin',
       repoName: 'cairo-contracts',
-      fileExtension: '.adoc',
+      fileExtensions: ['.adoc'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://docs.openzeppelin.com',
@@ -68,31 +68,33 @@ export class OpenZeppelinDocsIngester extends AsciiDocIngester {
         const entries = await fsPromises.readdir(dir, { withFileTypes: true });
 
         for (const entry of entries) {
-          const fullPath = path.join(dir, entry.name);
+          for (const fileExtension of config.fileExtensions) {
+            const fullPath = path.join(dir, entry.name);
 
-          if (entry.isDirectory()) {
-            // Recursively process subdirectories
-            await processDirectory(fullPath);
-          } else if (
-            entry.isFile() &&
-            path.extname(entry.name).toLowerCase() === config.fileExtension
-          ) {
-            // Process AsciiDoc files
-            const content = await fsPromises.readFile(fullPath, 'utf8');
+            if (entry.isDirectory()) {
+              // Recursively process subdirectories
+              await processDirectory(fullPath);
+            } else if (
+              entry.isFile() &&
+              path.extname(entry.name).toLowerCase() === fileExtension // TODO: handle multiple extensions
+            ) {
+              // Process AsciiDoc files
+              const content = await fsPromises.readFile(fullPath, 'utf8');
 
-            // Get the relative path of the file from the base directory - which reflects the online website directory structure
-            const relativePath = path.relative(directory, fullPath);
+              // Get the relative path of the file from the base directory - which reflects the online website directory structure
+              const relativePath = path.relative(directory, fullPath);
 
-            // Inject cairo-contracts/2.0.0 in the fullPath to reflect online website directory structure
-            // This is the special handling for OpenZeppelin docs
-            const adaptedFullPageName = path
-              .join('contracts-cairo', OZ_DOCS_VERSION, relativePath)
-              .replace(config.fileExtension, '');
+              // Inject cairo-contracts/2.0.0 in the fullPath to reflect online website directory structure
+              // This is the special handling for OpenZeppelin docs
+              const adaptedFullPageName = path
+                .join('contracts-cairo', OZ_DOCS_VERSION, relativePath)
+                .replace(fileExtension, '');
 
-            pages.push({
-              name: adaptedFullPageName,
-              content,
-            });
+              pages.push({
+                name: adaptedFullPageName,
+                content,
+              });
+            }
           }
         }
       }
diff --git a/ingesters/src/ingesters/ScarbDocsIngester.ts b/ingesters/src/ingesters/ScarbDocsIngester.ts
@@ -23,7 +23,7 @@ export class ScarbDocsIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'software-mansion',
       repoName: 'scarb',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://docs.swmansion.com/scarb/docs',
diff --git a/ingesters/src/ingesters/StarknetBlogIngester.ts b/ingesters/src/ingesters/StarknetBlogIngester.ts
@@ -30,7 +30,7 @@ export class StarknetBlogIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'starknet',
       repoName: 'starknet-blog',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://www.starknet.io/blog',
diff --git a/ingesters/src/ingesters/StarknetDocsIngester.ts b/ingesters/src/ingesters/StarknetDocsIngester.ts
@@ -33,7 +33,7 @@ export class StarknetDocsIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'starknet-io',
       repoName: 'starknet-docs',
-      fileExtension: '.mdx',
+      fileExtensions: ['.mdx'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: StarknetDocsIngester.BASE_URL,
@@ -112,26 +112,28 @@ export class StarknetDocsIngester extends MarkdownIngester {
       const entries = await fs.readdir(dir, { withFileTypes: true });
 
       for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
-
-        if (entry.isDirectory()) {
-          // Recursively process subdirectories
-          await processDirectory(fullPath);
-        } else if (
-          entry.isFile() &&
-          path.extname(entry.name).toLowerCase() === this.config.fileExtension
-        ) {
-          // Process MDX files
-          const content = await fs.readFile(fullPath, 'utf8');
-
-          // Remove the repository path to get relative path
-          const relativePath = path.relative(this.getExtractDir(), fullPath);
-          const pageName = relativePath.replace('.mdx', '');
-
-          pages.push({
-            name: pageName,
-            content,
-          });
+        for (const extension of this.config.fileExtensions) {
+          const fullPath = path.join(dir, entry.name);
+
+          if (entry.isDirectory()) {
+            // Recursively process subdirectories
+            await processDirectory(fullPath);
+          } else if (
+            entry.isFile() &&
+            path.extname(entry.name).toLowerCase() === extension // Handle specified extensions
+          ) {
+            // Process MDX files
+            const content = await fs.readFile(fullPath, 'utf8');
+
+            // Remove the repository path to get relative path
+            const relativePath = path.relative(this.getExtractDir(), fullPath);
+            const pageName = relativePath.replace('.mdx', '');
+
+            pages.push({
+              name: pageName,
+              content,
+            });
+          }
         }
       }
     };
diff --git a/ingesters/src/ingesters/StarknetFoundryIngester.ts b/ingesters/src/ingesters/StarknetFoundryIngester.ts
@@ -27,7 +27,7 @@ export class StarknetFoundryIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'foundry-rs',
       repoName: 'starknet-foundry',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://foundry-rs.github.io/starknet-foundry',
diff --git a/ingesters/src/ingesters/StarknetJSIngester.ts b/ingesters/src/ingesters/StarknetJSIngester.ts
@@ -18,15 +18,15 @@ export class StarknetJSIngester extends MarkdownIngester {
     const config: BookConfig = {
       repoOwner: 'starknet-io',
       repoName: 'starknet.js',
-      fileExtension: '.md',
+      fileExtensions: ['.md'],
       chunkSize: 4096,
       chunkOverlap: 512,
       baseUrl: 'https://starknetjs.com/docs/next/guides',
       urlSuffix: '',
       useUrlMapping: true,
     };
 
-    super(config, DocumentSource.CAIRO_BOOK);
+    super(config, DocumentSource.STARKNET_JS);
   }
 
   protected getExtractDir(): string {
diff --git a/ingesters/src/shared.ts b/ingesters/src/shared.ts
@@ -10,7 +10,7 @@ export interface BookPageDto {
 export type BookConfig = {
   repoOwner: string;
   repoName: string;
-  fileExtension: string;
+  fileExtensions: string;
   chunkSize: number;
   chunkOverlap: number;
 };
diff --git a/ingesters/src/utils/fileUtils.ts b/ingesters/src/utils/fileUtils.ts
@@ -26,23 +26,23 @@ export async function processDocFiles(
 
       for (const entry of entries) {
         const fullPath = path.join(dir, entry.name);
-
-        if (entry.isDirectory()) {
-          // Recursively process subdirectories
-          await processDirectory(fullPath);
-        } else if (
-          entry.isFile() &&
-          path.extname(entry.name).toLowerCase() === config.fileExtension
-        ) {
-          // Process documentation files
-          const content = await fs.readFile(fullPath, 'utf8');
-
-          pages.push({
-            name: path
-              .relative(directory, fullPath)
-              .replace(config.fileExtension, ''),
-            content,
-          });
+        for (const fileExtension of config.fileExtensions) {
+          if (entry.isDirectory()) {
+            // Recursively process subdirectories
+            await processDirectory(fullPath);
+          } else if (
+            entry.isFile() &&
+            path.extname(entry.name).toLowerCase() === fileExtension
+          ) {
+            // Process documentation files
+            const content = await fs.readFile(fullPath, 'utf8');
+            pages.push({
+              name: path
+                .relative(directory, fullPath)
+                .replace(fileExtension, ''),
+              content,
+            });
+          }
         }
       }
     }
diff --git a/ingesters/src/utils/types.ts b/ingesters/src/utils/types.ts
@@ -31,7 +31,7 @@ export type BookConfig = {
   repoName: string;
 
   /** The file extension of the documentation files */
-  fileExtension: string;
+  fileExtensions: Array<string>;
 
   /** The size of each chunk in characters */
   chunkSize: number;
diff --git a/python/optimizers/datasets/dojo_queries.json b/python/optimizers/datasets/dojo_queries.json
diff --git a/python/optimizers/results/optimized_generation_dojo-agent.json b/python/optimizers/results/optimized_generation_dojo-agent.json
diff --git a/python/src/cairo_coder/agents/registry.py b/python/src/cairo_coder/agents/registry.py
diff --git a/python/src/cairo_coder/dspy/generation_program.py b/python/src/cairo_coder/dspy/generation_program.py
diff --git a/python/src/cairo_coder/optimizers/generation_optimizer_dojo-agent.py b/python/src/cairo_coder/optimizers/generation_optimizer_dojo-agent.py
diff --git a/python/tests/unit/test_agent_factory.py b/python/tests/unit/test_agent_factory.py
