version with eni requested changment

Author: 0xhijo

ingesters/src/ingesters/AsciiDocIngester.ts

@@ -161,13 +161,13 @@ export abstract class AsciiDocIngester extends BaseIngester {
    *
    * @param sourceDir - Source directory
    * @param targetDir - Target directory
-   * @param fileExtensions - File extension to process
+   * @param fileExtension - File extension to process
    * @returns Promise<boolean> - Whether relevant files were found
    */
   private async restructureFiles(
     sourceDir: string,
     targetDir: string,
-    fileExtensions: string = '.adoc',
+    fileExtension: string = '.adoc',
   ): Promise<boolean> {
     const entries = await fs.readdir(sourceDir, { withFileTypes: true });
     let hasRelevantFiles = false;
@@ -185,7 +185,7 @@ export abstract class AsciiDocIngester extends BaseIngester {
           const subDirHasRelevantFiles = await this.restructureFiles(
             sourcePath,
             targetDir,
-            fileExtensions,
+            fileExtension,
           );
           if (subDirHasRelevantFiles) {
             hasRelevantFiles = true;
@@ -194,16 +194,13 @@ export abstract class AsciiDocIngester extends BaseIngester {
           const subDirHasRelevantFiles = await this.restructureFiles(
             sourcePath,
             targetPath,
-            fileExtensions,
+            fileExtension,
           );
           if (subDirHasRelevantFiles) {
             hasRelevantFiles = true;
           }
         }
-      } else if (
-        entry.isFile() &&
-        path.extname(entry.name) === fileExtensions
-      ) {
+      } else if (entry.isFile() && path.extname(entry.name) === fileExtension) {
         if (entry.name.toLowerCase() === 'nav.adoc') {
           continue;
         }

ingesters/src/ingesters/OpenZeppelinDocsIngester.ts

@@ -2,7 +2,6 @@ import * as fsPromises from 'fs/promises';
 import * as path from 'path';
 import { Document } from '@langchain/core/documents';
 import { type BookChunk, DocumentSource } from '../types';
-import { VectorStore } from '../db/postgresVectorStore';
 import { type BookConfig, type BookPageDto } from '../utils/types';
 import { logger } from '../utils/logger';
 import {
@@ -68,33 +67,36 @@ export class OpenZeppelinDocsIngester extends AsciiDocIngester {
         const entries = await fsPromises.readdir(dir, { withFileTypes: true });
 
         for (const entry of entries) {
-          for (const fileExtension of config.fileExtensions) {
-            const fullPath = path.join(dir, entry.name);
+          const fullPath = path.join(dir, entry.name);
 
-            if (entry.isDirectory()) {
-              // Recursively process subdirectories
-              await processDirectory(fullPath);
-            } else if (
-              entry.isFile() &&
-              path.extname(entry.name).toLowerCase() === fileExtension // TODO: handle multiple extensions
-            ) {
-              // Process AsciiDoc files
-              const content = await fsPromises.readFile(fullPath, 'utf8');
+          if (entry.isDirectory()) {
+            // Recursively process subdirectories
+            await processDirectory(fullPath);
+          } else if (
+            entry.isFile() &&
+            config.fileExtensions.includes(
+              path.extname(entry.name).toLowerCase(),
+            )
+          ) {
+            // Process AsciiDoc files
+            const content = await fsPromises.readFile(fullPath, 'utf8');
 
-              // Get the relative path of the file from the base directory - which reflects the online website directory structure
-              const relativePath = path.relative(directory, fullPath);
+            // Get the relative path of the file from the base directory - which reflects the online website directory structure
+            const relativePath = path.relative(directory, fullPath);
 
-              // Inject cairo-contracts/2.0.0 in the fullPath to reflect online website directory structure
-              // This is the special handling for OpenZeppelin docs
-              const adaptedFullPageName = path
-                .join('contracts-cairo', OZ_DOCS_VERSION, relativePath)
-                .replace(fileExtension, '');
+            // Get the file extension to remove it from the final page name
+            const fileExtension = path.extname(entry.name).toLowerCase();
 
-              pages.push({
-                name: adaptedFullPageName,
-                content,
-              });
-            }
+            // Inject cairo-contracts/2.0.0 in the fullPath to reflect online website directory structure
+            // This is the special handling for OpenZeppelin docs
+            const adaptedFullPageName = path
+              .join('contracts-cairo', OZ_DOCS_VERSION, relativePath)
+              .replace(fileExtension, '');
+
+            pages.push({
+              name: adaptedFullPageName,
+              content,
+            });
           }
         }
       }

ingesters/src/ingesters/StarknetDocsIngester.ts

@@ -112,28 +112,28 @@ export class StarknetDocsIngester extends MarkdownIngester {
       const entries = await fs.readdir(dir, { withFileTypes: true });
 
       for (const entry of entries) {
-        for (const extension of this.config.fileExtensions) {
-          const fullPath = path.join(dir, entry.name);
-
-          if (entry.isDirectory()) {
-            // Recursively process subdirectories
-            await processDirectory(fullPath);
-          } else if (
-            entry.isFile() &&
-            path.extname(entry.name).toLowerCase() === extension // Handle specified extensions
-          ) {
-            // Process MDX files
-            const content = await fs.readFile(fullPath, 'utf8');
-
-            // Remove the repository path to get relative path
-            const relativePath = path.relative(this.getExtractDir(), fullPath);
-            const pageName = relativePath.replace('.mdx', '');
-
-            pages.push({
-              name: pageName,
-              content,
-            });
-          }
+        const fullPath = path.join(dir, entry.name);
+
+        if (entry.isDirectory()) {
+          // Recursively process subdirectories
+          await processDirectory(fullPath);
+        } else if (
+          entry.isFile() &&
+          this.config.fileExtensions.includes(
+            path.extname(entry.name).toLowerCase(),
+          )
+        ) {
+          // Process MDX files
+          const content = await fs.readFile(fullPath, 'utf8');
+
+          // Remove the repository path to get relative path
+          const relativePath = path.relative(this.getExtractDir(), fullPath);
+          const pageName = relativePath.replace('.mdx', '');
+
+          pages.push({
+            name: pageName,
+            content,
+          });
         }
       }
     };

python/optimizers/results/optimized_retrieval_program.json

@@ -9,10 +9,10 @@
 import os
 from typing import Optional
 
+import dspy
 import structlog
 from langsmith import traceable
 
-import dspy
 from cairo_coder.core.types import DocumentSource, ProcessedQuery
 
 logger = structlog.get_logger(__name__)