version with all updated

Author: 0xhijo

ingesters/__tests__/AsciiDocIngester.test.ts

@@ -66,7 +66,7 @@ describe('AsciiDocIngester', () => {
       bookConfig: {
         repoOwner: 'test-owner',
         repoName: 'test-repo',
-        fileExtensions: '.adoc',
+        fileExtensions: ['.adoc'],
         chunkSize: 1000,
         chunkOverlap: 200,
         baseUrl: 'https://example.com',
@@ -300,7 +300,7 @@ This is page 2 content.`,
         bookConfig: {
           repoOwner: 'test-owner',
           repoName: 'test-repo',
-          fileExtensions: '.adoc',
+          fileExtensions: ['.adoc'],
           chunkSize: 1000,
           chunkOverlap: 200,
           baseUrl: '',

ingesters/__tests__/MarkdownIngester.test.ts

@@ -15,7 +15,7 @@ const markdownIngester = new TestMarkdownIngester(
     repoOwner: 'test',
     repoName: 'test',
     baseUrl: 'https://test.com',
-    fileExtensions: 'md',
+    fileExtensions: ['md'],
     urlSuffix: '.html',
     chunkSize: 1000,
     chunkOverlap: 100,
@@ -311,7 +311,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://docs.example.com',
-        fileExtensions: 'md',
+        fileExtensions: ['md'],
         urlSuffix: '.html',
         chunkSize: 1000,
         chunkOverlap: 100,
@@ -342,7 +342,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://docs.starknet.io',
-        fileExtensions: 'md',
+        fileExtensions: ['md'],
         urlSuffix: '',
         chunkSize: 1000,
         chunkOverlap: 100,
@@ -379,7 +379,7 @@ describe('URL sourcing and generation', () => {
         repoOwner: 'test',
         repoName: 'test',
         baseUrl: 'https://book.cairo-lang.org',
-        fileExtensions: 'md',
+        fileExtensions: ['md'],
         urlSuffix: '.html',
         chunkSize: 1000,
         chunkOverlap: 100,

ingesters/src/IngesterFactory.ts

@@ -49,8 +49,8 @@ export class IngesterFactory {
       case 'scarb_docs':
         return new ScarbDocsIngester();
 
-      case 'starknet_js':
-        return new StarknetJSIngester();
+      // case 'starknet_js':
+      //   return new StarknetJSIngester();
 
       case 'starknet_blog':
         return new StarknetBlogIngester();

ingesters/src/ingesters/StarknetJSIngester.ts

@@ -26,7 +26,7 @@ export class StarknetJSIngester extends MarkdownIngester {
       useUrlMapping: true,
     };
 
-    super(config, DocumentSource.STARKNET_JS);
+    super(config, DocumentSource.CAIRO_BOOK);
   }
 
   protected getExtractDir(): string {

ingesters/src/types/index.ts

@@ -15,7 +15,7 @@ export enum DocumentSource {
   OPENZEPPELIN_DOCS = 'openzeppelin_docs',
   CORELIB_DOCS = 'corelib_docs',
   SCARB_DOCS = 'scarb_docs',
-  STARKNET_JS = 'starknet_js',
+  // STARKNET_JS = 'starknet_js',
   STARKNET_BLOG = 'starknet_blog',
   DOJO_DOCS = 'dojo_docs',
 }

ingesters/src/utils/fileUtils.ts

@@ -26,20 +26,27 @@ export async function processDocFiles(
 
       for (const entry of entries) {
         const fullPath = path.join(dir, entry.name);
-        for (const fileExtension of config.fileExtensions) {
-          if (entry.isDirectory()) {
-            // Recursively process subdirectories
-            await processDirectory(fullPath);
-          } else if (
-            entry.isFile() &&
-            path.extname(entry.name).toLowerCase() === fileExtension
-          ) {
+
+        if (entry.isDirectory()) {
+          // Recursively process subdirectories
+          await processDirectory(fullPath);
+        } else if (entry.isFile()) {
+          // Check if the file matches any of the configured extensions
+          const fileExt = path.extname(entry.name).toLowerCase();
+          if (config.fileExtensions.includes(fileExt)) {
             // Process documentation files
             const content = await fs.readFile(fullPath, 'utf8');
+
+            // Skip empty files
+            if (content.trim().length === 0) {
+              logger.warn(`Skipping empty file: ${fullPath}`);
+              continue;
+            }
+
             pages.push({
               name: path
                 .relative(directory, fullPath)
-                .replace(fileExtension, ''),
+                .replace(fileExt, ''),
               content,
             });
           }