improve prompt of AI mdx-to-md convertor

Author: A-Najmabadi

mdx-to-md-converter/.gitignore

@@ -1,2 +1,4 @@
 node_modules
 dist
+.env
+.hash-cache.json

mdx-to-md-converter/src/index.ts

@@ -18,6 +18,7 @@ import { convertStep } from './modules/convertStep';
 import { removeHrAndLayout } from './modules/removeHrAndLayout';
 import { removeIndentations } from './modules/removeIndentations';
 import { overviewByAI } from './modules/overviewByAI';
+import { loadHashCache, saveHashCache, hasFileChanged, updateFileHash } from './modules/hashCache';
 
 import fs from 'fs';
 import path from 'path';
@@ -125,21 +126,50 @@ async function main() {
       fs.mkdirSync(outputDir, { recursive: true });
     }
 
-    // Array to store all the generated links
+    const hashCache = loadHashCache();
+    console.log('Loaded hash cache');
+    
     const allLinks: string[] = [];
 
+    let processedCount = 0;
+    let skippedCount = 0;
+    
     for (const filePath of mdxFiles) {
     try {
-      console.log(`Processing: ${filePath}`);
-      
       const mdxContent = readMdxFile(filePath);
 
+      if (!hasFileChanged(filePath, mdxContent, hashCache)) {
+        console.log(`Skipped (unchanged): ${filePath}`);
+        skippedCount++;
+        
+        const relativePath = getRelativePath(filePath, srcPagesPath);
+        const mdFileName = relativePath.replace(/\.mdx$/, '.md');
+        const outputFilePath = path.join(outputDir, mdFileName);
+        
+        if (fs.existsSync(outputFilePath)) {
+          const existingContent = fs.readFileSync(outputFilePath, 'utf-8');
+          const urlPath = `llms/${mdFileName.replace(/\\/g, '/')}`;
+          const url = `https://docs.liara.ir/${urlPath}`;
+          
+          let title = mdFileName.replace(/\.md$/, '').replace(/\//g, ' > ');
+          const headingMatch = existingContent.match(/^#\s+(.+)$/m);
+          if (headingMatch) {
+            title = headingMatch[1].trim();
+          }
+          
+          allLinks.push(`- [${title}](${url})`);
+        }
+        
+        continue;
+      }
+      
+      console.log(`Processing (new/modified): ${filePath}`);
+      processedCount++;
+      
       const mdContent = convertMdxToMd(mdxContent);
 
-      // Extra step: Store the final MD output in informal_md variable
       const informal_md = mdContent;
 
-      // Process with AI to get overview
       console.log(`Processing with AI: ${filePath}`);
       const aiProcessedContent = await overviewByAI(informal_md);
 
@@ -152,32 +182,26 @@ async function main() {
         fs.mkdirSync(outputFileDir, { recursive: true });
       }
 
-      // Build "original link" header (maps MD back to the human docs page)
-      // Example:
-      //   relativePath: "ai/google-gemini.mdx"
-      //   original URL: "https://docs.liara.ir/ai/google-gemini/"
       let originalPath = relativePath.replace(/\.mdx$/, '');
       if (originalPath.endsWith('/index')) {
         originalPath = originalPath.slice(0, -('/index'.length));
       }
       const originalUrl = `https://docs.liara.ir/${originalPath}${originalPath.endsWith('/') ? '' : '/'}`;
       const originalHeader = `Original link: ${originalUrl}\n\n`;
 
-      // Add the "all links" section to the end of each MD file
       const finalMdContent =
         originalHeader +
         aiProcessedContent +
         '\n\n## all links\n\n[All links of docs](https://docs.liara.ir/all-links-llms.txt)\n';
 
-      // Write file with explicit UTF-8 encoding and BOM
       fs.writeFileSync(outputFilePath, '\ufeff' + finalMdContent, { encoding: 'utf8' });
       console.log(`Saved: ${outputFilePath}`);
 
-      // Generate URL for all-links.txt (keep .md extension for static files)
+      updateFileHash(filePath, mdxContent, hashCache);
+      
       const urlPath = `llms/${mdFileName.replace(/\\/g, '/')}`;
       const url = `https://docs.liara.ir/${urlPath}`;
 
-      // Extract title from the first heading in the content or use filename
       let title = mdFileName.replace(/\.md$/, '').replace(/\//g, ' > ');
       const headingMatch = mdContent.match(/^#\s+(.+)$/m);
       if (headingMatch) {
@@ -191,21 +215,24 @@ async function main() {
     }
   }
 
-  // Generate all-links.txt content
-  const allLinksContent = `# All Links\n\n${allLinks.sort().join('\n')}\n`;
-  
-  // Write all-links.txt with explicit UTF-8 encoding and BOM
-  fs.writeFileSync(allLinksPath, '\ufeff' + allLinksContent, { encoding: 'utf8' });
-  console.log(`✅ All links saved to: ${allLinksPath}`);
-  
-    console.log(`All files converted and saved to: ${outputDir}`);
-    console.log(`Total files processed: ${mdxFiles.length}`);
+    saveHashCache(hashCache);
+    console.log('Saved hash cache');
+    
+    const allLinksContent = `# All Links\n\n${allLinks.sort().join('\n')}\n`;
+    
+    fs.writeFileSync(allLinksPath, '\ufeff' + allLinksContent, { encoding: 'utf8' });
+    console.log(`All links saved to: ${allLinksPath}`);
+    
+    console.log('\nSummary:');
+    console.log(`   Total files found: ${mdxFiles.length}`);
+    console.log(`   Processed (new/modified): ${processedCount}`);
+    console.log(`   Skipped (unchanged): ${skippedCount}`);
+    console.log(`   All files saved to: ${outputDir}`);
 
   } catch (err) {
     console.error('Error:', err);
     process.exit(1);
   }
 }
 
-// Run the main function
 main();

mdx-to-md-converter/src/modules/hashCache.ts

@@ -0,0 +1,48 @@
+import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
+
+const CACHE_FILE = path.join(process.cwd(), '.hash-cache.json');
+
+interface HashCache {
+  [filePath: string]: string;
+}
+
+export function generateHash(content: string): string {
+  return crypto.createHash('md5').update(content).digest('hex');
+}
+
+export function loadHashCache(): HashCache {
+  try {
+    if (fs.existsSync(CACHE_FILE)) {
+      const data = fs.readFileSync(CACHE_FILE, 'utf-8');
+      return JSON.parse(data);
+    }
+  } catch (error) {
+    console.warn('Failed to load hash cache, starting fresh:', error);
+  }
+  return {};
+}
+
+export function saveHashCache(cache: HashCache): void {
+  try {
+    fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2), 'utf-8');
+  } catch (error) {
+    console.error('Failed to save hash cache:', error);
+  }
+}
+
+export function hasFileChanged(filePath: string, content: string, cache: HashCache): boolean {
+  const currentHash = generateHash(content);
+  const cachedHash = cache[filePath];
+  
+  if (!cachedHash) {
+    return true;
+  }
+  
+  return currentHash !== cachedHash;
+}
+
+export function updateFileHash(filePath: string, content: string, cache: HashCache): void {
+  cache[filePath] = generateHash(content);
+}

mdx-to-md-converter/src/modules/overviewByAI.ts

@@ -4,6 +4,7 @@ import { generateText } from 'ai';
 
 export async function overviewByAI(informal_md: string): Promise<string> {
   try {
+    
     const baseURL = process.env.MY_BASE_URL;
     const apiKey = process.env.MY_API_KEY;
 
@@ -16,8 +17,21 @@ export async function overviewByAI(informal_md: string): Promise<string> {
         name: 'liara-ai',
         baseURL,
         apiKey,
-      }).chatModel("google/gemini-2.0-flash-001"),
-      prompt: `convert the text below to the pure markdown file, do not change any content on it:\n\n${informal_md}`,
+      }).chatModel("openai/gpt-4.1-mini"),
+      prompt: `Convert the following incorrect Markdown content into standard Markdown. Do not rewrite, summarize, delete, add, or modify any content. Only remove or transform MDX elements into the valid Markdown.
+Input MDX:
+
+\n\n${informal_md}\n\n
+
+Output the result as pure Markdown only, with no explanations or commentary.
+
+considaring: 
+- If you encounter links with a full path (starting with \`/\`), rewrite them so they begin with: https://docs.liara.ir
+- If you encounter links with a relative path (starting with dot), DO NOT Change it
+- Treat <Card /> as lists in the resulting Markdown and only link matters (not icons)
+- Treat <Tab /> and their content as separated H2s. 
+
+`,
     });
 
     return text;