feature: add ability for repository refetch on wiki refresh

api/api.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import logging
 from fastapi import FastAPI, HTTPException, Query, Request, WebSocket
 from fastapi.middleware.cors import CORSMiddleware
@@ -400,6 +401,23 @@ def generate_json_export(repo_url: str, pages: List[WikiPage]) -> str:
 # Add the WebSocket endpoint
 app.add_websocket_route("/ws/chat", handle_websocket_chat)
 
+# --- Wiki Repo Helper Functions ---
+WIKI_REPOS_DIR = os.path.join(get_adalflow_default_root_path(), "repos")
+os.makedirs(WIKI_REPOS_DIR, exist_ok=True)
+
+def get_wiki_repo_path(owner: str, repo: str) -> str:
+    filename = f"{owner}_{repo}"
+    return os.path.join(WIKI_REPOS_DIR, filename)
+
+
+# --- Wiki Databases Helper Functions ---
+WIKI_DATABASE_DIR = os.path.join(get_adalflow_default_root_path(), "databases")
+os.makedirs(WIKI_DATABASE_DIR, exist_ok=True)
+
+def get_wiki_database_path(owner: str, repo: str) -> str:
+    filename = f"{owner}_{repo}.pkl"
+    return os.path.join(WIKI_DATABASE_DIR, filename)
+
 # --- Wiki Cache Helper Functions ---
 
 WIKI_CACHE_DIR = os.path.join(get_adalflow_default_root_path(), "wikicache")
@@ -507,7 +525,8 @@ async def delete_wiki_cache(
     repo: str = Query(..., description="Repository name"),
     repo_type: str = Query(..., description="Repository type (e.g., github, gitlab)"),
     language: str = Query(..., description="Language of the wiki content"),
-    authorization_code: Optional[str] = Query(None, description="Authorization code")
+    authorization_code: Optional[str] = Query(None, description="Authorization code"),
+    force_refetch: bool = Query(False, description="Force re-clone the repository")
 ):
     """
     Deletes a specific wiki cache from the file system.
@@ -527,16 +546,29 @@ async def delete_wiki_cache(
 
     if os.path.exists(cache_path):
         try:
+            if force_refetch:
+                repo_dir = get_wiki_repo_path(owner, repo)
+                database_file = get_wiki_database_path(owner, repo)
+                if os.path.exists(repo_dir):
+                    shutil.rmtree(repo_dir)
+                if os.path.exists(database_file):
+                    os.remove(database_file)
+                logger.info(f"Wiki cache, repository and database for {owner}/{repo} ({language}) deleted successfully")
             os.remove(cache_path)
             logger.info(f"Successfully deleted wiki cache: {cache_path}")
             return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"}
         except Exception as e:
-            logger.error(f"Error deleting wiki cache {cache_path}: {e}")
-            raise HTTPException(status_code=500, detail=f"Failed to delete wiki cache: {str(e)}")
+            logger.error(f"Error during deletion for {owner}/{repo} ({language}): {e}")
+            detail = "Failed to delete wiki cache"
+            if force_refetch:
+                detail += ", repository and database"
+            raise HTTPException(status_code=500, detail=f"{detail}: {str(e)}")
     else:
         logger.warning(f"Wiki cache not found, cannot delete: {cache_path}")
         raise HTTPException(status_code=404, detail="Wiki cache not found")
 
+
+
 @app.get("/health")
 async def health_check():
     """Health check endpoint for Docker and monitoring"""

api/data_pipeline.py

@@ -170,8 +170,12 @@ def read_all_documents(path: str, embedder_type: str = None, is_ollama_embedder:
         embedder_type = 'ollama' if is_ollama_embedder else None
     documents = []
     # File extensions to look for, prioritizing code files
-    code_extensions = [".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs",
-                       ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs"]
+    code_extensions = [
+        ".py", ".js", ".ts", ".java", ".cpp", ".c", ".h", ".hpp", ".go", ".rs",
+        ".jsx", ".tsx", ".html", ".css", ".php", ".swift", ".cs", ".ino", ".class", ".kts",
+        ".sh", ".bat", ".ps1", ".rb", ".pl", ".lua", ".m", ".mm", ".vb", ".dart",
+        ".sql", ".r", ".scala", ".erl", ".ex", ".exs", ".clj", ".groovy"
+    ]
     doc_extensions = [".md", ".txt", ".rst", ".json", ".yaml", ".yml"]
 
     # Determine filtering mode: inclusion or exclusion

src/app/[owner]/[repo]/page.tsx

@@ -245,7 +245,7 @@ export default function RepoWikiPage() {
   const includedFiles = searchParams.get('included_files') || '';
   const [modelIncludedDirs, setModelIncludedDirs] = useState(includedDirs);
   const [modelIncludedFiles, setModelIncludedFiles] = useState(includedFiles);
-
+  const [forceRefetch, setForceRefetch] = useState(false);
 
   // Wiki type state - default to comprehensive view
   const isComprehensiveParam = searchParams.get('comprehensive') !== 'false';
@@ -1562,6 +1562,7 @@ IMPORTANT:
         is_custom_model: isCustomSelectedModelState.toString(),
         custom_model: customSelectedModelState,
         comprehensive: isComprehensiveView.toString(),
+        force_refetch: forceRefetch.toString(),
         authorization_code: authCode,
       });
 
@@ -1658,7 +1659,7 @@ IMPORTANT:
     // For now, we rely on the standard loadData flow initiated by resetting effectRan and dependencies.
     // This will re-trigger the main data loading useEffect.
     // No direct call to fetchRepositoryStructure here, let the useEffect handle it based on effectRan.current = false.
-  }, [effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]);
+  }, [forceRefetch, effectiveRepoInfo, language, messages.loading, activeContentRequests, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, isComprehensiveView, authCode, authRequired]);
 
   // Start wiki generation when component mounts
   useEffect(() => {
@@ -2248,6 +2249,8 @@ IMPORTANT:
         authCode={authCode}
         setAuthCode={setAuthCode}
         isAuthLoading={isAuthLoading}
+        forceRefetch={forceRefetch}
+        setForceRefetch={setForceRefetch}
       />
     </div>
   );

src/app/page.tsx

@@ -135,6 +135,7 @@ export default function Home() {
   const [includedDirs, setIncludedDirs] = useState('');
   const [includedFiles, setIncludedFiles] = useState('');
   const [selectedPlatform, setSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket'>('github');
+  const [forceRefetch, setForceRefetch] = useState(false);
   const [accessToken, setAccessToken] = useState('');
   const [error, setError] = useState<string | null>(null);
   const [isSubmitting, setIsSubmitting] = useState(false);
@@ -476,6 +477,8 @@ export default function Home() {
             authCode={authCode}
             setAuthCode={setAuthCode}
             isAuthLoading={isAuthLoading}
+            forceRefetch={forceRefetch}
+            setForceRefetch={setForceRefetch}
           />
 
         </div>

src/components/Ask.tsx

@@ -920,6 +920,8 @@ const Ask: React.FC<AskProps> = ({
         showWikiType={false}
         authRequired={false}
         isAuthLoading={false}
+        forceRefetch={false}
+        setForceRefetch={() => {}}
       />
     </div>
   );

src/components/ConfigurationModal.tsx

@@ -49,6 +49,10 @@ interface ConfigurationModalProps {
   includedFiles: string;
   setIncludedFiles: (value: string) => void;
 
+  // Repository refetch options
+  forceRefetch: boolean;
+  setForceRefetch: (forceRefetch: boolean) => void;
+
   // Form submission
   onSubmit: () => void;
   isSubmitting: boolean;
@@ -94,7 +98,9 @@ export default function ConfigurationModal({
   authRequired,
   authCode,
   setAuthCode,
-  isAuthLoading
+  isAuthLoading,
+  forceRefetch,
+  setForceRefetch,
 }: ConfigurationModalProps) {
   const { messages: t } = useLanguage();
 
@@ -211,23 +217,25 @@ export default function ConfigurationModal({
             {/* Model Selector */}
             <div className="mb-4">
               <UserSelector
-                provider={provider}
-                setProvider={setProvider}
-                model={model}
-                setModel={setModel}
-                isCustomModel={isCustomModel}
-                setIsCustomModel={setIsCustomModel}
-                customModel={customModel}
-                setCustomModel={setCustomModel}
-                showFileFilters={true}
-                excludedDirs={excludedDirs}
-                setExcludedDirs={setExcludedDirs}
-                excludedFiles={excludedFiles}
-                setExcludedFiles={setExcludedFiles}
-                includedDirs={includedDirs}
-                setIncludedDirs={setIncludedDirs}
-                includedFiles={includedFiles}
-                setIncludedFiles={setIncludedFiles}
+                  provider={provider}
+                  setProvider={setProvider}
+                  model={model}
+                  setModel={setModel}
+                  isCustomModel={isCustomModel}
+                  setIsCustomModel={setIsCustomModel}
+                  customModel={customModel}
+                  setCustomModel={setCustomModel}
+                  showFileFilters={true}
+                  excludedDirs={excludedDirs}
+                  setExcludedDirs={setExcludedDirs}
+                  excludedFiles={excludedFiles}
+                  setExcludedFiles={setExcludedFiles}
+                  includedDirs={includedDirs}
+                  setIncludedDirs={setIncludedDirs}
+                  includedFiles={includedFiles}
+                  setIncludedFiles={setIncludedFiles}
+                  forceRefetch={forceRefetch}
+                  setForceRefetch={setForceRefetch}
               />
             </div>
 

src/components/ModelSelectionModal.tsx

@@ -18,6 +18,8 @@ interface ModelSelectionModalProps {
   customModel: string;
   setCustomModel: (value: string) => void;
   onApply: (token?: string) => void;
+  forceRefetch: boolean;
+  setForceRefetch: (forceRefetch: boolean) => void;
 
   // Wiki type options
   isComprehensiveView: boolean;
@@ -75,6 +77,8 @@ export default function ModelSelectionModal({
   showWikiType = true,
   showTokenInput = false,
   repositoryType = 'github',
+  forceRefetch,
+  setForceRefetch,
 }: ModelSelectionModalProps) {
   const { messages: t } = useLanguage();
 
@@ -187,6 +191,8 @@ export default function ModelSelectionModal({
               setIncludedDirs={showFileFilters ? (value: string) => setLocalIncludedDirs(value) : undefined}
               includedFiles={localIncludedFiles}
               setIncludedFiles={showFileFilters ? (value: string) => setLocalIncludedFiles(value) : undefined}
+              forceRefetch={forceRefetch}
+              setForceRefetch={setForceRefetch}
             />
 
             {/* Token Input Section for refresh */}