timeout change

Author: Farid-Karimli

edubotics_core/config/constants.py

@@ -6,7 +6,7 @@
 load_dotenv(".env")
 
 # Centralized definition of required constants for easy management and access
-TIMEOUT = os.getenv("TIMEOUT", 60)
+TIMEOUT = os.getenv("TIMEOUT", 120)
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
 LLAMA_CLOUD_API_KEY = os.getenv("LLAMA_CLOUD_API_KEY", "")
 HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN", "")

edubotics_core/dataloader/webpage_crawler.py

@@ -24,7 +24,8 @@ def url_exists(self, url: str) -> bool:
         try:
             response = requests.head(url, timeout=TIMEOUT)
             return response.status_code == 200
-        except requests.ConnectionError:
+        except Exception as e:
+            print(f"Error checking if URL exists: {e}")
             return False
 
     async def get_links(self, session: ClientSession, website_link: str, base_url: str):