feat: add user agent

Author: VinciGit00

scrapegraph-py/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [1.26.0](https://github.com/ScrapeGraphAI/scrapegraph-sdk/compare/v1.25.1...v1.26.0) (2025-09-11)
+
+
+### Features
+
+* refactoring of the example folder ([78f2318](https://github.com/ScrapeGraphAI/scrapegraph-sdk/commit/78f23184626061b22dfde6d6b5b3c3df93f2a73a))
+
 ## [1.25.1](https://github.com/ScrapeGraphAI/scrapegraph-sdk/compare/v1.25.0...v1.25.1) (2025-09-08)
 
 

scrapegraph-py/scrapegraph_py/models/agenticscraper.py

@@ -48,7 +48,15 @@ class AgenticScraperRequest(BaseModel):
         default=False,
         description="Whether to use AI for data extraction from the scraped content"
     )
-
+    headers: Optional[dict[str, str]] = Field(
+        None,
+        example={
+            "User-Agent": "scrapegraph-py",
+            "Cookie": "cookie1=value1; cookie2=value2",
+        },
+        description="Optional headers to send with the request, including cookies "
+        "and user agent",
+    )
     mock: bool = Field(default=False, description="Whether to use mock mode for the request")
     @model_validator(mode="after")
     def validate_url(self) -> "AgenticScraperRequest":

scrapegraph-py/scrapegraph_py/models/crawl.py

@@ -47,6 +47,15 @@ class CrawlRequest(BaseModel):
     sitemap: bool = Field(
         default=False, description="Whether to use sitemap for better page discovery"
     )
+    headers: Optional[dict[str, str]] = Field(
+        None,
+        example={
+            "User-Agent": "scrapegraph-py",
+            "Cookie": "cookie1=value1; cookie2=value2",
+        },
+        description="Optional headers to send with the request, including cookies "
+        "and user agent",
+    )
 
     @model_validator(mode="after")
     def validate_url(self) -> "CrawlRequest":

scrapegraph-py/scrapegraph_py/models/markdownify.py

@@ -11,8 +11,7 @@ class MarkdownifyRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36",
+            "User-Agent": "scrapegraph-py",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
         description="Optional headers to send with the request, including cookies "

scrapegraph-py/scrapegraph_py/models/scheduled_jobs.py

@@ -15,7 +15,18 @@ class ScheduledJobCreate(BaseModel):
     job_name: str = Field(..., description="Name of the scheduled job")
     service_type: str = Field(..., description="Type of service (smartscraper, searchscraper, etc.)")
     cron_expression: str = Field(..., description="Cron expression for scheduling")
-    job_config: Dict[str, Any] = Field(..., description="Configuration for the job")
+    job_config: Dict[str, Any] = Field(
+        ..., 
+        example={
+            "website_url": "https://example.com",
+            "user_prompt": "Extract company information",
+            "headers": {
+                "User-Agent": "scrapegraph-py",
+                "Cookie": "session=abc123"
+            }
+        },
+        description="Configuration for the job"
+    )
     is_active: bool = Field(default=True, description="Whether the job is active")
 
 

scrapegraph-py/scrapegraph_py/models/scrape.py

@@ -15,8 +15,7 @@ class ScrapeRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36",
+            "User-Agent": "scrapegraph-py",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
         description="Optional headers to send with the request, including cookies "

scrapegraph-py/scrapegraph_py/models/searchscraper.py

@@ -19,8 +19,7 @@ class SearchScraperRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36",
+            "User-Agent": "scrapegraph-py",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
         description="Optional headers to send with the request, including cookies "

scrapegraph-py/scrapegraph_py/models/smartscraper.py

@@ -23,8 +23,7 @@ class SmartScraperRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36",
+            "User-Agent": "scrapegraph-py",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
         description="Optional headers to send with the request, including cookies "