Merge pull request #8 from ScrapeGraphAI/js-schema-implementation

Implemented support for requests with schema.

Author: DPende

package.json

@@ -9,9 +9,6 @@
   },
   "author": "ScrapeGraphAI",
   "license": "MIT",
-  "workspaces": [
-    "scrapegraph-js"
-  ],
   "scripts": {
     "semantic-release": "semantic-release"
   },

scrapegraph-js/examples/.env.example

@@ -1,2 +1,2 @@
 # ScrapegraphAI API Key
-SGAI-APIKEY="your ScrapegraphAI API Key"
+SGAI_APIKEY="your ScrapegraphAI API Key"

scrapegraph-js/examples/getCredits_example.js

@@ -1,11 +1,10 @@
-import { getCredits } from 'scrapegraph-sdk';
+import { getCredits } from 'scrapegraph-js';
 import 'dotenv/config';
 
-try {
-	const apiKey = process.env.SGAI_APIKEY;
+const apiKey = process.env.SGAI_APIKEY;
 
+try {
 	const myCredit = await getCredits(apiKey);
-
 	console.log(myCredit)
 } catch (error) {
 	console.error(error)

scrapegraph-js/examples/getSmartScraperRequest_example.js

@@ -1,12 +1,11 @@
-import { getSmartScraperRequest } from 'scrapegraph-sdk';
+import { getSmartScraperRequest } from 'scrapegraph-js';
 import 'dotenv/config';
 
-try {
-  const apiKey = process.env.SGAI_APIKEY;
-  const requestId = '3fa85f64-5717-4562-b3fc-2c963f66afa6'
+const apiKey = process.env.SGAI_APIKEY;
+const requestId = '3fa85f64-5717-4562-b3fc-2c963f66afa6'
 
+try {
   const requestInfo = await getSmartScraperRequest(apiKey, requestId);
-
   console.log(requestInfo);
 } catch (error) {
   console.error(error);

scrapegraph-js/examples/schema_smartScraper_example.js

@@ -0,0 +1,16 @@
+import { smartScraper } from 'scrapegraph-js';
+import { z } from 'zod';
+import 'dotenv/config';
+
+const apiKey = process.env.SGAI_APIKEY;
+const url = 'https://scrapegraphai.com/';
+const prompt = 'What does the company do? and ';
+
+const schema = 2;
+
+try {
+  const response = await smartScraper(apiKey, url, prompt, schema);
+  console.log(response.result);
+} catch (error) {
+  console.error(error);
+}

scrapegraph-js/examples/sendFeedback_example.js

@@ -1,12 +1,12 @@
-import { sendFeedback } from 'scrapegraph-sdk';
+import { sendFeedback } from 'scrapegraph-js';
 import 'dotenv/config';
 
-try {
-  const apiKey = process.env.SGAI_APIKEY;
-  const requestId = '16a63a80-c87f-4cde-b005-e6c3ecda278b';
-  const rating = 5;
-  const feedbackMessage = 'This is a test feedback message.';
+const apiKey = process.env.SGAI_APIKEY;
+const requestId = '16a63a80-c87f-4cde-b005-e6c3ecda278b';
+const rating = 5;
+const feedbackMessage = 'This is a test feedback message.';
 
+try {
   const feedback_response = await sendFeedback(apiKey, requestId, rating, feedbackMessage);
   console.log(feedback_response);
 } catch (error) {

scrapegraph-js/examples/smartScraper_example.js

@@ -1,13 +1,12 @@
-import { smartScraper } from 'scrapegraph-sdk';
+import { smartScraper } from 'scrapegraph-js';
 import 'dotenv/config';
 
-try {
-  const apiKey = process.env.SGAI_APIKEY;
-  const url = 'https://scrapegraphai.com';
-  const prompt = 'What does the company do?';
+const apiKey = process.env.SGAI_APIKEY;
+const url = 'https://scrapegraphai.com';
+const prompt = 'What does the company do?';
 
+try {
   const response = await smartScraper(apiKey, url, prompt);
-
   console.log(response);
 } catch (error) {
   console.error(error);

scrapegraph-js/package-lock.json

@@ -24,7 +24,8 @@
   "type": "module",
   "dependencies": {
     "axios": "^1.6.0",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.23.5"
   },
   "devDependencies": {
     "dotenv": "^16.4.5"

scrapegraph-js/package.json

@@ -77,9 +77,34 @@ const prompt = 'Extract the main heading and description.';
 ```
 
 #### Scraping with Custom Output Schema
+> [!NOTE]  
+> To use this feature, it is necessary to employ the [Zod](https://www.npmjs.com/package/zod) package for schema creation.
+
+Here is a real-world example:
 
 ```javascript
-//TODO
+import { smartScraper } from 'scrapegraph-js';
+import { z } from 'zod';
+import 'dotenv/config';
+
+const apiKey = 'your-api-key';
+const url = 'https://scrapegraphai.com/';
+const prompt = 'What does the company do? and ';
+
+const schema = z.object({
+  title: z.string().describe('The title of the webpage'),
+  description: z.string().describe('The description of the webpage'),
+  summary: z.string().describe('A brief summary of the webpage')
+});
+
+(async () => {
+  try {
+    const response = await smartScraper(apiKey, url, prompt, schema);
+    console.log(response.result);
+  } catch (error) {
+    console.error('Error:', error);
+  }
+})();
 ```
 
 ### Checking API Credits

scrapegraph-js/readme.md

@@ -1,5 +1,7 @@
 import axios from 'axios';
-import handleError from './utils/handleError.js'
+import handleError from './utils/handleError.js';
+import { ZodType } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
 
 /**
  * Scrape and extract structured data from a webpage using ScrapeGraph AI.
@@ -25,12 +27,11 @@ export async function smartScraper(apiKey, url, prompt, schema = null) {
   };
 
   if (schema) {
-    payload.output_schema = {
-      description: schema.title || 'Schema',
-      name: schema.title || 'Schema',
-      properties: schema.properties || {},
-      required: schema.required || []
-    };
+    if (schema instanceof ZodType) {
+      payload.output_schema = zodToJsonSchema(schema);
+    } else {
+      throw new Error('The schema must be an instance of a valid Zod schema');
+    }
   }
 
   try {