From 3c2178e04e873885abc8aca0312f5a4a1dd9cdd0 Mon Sep 17 00:00:00 2001 From: Lorenzo Padoan Date: Thu, 28 Nov 2024 17:28:58 +0100 Subject: [PATCH 1/3] fix: readme js sdk --- scrapegraph-js/readme.md | 238 +++++++++++++++++++++++---------------- scrapegraph-py/README.md | 7 +- 2 files changed, 147 insertions(+), 98 deletions(-) diff --git a/scrapegraph-js/readme.md b/scrapegraph-js/readme.md index f4db6b6..2abcf17 100644 --- a/scrapegraph-js/readme.md +++ b/scrapegraph-js/readme.md @@ -1,150 +1,200 @@ -# ScrapeGraph JS SDK +# 🌐 ScrapeGraph JavaScript SDK -A JavaScript SDK for interacting with the ScrapeGraph AI API. This SDK provides easy-to-use functions for web scraping, managing credits, and submitting feedback. +[![npm version](https://badge.fury.io/js/scrapegraph-js.svg)](https://badge.fury.io/js/scrapegraph-js) +[![TypeScript Support](https://img.shields.io/badge/TypeScript-Ready-blue.svg)](https://www.typescriptlang.org/) +[![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +[![Build Status](https://github.com/ScrapeGraphAI/scrapegraph-sdk/actions/workflows/ci.yml/badge.svg)](https://github.com/ScrapeGraphAI/scrapegraph-sdk/actions) +[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://docs.scrapegraphai.com) -## Installation +Official JavaScript/TypeScript SDK for the ScrapeGraph AI API - Smart web scraping powered by AI. -Install the package using npm: +## 🚀 Features -```bash -npm install scrapegraph-js -``` +- ✨ Smart web scraping with AI +- 🔄 Fully asynchronous design +- 📊 TypeScript-ready with strongly typed responses +- 🔍 Detailed error handling +- ⚡ Automatic retries and logging +- 🔐 Secure API authentication -## Usage +## 📦 Installation -> [!WARNING] -> Remember not to write API keys directly in the code; instead, store them securely in `.env` files. +Install the package using npm or yarn: -First, import the required functions: +```bash +# Using npm +npm install scrapegraph-js -```javascript -import { smartScraper, getSmartScraperRequest, getCredits, sendFeedback } from 'scrapegraph-sdk'; -``` -### Scraping Websites +## 🔧 Quick Start + +> **Note**: Store your API keys securely in environment variables. Use `.env` files and libraries like `dotenv` to load them into your app. -#### Basic scraping +### Basic Example ```javascript -import { smartScraper } from 'scrapegraph-sdk'; +import { smartScraper } from 'scrapegraph-js'; -const apiKey = process.env.SGAI_APIKEY; -const url = 'https://scrapegraphai.com'; +// Initialize variables +const apiKey = process.env.SGAI_APIKEY; // Set your API key as an environment variable +const websiteUrl = 'https://example.com'; const prompt = 'What does the company do?'; -try { - const response = await smartScraper(apiKey, url, prompt); - console.log(response); -} catch (error) { - console.error(error); -} +(async () => { + try { + const response = await smartScraper(apiKey, websiteUrl, prompt); + console.log(response.result); + } catch (error) { + console.error('Error:', error); + } +})(); ``` -#### Scraping with custom output schema +## 🎯 Examples + +### Scraping Websites + +#### Basic Scraping ```javascript -import { smartScraper } from 'scrapegraph-sdk'; +import { smartScraper } from 'scrapegraph-js'; + +const apiKey = 'your-api-key'; +const url = 'https://example.com'; +const prompt = 'Extract the main heading and description.'; + +(async () => { + try { + const response = await smartScraper(apiKey, url, prompt); + console.log(response.result); + } catch (error) { + console.error('Error:', error); + } +})(); +``` -const apiKey = 'your_api_key'; -const url = 'https://scrapegraphai.com'; -const prompt = 'What does the company do?'; -const schema = //TODO +#### Scraping with Custom Output Schema + +```typescript +import { smartScraper } from 'scrapegraph-js'; -try { - const response = await smartScraper(apiKey, url, prompt, schema); - console.log(response); -} catch (error) { - console.error(error); +interface WebsiteData { + title: string; + description: string; } + +const apiKey = 'your-api-key'; +const url = 'https://example.com'; +const prompt = 'Extract the title and description.'; + +(async () => { + try { + const response = await smartScraper(apiKey, url, prompt); + console.log(response.result.title, response.result.description); + } catch (error) { + console.error('Error:', error); + } +})(); ``` -### Checking Credits +### Checking API Credits ```javascript -import { getCredist } from 'scrapegraph-sdk'; - -const apiKey = 'your_api_key'; - -try { - const myCredit = await getCredits(apiKey); - console.log(myCredit) -} catch (error) { - console.error(error) -} +import { getCredits } from 'scrapegraph-js'; + +const apiKey = 'your-api-key'; + +(async () => { + try { + const credits = await getCredits(apiKey); + console.log('Available credits:', credits); + } catch (error) { + console.error('Error fetching credits:', error); + } +})(); ``` ### Submitting Feedback ```javascript -import { sendFeedback } from 'scrapegraph-sdk'; +import { sendFeedback } from 'scrapegraph-js'; -const apiKey = 'your_api_key'; +const apiKey = 'your-api-key'; const requestId = '16a63a80-c87f-4cde-b005-e6c3ecda278b'; const rating = 5; -const feedbackMessage = 'This is a test feedback message.'; - -try { - const feedback_response = await sendFeedback(apiKey, requestId, rating, feedbackMessage); - console.log(feedback_response); -} catch (error) { - console.error(error) -} +const feedbackText = 'This is a test feedback message.'; + +(async () => { + try { + const response = await sendFeedback(apiKey, requestId, rating, feedbackText); + console.log('Feedback response:', response); + } catch (error) { + console.error('Error sending feedback:', error); + } +})(); ``` -## API Reference - -### scrape(apiKey, url[, options]) +## 📚 Documentation -Scrapes a website and returns the extracted data. +For detailed documentation, visit [docs.scrapegraphai.com](https://docs.scrapegraphai.com) -Parameters: -- `apiKey` (string): Your ScrapeGraph AI API key -- `url` (string): The URL to scrape -- `options` (object, optional): - - `elements` (array): Specific elements to extract - - `wait_for` (string): CSS selector to wait for before scraping - - `javascript` (boolean): Enable JavaScript rendering +## 🛠️ Development -### credits(apiKey) +### Setup -Retrieves your current credit balance. +1. Clone the repository: + ```bash + git clone https://github.com/ScrapeGraphAI/scrapegraph-sdk.git + cd scrapegraph-sdk/scrapegraph-js + ``` -Parameters: -- `apiKey` (string): Your ScrapeGraph AI API key +2. Install dependencies: + ```bash + npm install + ``` -### feedback(apiKey, requestId, rating, feedbackText) +3. Run linting and testing: + ```bash + npm run lint + npm test + ``` -Submits feedback for a scraping request. +### Running Tests -Parameters: -- `apiKey` (string): Your ScrapeGraph AI API key -- `requestId` (string): The request ID from the scrape response -- `rating` (number): Rating score -- `feedbackText` (string) (optional): Feedback message +```bash +# Run all tests +npm test -## Error Handling +# Run tests with coverage +npm run test:coverage +``` -All functions return javascript `Error` object with imformation. In case of errors, the response will include error details: +## 📝 License -// TODO error list +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -```javascript -{ - "statusCode": 400, - "title": "HTTP error occurred" - "details": "Error details", - -} -``` +## 🤝 Contributing -## License +Contributions are welcome! Please feel free to submit a Pull Request. For major changes, please open an issue first to discuss what you would like to change. -MIT +1. Fork the repository +2. Create your feature branch (`git checkout -b feature/AmazingFeature`) +3. Commit your changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the branch (`git push origin feature/AmazingFeature`) +5. Open a Pull Request -## Support +## 🔗 Links -For support, please visit [ScrapeGraph AI Documentation](https://sgai-api.onrender.com/docs). +- [Website](https://scrapegraphai.com) +- [Documentation](https://scrapegraphai.com/documentation) +- [GitHub](https://github.com/ScrapeGraphAI/scrapegraph-sdk) +## 💬 Support +- 📧 Email: support@scrapegraphai.com +- 💻 GitHub Issues: [Create an issue](https://github.com/ScrapeGraphAI/scrapegraph-sdk/issues) +- 🌟 Feature Requests: [Request a feature](https://github.com/ScrapeGraphAI/scrapegraph-sdk/issues/new) +--- +Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com) diff --git a/scrapegraph-py/README.md b/scrapegraph-py/README.md index f2e40f9..221698e 100644 --- a/scrapegraph-py/README.md +++ b/scrapegraph-py/README.md @@ -155,10 +155,9 @@ Contributions are welcome! Please feel free to submit a Pull Request. For major ## 🔗 Links -- [Website](https://scrapegraphai.com) -- [Documentation](https://docs.scrapegraphai.com) -- [API Reference](https://docs.scrapegraphai.com/api) -- [GitHub](https://github.com/ScrapeGraphAI/scrapegraph-sdk) +- [Website](https://scrapegraphai.com) +- [Documentation](https://scrapegraphai.com/documentation) +- [GitHub](https://github.com/ScrapeGraphAI/scrapegraph-sdk) ## 💬 Support From 9e9e138617658e068a1c77a4dbac24b4d550d42a Mon Sep 17 00:00:00 2001 From: DPende Date: Thu, 28 Nov 2024 17:57:23 +0100 Subject: [PATCH 2/3] chore: changed pakage name --- scrapegraph-js/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapegraph-js/package.json b/scrapegraph-js/package.json index e1da1c4..e2d089d 100644 --- a/scrapegraph-js/package.json +++ b/scrapegraph-js/package.json @@ -1,5 +1,5 @@ { - "name": "scrapegraph-sdk", + "name": "scrapegraph-js", "author": "ScrapeGraphAI", "version": "0.0.1", "description": "Scrape and extract structured data from a webpage using ScrapeGraph AI.", @@ -23,7 +23,8 @@ "module": "index.js", "type": "module", "dependencies": { - "axios": "^1.6.0" + "axios": "^1.6.0", + "zod": "^3.23.8" }, "devDependencies": { "dotenv": "^16.4.5" From 88a2f509dc34ad69f41fe6d13f31de191895bc1a Mon Sep 17 00:00:00 2001 From: DPende Date: Thu, 28 Nov 2024 17:58:17 +0100 Subject: [PATCH 3/3] fix: removed wrong information --- scrapegraph-js/readme.md | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/scrapegraph-js/readme.md b/scrapegraph-js/readme.md index 2abcf17..a69e313 100644 --- a/scrapegraph-js/readme.md +++ b/scrapegraph-js/readme.md @@ -1,7 +1,6 @@ # 🌐 ScrapeGraph JavaScript SDK [![npm version](https://badge.fury.io/js/scrapegraph-js.svg)](https://badge.fury.io/js/scrapegraph-js) -[![TypeScript Support](https://img.shields.io/badge/TypeScript-Ready-blue.svg)](https://www.typescriptlang.org/) [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![Build Status](https://github.com/ScrapeGraphAI/scrapegraph-sdk/actions/workflows/ci.yml/badge.svg)](https://github.com/ScrapeGraphAI/scrapegraph-sdk/actions) [![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://docs.scrapegraphai.com) @@ -11,8 +10,7 @@ Official JavaScript/TypeScript SDK for the ScrapeGraph AI API - Smart web scrapi ## 🚀 Features - ✨ Smart web scraping with AI -- 🔄 Fully asynchronous design -- 📊 TypeScript-ready with strongly typed responses +- 🔄 Fully asynchronous design - 🔍 Detailed error handling - ⚡ Automatic retries and logging - 🔐 Secure API authentication @@ -23,7 +21,11 @@ Install the package using npm or yarn: ```bash # Using npm -npm install scrapegraph-js +npm i scrapegraph-js + +# Using yarn +yarn add scrapegraph-js +``` ## 🔧 Quick Start @@ -34,6 +36,7 @@ npm install scrapegraph-js ```javascript import { smartScraper } from 'scrapegraph-js'; +import 'dotenv/config'; // Initialize variables const apiKey = process.env.SGAI_APIKEY; // Set your API key as an environment variable @@ -75,26 +78,8 @@ const prompt = 'Extract the main heading and description.'; #### Scraping with Custom Output Schema -```typescript -import { smartScraper } from 'scrapegraph-js'; - -interface WebsiteData { - title: string; - description: string; -} - -const apiKey = 'your-api-key'; -const url = 'https://example.com'; -const prompt = 'Extract the title and description.'; - -(async () => { - try { - const response = await smartScraper(apiKey, url, prompt); - console.log(response.result.title, response.result.description); - } catch (error) { - console.error('Error:', error); - } -})(); +```javascript +//TODO ``` ### Checking API Credits