github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

added idempotency key to python sdk

Browse Source
This commit is contained in:
rafaelsideguide 2024-05-23 12:52:59 -03:00
parent 35927a65a5
commit c201ea1986
4 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/build/index.js
View File

@ -19,7 +19,7 @@ export default class FirecrawlApp {
* @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
*/ */
constructor({ apiKey = null }) { constructor({ apiKey = null }) {
this.apiKey = apiKey || ''; this.apiKey = apiKey || "";
if (!this.apiKey) { if (!this.apiKey) {
throw new Error("No API key provided"); throw new Error("No API key provided");
} }

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.21", "version": "0.0.22",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/index.js", "main": "build/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",

6
apps/python-sdk/example.py
View File

@ -1,4 +1,5 @@
from firecrawl import FirecrawlApp import uuid
from firecrawl.firecrawl import FirecrawlApp
app = FirecrawlApp(api_key="fc-YOUR_API_KEY") app = FirecrawlApp(api_key="fc-YOUR_API_KEY")
@ -7,7 +8,8 @@ scrape_result = app.scrape_url('firecrawl.dev')
print(scrape_result['markdown']) print(scrape_result['markdown'])
# Crawl a website: # Crawl a website:
crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}}) idempotency_key = str(uuid.uuid4()) # optional idempotency key
crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}}, True, 2, idempotency_key)
print(crawl_result) print(crawl_result)
# LLM Extraction: # LLM Extraction:

14
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -81,8 +81,8 @@ class FirecrawlApp:
else: else:
raise Exception(f'Failed to search. Status code: {response.status_code}') raise Exception(f'Failed to search. Status code: {response.status_code}')
def crawl_url(self, url, params=None, wait_until_done=True, timeout=2): def crawl_url(self, url, params=None, wait_until_done=True, timeout=2, idempotency_key=None):
headers = self._prepare_headers() headers = self._prepare_headers(idempotency_key)
json_data = {'url': url} json_data = {'url': url}
if params: if params:
json_data.update(params) json_data.update(params)
@ -104,10 +104,16 @@ class FirecrawlApp:
else: else:
self._handle_error(response, 'check crawl status') self._handle_error(response, 'check crawl status')
def _prepare_headers(self): def _prepare_headers(self, idempotency_key=None):
if idempotency_key:
return {
'Content-Type': 'application/json',
'Authorization': f'Bearer {self.api_key}',
'x-idempotency-key': idempotency_key
}
return { return {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Authorization': f'Bearer {self.api_key}' 'Authorization': f'Bearer {self.api_key}',
} }
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5): def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5):