github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge branch 'main' of https://github.com/mendableai/firecrawl

Browse Source
This commit is contained in:
Nicolas 2024-05-20 10:06:52 -07:00
commit d5d0d48848
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -4,10 +4,11 @@ import requests
import time
class FirecrawlApp:
def __init__(self, api_key=None):
def __init__(self, api_key=None, api_url='https://api.firecrawl.dev'):
self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
if self.api_key is None:
raise ValueError('No API key provided')
self.api_url = api_url or os.getenv('FIRECRAWL_API_URL')
@ -38,7 +39,7 @@ class FirecrawlApp:
scrape_params[key] = value
# Make the POST request with the prepared headers and JSON data
response = requests.post(
'https://api.firecrawl.dev/v0/scrape',
f'{self.api_url}/v0/scrape',
headers=headers,
json=scrape_params
)
@ -63,7 +64,7 @@ class FirecrawlApp:
if params:
json_data.update(params)
response = requests.post(
'https://api.firecrawl.dev/v0/search',
f'{self.api_url}/v0/search',
headers=headers,
json=json_data
)
@ -85,7 +86,7 @@ class FirecrawlApp:
json_data = {'url': url}
if params:
json_data.update(params)
response = self._post_request('https://api.firecrawl.dev/v0/crawl', json_data, headers)
response = self._post_request(f'{self.api_url}/v0/crawl', json_data, headers)
if response.status_code == 200:
job_id = response.json().get('jobId')
if wait_until_done:
@ -97,7 +98,7 @@ class FirecrawlApp:
def check_crawl_status(self, job_id):
headers = self._prepare_headers()
response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers)
response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
if response.status_code == 200:
return response.json()
else:
@ -130,7 +131,7 @@ class FirecrawlApp:
def _monitor_job_status(self, job_id, headers, timeout):
import time
while True:
status_response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers)
status_response = self._get_request(f'{self.api_url}/v0/crawl/status/{job_id}', headers)
if status_response.status_code == 200:
status_data = status_response.json()
if status_data['status'] == 'completed':