Merge pull request #218 from mendableai/nsc/python-sdk-waiting-fixes

[sdk] Fixes waiting status not being present on check status

apps/api/src/controllers/auth.ts

@@ -132,7 +132,7 @@ export async function supaAuthenticateUser(
     const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);
     return {
       success: false,
-      error: `Rate limit exceeded. Consumed points: ${rateLimiterRes.consumedPoints}, Remaining points: ${rateLimiterRes.remainingPoints}. Please retry after ${secs}s, resets at ${retryDate}`,
+      error: `Rate limit exceeded. Consumed points: ${rateLimiterRes.consumedPoints}, Remaining points: ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,
       status: 429,
     };
   }

apps/python-sdk/build/lib/firecrawl/firecrawl.py

@@ -273,7 +273,7 @@ class FirecrawlApp:
                         return status_data['data']
                     else:
                         raise Exception('Crawl job completed but no data was returned')
-                elif status_data['status'] in ['active', 'paused', 'pending', 'queued']:
+                elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting']:
                     timeout=max(timeout,2)
                     time.sleep(timeout)  # Wait for the specified timeout before checking again
                 else:

apps/python-sdk/firecrawl/firecrawl.py

@@ -273,7 +273,7 @@ class FirecrawlApp:
                         return status_data['data']
                     else:
                         raise Exception('Crawl job completed but no data was returned')
-                elif status_data['status'] in ['active', 'paused', 'pending', 'queued']:
+                elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting']:
                     timeout=max(timeout,2)
                     time.sleep(timeout)  # Wait for the specified timeout before checking again
                 else:

apps/python-sdk/firecrawl_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: firecrawl-py
-Version: 0.0.11
+Version: 0.0.12
 Summary: Python SDK for Firecrawl API
 Home-page: https://github.com/mendableai/firecrawl
 Author: Mendable.ai
@@ -151,6 +151,25 @@ status = app.check_crawl_status(job_id)
 
 The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
 
+## Running the Tests with Pytest
+
+To ensure the functionality of the Firecrawl Python SDK, we have included end-to-end tests using `pytest`. These tests cover various aspects of the SDK, including URL scraping, web searching, and website crawling.
+
+### Running the Tests
+
+To run the tests, execute the following commands:
+
+Install pytest:
+```bash
+pip install pytest
+```
+
+Run:
+```bash
+pytest firecrawl/__tests__/e2e_withAuth/test.py
+```
+
+
 ## Contributing
 
 Contributions to the Firecrawl Python SDK are welcome! If you find any issues or have suggestions for improvements, please open an issue or submit a pull request on the GitHub repository.

apps/python-sdk/firecrawl_py.egg-info/requires.txt

@@ -1 +1,3 @@
 requests
+pytest
+python-dotenv

apps/python-sdk/setup.py

@@ -7,7 +7,7 @@ long_description_content = (this_directory / "README.md").read_text()
 
 setup(
     name="firecrawl-py",
-    version="0.0.11",
+    version="0.0.12",
     url="https://github.com/mendableai/firecrawl",
     author="Mendable.ai",
     author_email="nick@mendable.ai",