Merge pull request #212 from mendableai/bugfix/partial-data-js-sdk

[Bug] Improved js response and test for getting partial_data

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",

apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts

@@ -99,8 +99,17 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
-    await new Promise(resolve => setTimeout(resolve, 30000)); // wait for 30 seconds
+    let statusResponse = await app.checkCrawlStatus(response.jobId);
-    const statusResponse = await app.checkCrawlStatus(response.jobId);
+    const maxChecks = 15;
+    let checks = 0;
+
+    while (statusResponse.status === 'active' && checks < maxChecks) {
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      expect(statusResponse.partial_data).not.toBeNull();
+      statusResponse = await app.checkCrawlStatus(response.jobId);
+      checks++;
+    }
+
     expect(statusResponse).not.toBeNull();
     expect(statusResponse.status).toBe('completed');
     expect(statusResponse.data.length).toBeGreaterThan(0);

apps/js-sdk/firecrawl/src/index.ts

@@ -56,6 +56,7 @@ export interface JobStatusResponse {
   status: string;
   jobId?: string;
   data?: any;
+  partial_data?: any,
   error?: string;
 }
 
@@ -226,7 +227,12 @@ export default class FirecrawlApp {
         headers
       );
       if (response.status === 200) {
-        return response.data;
+        return {
+          success: true,
+          status: response.data.status,
+          data: response.data.data,
+          partial_data: !response.data.data ? response.data.partial_data : undefined,
+        };
       } else {
         this.handleError(response, "check crawl status");
       }