github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #212 from mendableai/bugfix/partial-data-js-sdk

Browse Source 
[Bug] Improved js response and test for getting partial_data
This commit is contained in:
Rafael Miller 2024-06-04 14:05:23 -03:00 committed by GitHub
commit 665a40d9f4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.22",
"version": "0.0.23",
"description": "JavaScript SDK for Firecrawl API",
"main": "build/index.js",
"types": "types/index.d.ts",

13
apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -99,8 +99,17 @@ describe('FirecrawlApp E2E Tests', () => {
expect(response).not.toBeNull();
expect(response.jobId).toBeDefined();
await new Promise(resolve => setTimeout(resolve, 30000)); // wait for 30 seconds
const statusResponse = await app.checkCrawlStatus(response.jobId);
let statusResponse = await app.checkCrawlStatus(response.jobId);
const maxChecks = 15;
let checks = 0;
while (statusResponse.status === 'active' && checks < maxChecks) {
await new Promise(resolve => setTimeout(resolve, 1000));
expect(statusResponse.partial_data).not.toBeNull();
statusResponse = await app.checkCrawlStatus(response.jobId);
checks++;
}
expect(statusResponse).not.toBeNull();
expect(statusResponse.status).toBe('completed');
expect(statusResponse.data.length).toBeGreaterThan(0);

8
apps/js-sdk/firecrawl/src/index.ts
View File

@ -56,6 +56,7 @@ export interface JobStatusResponse {
status: string;
jobId?: string;
data?: any;
partial_data?: any,
error?: string;
}
@ -226,7 +227,12 @@ export default class FirecrawlApp {
headers
);
if (response.status === 200) {
return response.data;
return {
success: true,
status: response.data.status,
data: response.data.data,
partial_data: !response.data.data ? response.data.partial_data : undefined,
};
} else {
this.handleError(response, "check crawl status");
}