github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-05-07 09:26:52 -07:00
parent 2e3ff85509
commit f46bf19fa5
2 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -280,7 +280,6 @@ describe("E2E Tests for API Routes", () => {
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("failed");
expect(completedResponse.body.partial_data?.length ?? 0).toBeLessThanOrEqual(completedResponse.body.data?.length ?? 0);
}, 60000); // 60 seconds

18
apps/api/src/scraper/WebScraper/index.ts
View File

@ -70,13 +70,17 @@ export class WebScraperDataProvider {
results[i + index] = result;
})
);
const job = await getWebScraperQueue().getJob(this.bullJobId);
const jobStatus = await job.getState();
if (jobStatus === "failed") {
throw new Error(
"Job has failed or has been cancelled by the user. Stopping the job..."
);
}
try {
if (this.mode === "crawl" && this.bullJobId) {
const job = await getWebScraperQueue().getJob(this.bullJobId);
const jobStatus = await job.getState();
if (jobStatus === "failed") {
throw new Error(
"Job has failed or has been cancelled by the user. Stopping the job..."
);
}
}
} catch (error) {}
}
return results.filter((result) => result !== null) as Document[];
}