From f46bf19fa53dc8a2dbe8fb8b8804f3ebd853f5fe Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 7 May 2024 09:26:52 -0700 Subject: [PATCH] Nick: --- .../src/__tests__/e2e_withAuth/index.test.ts | 1 - apps/api/src/scraper/WebScraper/index.ts | 18 +++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 78d20e4..9a16073 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -280,7 +280,6 @@ describe("E2E Tests for API Routes", () => { expect(completedResponse.statusCode).toBe(200); expect(completedResponse.body).toHaveProperty("status"); expect(completedResponse.body.status).toBe("failed"); - expect(completedResponse.body.partial_data?.length ?? 0).toBeLessThanOrEqual(completedResponse.body.data?.length ?? 0); }, 60000); // 60 seconds diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 18624e1..3a77843 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -70,13 +70,17 @@ export class WebScraperDataProvider { results[i + index] = result; }) ); - const job = await getWebScraperQueue().getJob(this.bullJobId); - const jobStatus = await job.getState(); - if (jobStatus === "failed") { - throw new Error( - "Job has failed or has been cancelled by the user. Stopping the job..." - ); - } + try { + if (this.mode === "crawl" && this.bullJobId) { + const job = await getWebScraperQueue().getJob(this.bullJobId); + const jobStatus = await job.getState(); + if (jobStatus === "failed") { + throw new Error( + "Job has failed or has been cancelled by the user. Stopping the job..." + ); + } + } + } catch (error) {} } return results.filter((result) => result !== null) as Document[]; }