From 99059814a861ab3abdcf9a6de9e3c713d92a03d3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 3 Jun 2024 21:32:48 -0700 Subject: [PATCH] Nick: --- apps/api/src/__tests__/e2e_withAuth/index.test.ts | 2 +- .../src/scraper/WebScraper/utils/custom/website_params.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index e81d18e..2042abf 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -66,7 +66,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/scrape") .set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://roastmywebsite.ai" }); expect(response.statusCode).toBe(200); }, 30000); // 30 seconds timeout diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 01f6614..3828e47 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -145,8 +145,6 @@ export const urlSpecificParams = { "firecrawl.dev":{ defaultScraper: "fire-engine", params: { - - }, headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", @@ -159,5 +157,6 @@ export const urlSpecificParams = { accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", }, - } + }, + }, };