github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-06-03 21:32:48 -07:00
parent 93bb53271e
commit 99059814a8
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -66,7 +66,7 @@ describe("E2E Tests for API Routes", () => {
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Authorization", `Bearer this_is_just_a_preview_token`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" }); .send({ url: "https://roastmywebsite.ai" });
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
}, 30000); // 30 seconds timeout }, 30000); // 30 seconds timeout

5
apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
View File

@ -145,8 +145,6 @@ export const urlSpecificParams = {
"firecrawl.dev":{ "firecrawl.dev":{
defaultScraper: "fire-engine", defaultScraper: "fire-engine",
params: { params: {
},
headers: { headers: {
"User-Agent": "User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@ -159,5 +157,6 @@ export const urlSpecificParams = {
accept: accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
}, },
} },
},
}; };