github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: better

Browse Source
This commit is contained in:
Nicolas 2024-05-09 17:48:11 -07:00
parent 73e8da9185
commit be85008622
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -78,7 +78,7 @@ export async function scrapWithScrapingBee(
export async function scrapWithPlaywright(url: string): Promise<string> { export async function scrapWithPlaywright(url: string): Promise<string> {
try { try {
const reqParams = await generateRequestParams(url); const reqParams = await generateRequestParams(url);
const wait_playwright = reqParams["params"]["wait"]; const wait_playwright = reqParams["params"]?.wait ?? 0;
const response = await fetch(process.env.PLAYWRIGHT_MICROSERVICE_URL, { const response = await fetch(process.env.PLAYWRIGHT_MICROSERVICE_URL, {
method: "POST", method: "POST",

4
apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
View File

@ -44,7 +44,7 @@ export const urlSpecificParams = {
params: { params: {
wait_browser: "networkidle2", wait_browser: "networkidle2",
block_resources: false, block_resources: false,
wait: 5000, wait: 3000,
}, },
headers: { headers: {
"User-Agent": "User-Agent":
@ -64,7 +64,7 @@ export const urlSpecificParams = {
params: { params: {
wait_browser: "networkidle2", wait_browser: "networkidle2",
block_resources: false, block_resources: false,
wait: 5000, wait: 3000,
}, },
headers: { headers: {
"User-Agent": "User-Agent":