From be85008622b88abc3c48d474aad15aee8f17894d Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 9 May 2024 17:48:11 -0700 Subject: [PATCH] Nick: better --- apps/api/src/scraper/WebScraper/single_url.ts | 2 +- .../api/src/scraper/WebScraper/utils/custom/website_params.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 75a9d5c..fee126a 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -78,7 +78,7 @@ export async function scrapWithScrapingBee( export async function scrapWithPlaywright(url: string): Promise { try { const reqParams = await generateRequestParams(url); - const wait_playwright = reqParams["params"]["wait"]; + const wait_playwright = reqParams["params"]?.wait ?? 0; const response = await fetch(process.env.PLAYWRIGHT_MICROSERVICE_URL, { method: "POST", diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 069f433..17036f4 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -44,7 +44,7 @@ export const urlSpecificParams = { params: { wait_browser: "networkidle2", block_resources: false, - wait: 5000, + wait: 3000, }, headers: { "User-Agent": @@ -64,7 +64,7 @@ export const urlSpecificParams = { params: { wait_browser: "networkidle2", block_resources: false, - wait: 5000, + wait: 3000, }, headers: { "User-Agent":