diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 75a9d5c..fee126a 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -78,7 +78,7 @@ export async function scrapWithScrapingBee( export async function scrapWithPlaywright(url: string): Promise { try { const reqParams = await generateRequestParams(url); - const wait_playwright = reqParams["params"]["wait"]; + const wait_playwright = reqParams["params"]?.wait ?? 0; const response = await fetch(process.env.PLAYWRIGHT_MICROSERVICE_URL, { method: "POST", diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 069f433..17036f4 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -44,7 +44,7 @@ export const urlSpecificParams = { params: { wait_browser: "networkidle2", block_resources: false, - wait: 5000, + wait: 3000, }, headers: { "User-Agent": @@ -64,7 +64,7 @@ export const urlSpecificParams = { params: { wait_browser: "networkidle2", block_resources: false, - wait: 5000, + wait: 3000, }, headers: { "User-Agent":