From 3b8059edb64ba4ef2362e72a6c740ad64ed5b6e1 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 31 May 2024 15:43:06 -0700 Subject: [PATCH] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 3a04004..b73d121 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -130,7 +130,8 @@ export async function scrapWithScrapingBee( export async function scrapWithPlaywright( url: string, - waitFor: number = 0 + waitFor: number = 0, + headers?: Record ): Promise { try { const reqParams = await generateRequestParams(url); @@ -142,7 +143,7 @@ export async function scrapWithPlaywright( headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ url: url, wait: waitParam }), + body: JSON.stringify({ url: url, wait: waitParam, headers: headers }), }); if (!response.ok) { @@ -315,7 +316,7 @@ export async function scrapSingleUrl( break; case "playwright": if (process.env.PLAYWRIGHT_MICROSERVICE_URL) { - text = await scrapWithPlaywright(url, pageOptions.waitFor); + text = await scrapWithPlaywright(url, pageOptions.waitFor, pageOptions.headers); } break; case "scrapingBeeLoad":