github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-31 15:43:06 -07:00
parent 6bea803120
commit 3b8059edb6
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -130,7 +130,8 @@ export async function scrapWithScrapingBee(
export async function scrapWithPlaywright( export async function scrapWithPlaywright(
url: string, url: string,
waitFor: number = 0 waitFor: number = 0,
headers?: Record<string, string>
): Promise<string> { ): Promise<string> {
try { try {
const reqParams = await generateRequestParams(url); const reqParams = await generateRequestParams(url);
@ -142,7 +143,7 @@ export async function scrapWithPlaywright(
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
}, },
body: JSON.stringify({ url: url, wait: waitParam }), body: JSON.stringify({ url: url, wait: waitParam, headers: headers }),
}); });
if (!response.ok) { if (!response.ok) {
@ -315,7 +316,7 @@ export async function scrapSingleUrl(
break; break;
case "playwright": case "playwright":
if (process.env.PLAYWRIGHT_MICROSERVICE_URL) { if (process.env.PLAYWRIGHT_MICROSERVICE_URL) {
text = await scrapWithPlaywright(url, pageOptions.waitFor); text = await scrapWithPlaywright(url, pageOptions.waitFor, pageOptions.headers);
} }
break; break;
case "scrapingBeeLoad": case "scrapingBeeLoad":