github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Final push

Browse Source
This commit is contained in:
Eric Ciarla 2024-05-29 21:18:24 -04:00
parent a0e404f94e
commit 37915e11e8
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -53,7 +53,7 @@ export async function scrapWithFireEngine(
// If the user has passed a wait parameter in the request, use that // If the user has passed a wait parameter in the request, use that
const waitParam = reqParams["params"]?.wait ?? waitFor; const waitParam = reqParams["params"]?.wait ?? waitFor;
const screenshotParam = reqParams["params"]?.screenshot ?? screenshot; const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
console.log(`[Fire-Engine] Scraping ${url} with wait: ${waitParam}`); console.log(`[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}`);
const response = await fetch(process.env.FIRE_ENGINE_BETA_URL+ "/scrape", { const response = await fetch(process.env.FIRE_ENGINE_BETA_URL+ "/scrape", {
method: "POST", method: "POST",
@ -330,7 +330,7 @@ export async function scrapSingleUrl(
content: text, content: text,
markdown: text, markdown: text,
html: pageOptions.includeHtml ? html : undefined, html: pageOptions.includeHtml ? html : undefined,
metadata: { ...metadata, screenshot_base64: screenshot, sourceURL: urlToScrap, }, metadata: { ...metadata, screenshot: screenshot, sourceURL: urlToScrap, },
} }
}else{ }else{
document = { document = {