github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-04-23 17:04:10 -07:00
parent 4328a68ec1
commit f0695c7123
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -23,13 +23,14 @@ export async function scrapWithCustomFirecrawl(
export async function scrapWithScrapingBee( export async function scrapWithScrapingBee(
url: string, url: string,
wait_browser: string = "domcontentloaded" wait_browser: string = "domcontentloaded",
timeout: number = 15000
): Promise<string> { ): Promise<string> {
try { try {
const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY); const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
const response = await client.get({ const response = await client.get({
url: url, url: url,
params: { timeout: 15000, wait_browser: wait_browser }, params: { timeout: timeout, wait_browser: wait_browser },
headers: { "ScrapingService-Request": "TRUE" }, headers: { "ScrapingService-Request": "TRUE" },
}); });
@ -106,11 +107,11 @@ export async function scrapSingleUrl(
let text = ""; let text = "";
switch (method) { switch (method) {
case "firecrawl-scraper": case "firecrawl-scraper":
text = await scrapWithCustomFirecrawl(url); text = await scrapWithCustomFirecrawl(url,);
break; break;
case "scrapingBee": case "scrapingBee":
if (process.env.SCRAPING_BEE_API_KEY) { if (process.env.SCRAPING_BEE_API_KEY) {
text = await scrapWithScrapingBee(url); text = await scrapWithScrapingBee(url,"domcontentloaded", pageOptions.fallback === false? 7000 : 15000);
} }
break; break;
case "playwright": case "playwright":