github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update single_url.ts

Browse Source
This commit is contained in:
Nicolas 2024-04-23 17:04:10 -07:00
parent 4328a68ec1
commit f0695c7123
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -23,13 +23,14 @@ export async function scrapWithCustomFirecrawl(
export async function scrapWithScrapingBee(
url: string,
wait_browser: string = "domcontentloaded"
wait_browser: string = "domcontentloaded",
timeout: number = 15000
): Promise<string> {
try {
const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY);
const response = await client.get({
url: url,
params: { timeout: 15000, wait_browser: wait_browser },
params: { timeout: timeout, wait_browser: wait_browser },
headers: { "ScrapingService-Request": "TRUE" },
});
@ -106,11 +107,11 @@ export async function scrapSingleUrl(
let text = "";
switch (method) {
case "firecrawl-scraper":
text = await scrapWithCustomFirecrawl(url);
text = await scrapWithCustomFirecrawl(url,);
break;
case "scrapingBee":
if (process.env.SCRAPING_BEE_API_KEY) {
text = await scrapWithScrapingBee(url);
text = await scrapWithScrapingBee(url,"domcontentloaded", pageOptions.fallback === false? 7000 : 15000);
}
break;
case "playwright":