github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update handleCustomScraping.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-04 12:22:46 -07:00
parent 674500affa
commit 96257b7b17
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/scraper/WebScraper/custom/handleCustomScraping.ts
View File

@ -14,6 +14,7 @@ export async function handleCustomScraping(
}; };
} }
// Check for Vanta security portals
if (text.includes('<link href="https://static.vanta.com')) { if (text.includes('<link href="https://static.vanta.com')) {
console.log( console.log(
`Vanta link detected for ${url}, using Fire Engine with wait time 3000ms` `Vanta link detected for ${url}, using Fire Engine with wait time 3000ms`