github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update single_url.ts

Browse Source
This commit is contained in:
rafaelsideguide 2024-06-04 15:28:09 -03:00
parent 93f3098672
commit 5ae4d1caf5
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -257,12 +257,22 @@ async function handleCustomScraping(
text: string, text: string,
url: string url: string
): Promise<FireEngineResponse | null> { ): Promise<FireEngineResponse | null> {
// Check for Readme Docs special case
if (text.includes('<meta name="readme-deploy"')) { if (text.includes('<meta name="readme-deploy"')) {
console.log( console.log(
`Special use case detected for ${url}, using Fire Engine with wait time 1000ms` `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
); );
return await scrapWithFireEngine(url, 1000); return await scrapWithFireEngine(url, 1000);
} }
// Check for Google Drive PDF links in the raw HTML
const googleDrivePdfPattern = /https:\/\/drive\.google\.com\/file\/d\/[^\/]+\/view/;
const googleDrivePdfLink = text.match(googleDrivePdfPattern);
if (googleDrivePdfLink) {
console.log(`Google Drive PDF link detected for ${url}: ${googleDrivePdfLink[0]}`);
return await scrapWithFireEngine(url, 1000);
}
return null; return null;
} }