github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-05-13 20:51:42 -07:00
parent e26008a833
commit 86b8439844
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import async from "async";
import { Progress } from "../../lib/entities";
import { scrapWithScrapingBee } from "./single_url";
import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url";
import robotsParser from "robots-parser";
export class WebCrawler {
@ -196,7 +196,8 @@ export class WebCrawler {
let content;
// If it is the first link, fetch with scrapingbee
if (this.visited.size === 1) {
content = await scrapWithScrapingBee(url, "load");
const page = await scrapSingleUrl(url, {includeHtml: true});
content = page.html;
} else {
const response = await axios.get(url);
content = response.data;

4
apps/api/src/scraper/WebScraper/index.ts
View File

@ -140,6 +140,7 @@ export class WebScraperDataProvider {
generateImgAltText: this.generateImgAltText,
});
let links = await crawler.start(inProgress, 5, this.limit, this.maxCrawledDepth);
if (this.returnOnlyUrls) {
return this.returnOnlyUrlsResponse(links, inProgress);
}
@ -163,6 +164,7 @@ export class WebScraperDataProvider {
return this.returnOnlyUrlsResponse(links, inProgress);
}
let documents = await this.processLinks(links, inProgress);
return this.cacheAndFinalizeDocuments(documents, links);
}
@ -237,6 +239,8 @@ export class WebScraperDataProvider {
links: string[]
): Promise<Document[]> {
await this.setCachedDocuments(documents, links);
documents = this.filterDocsExcludeInclude(documents);
documents = this.filterDepth(documents);
documents = this.removeChildLinks(documents);
return documents.splice(0, this.limit);
}