github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-05-14 12:04:36 -07:00
parent 8a72cf556b
commit 7f31959be7
2 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -15,7 +15,7 @@ export class WebCrawler {
private maxCrawledLinks: number;
private maxCrawledDepth: number;
private visited: Set<string> = new Set();
private crawledUrls: { url: string, html: string }[] = [];
private crawledUrls: Set<{ url: string, html: string }> = new Set();
private limit: number;
private robotsTxtUrl: string;
private robots: any;
@ -136,24 +136,24 @@ export class WebCrawler {
inProgress?: (progress: Progress) => void
): Promise<{ url: string, html: string }[]> {
const queue = async.queue(async (task: string, callback) => {
if (this.crawledUrls.length >= this.maxCrawledLinks) {
if (this.crawledUrls.size >= this.maxCrawledLinks) {
if (callback && typeof callback === "function") {
callback();
}
return;
}
const newUrls = await this.crawl(task);
newUrls.forEach((page) => this.crawledUrls.push(page));
newUrls.forEach((page) => this.crawledUrls.add(page));
if (inProgress && newUrls.length > 0) {
inProgress({
current: this.crawledUrls.length,
current: this.crawledUrls.size,
total: this.maxCrawledLinks,
status: "SCRAPING",
currentDocumentUrl: newUrls[newUrls.length - 1].url,
});
} else if (inProgress) {
inProgress({
current: this.crawledUrls.length,
current: this.crawledUrls.size,
total: this.maxCrawledLinks,
status: "SCRAPING",
currentDocumentUrl: task,
@ -175,7 +175,7 @@ export class WebCrawler {
}
);
await queue.drain();
return this.crawledUrls;
return Array.from(this.crawledUrls);
}
async crawl(url: string): Promise<{url: string, html: string}[]> {
@ -311,3 +311,4 @@ export class WebCrawler {
}
}

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -277,8 +277,6 @@ export class WebScraperDataProvider {
): Promise<Document[]> {
await this.setCachedDocuments(documents, links);
documents = this.removeChildLinks(documents);
documents = this.filterDocsExcludeInclude(documents);
documents = this.filterDepth(documents);
return documents.splice(0, this.limit);
}