github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #149 from mendableai/nsc/speed-up-crawl-4x

Browse Source 
feat: 4x-5x faster crawler (fast mode)
This commit is contained in:
Nicolas 2024-05-15 11:48:32 -07:00 committed by GitHub
commit ed211dc7f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 163 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -511,6 +511,107 @@ describe("E2E Tests for API Routes", () => {
// }, 120000); // 120 secs
// });
describe("POST /v0/crawl with fast mode", () => {
it("should complete the crawl under 20 seconds", async () => {
const startTime = Date.now();
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://flutterbricks.com",
crawlerOptions: {
mode: "fast"
}
});
expect(crawlResponse.statusCode).toBe(200);
const jobId = crawlResponse.body.jobId;
let statusResponse;
let isFinished = false;
while (!isFinished) {
statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(statusResponse.statusCode).toBe(200);
isFinished = statusResponse.body.status === "completed";
if (!isFinished) {
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const endTime = Date.now();
const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
console.log(`Time elapsed: ${timeElapsed} seconds`);
expect(statusResponse.body.status).toBe("completed");
expect(statusResponse.body).toHaveProperty("data");
expect(statusResponse.body.data[0]).toHaveProperty("content");
expect(statusResponse.body.data[0]).toHaveProperty("markdown");
const results = statusResponse.body.data;
// results.forEach((result, i) => {
// console.log(result.metadata.sourceURL);
// });
expect(results.length).toBeGreaterThanOrEqual(10);
expect(results.length).toBeLessThanOrEqual(15);
}, 20000);
// it("should complete the crawl in more than 10 seconds", async () => {
// const startTime = Date.now();
// const crawlResponse = await request(TEST_URL)
// .post("/v0/crawl")
// .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://flutterbricks.com",
// });
// expect(crawlResponse.statusCode).toBe(200);
// const jobId = crawlResponse.body.jobId;
// let statusResponse;
// let isFinished = false;
// while (!isFinished) {
// statusResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${jobId}`)
// .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
// expect(statusResponse.statusCode).toBe(200);
// isFinished = statusResponse.body.status === "completed";
// if (!isFinished) {
// await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
// }
// }
// const endTime = Date.now();
// const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
// console.log(`Time elapsed: ${timeElapsed} seconds`);
// expect(statusResponse.body.status).toBe("completed");
// expect(statusResponse.body).toHaveProperty("data");
// expect(statusResponse.body.data[0]).toHaveProperty("content");
// expect(statusResponse.body.data[0]).toHaveProperty("markdown");
// const results = statusResponse.body.data;
// // results.forEach((result, i) => {
// // console.log(result.metadata.sourceURL);
// // });
// expect(results.length).toBeGreaterThanOrEqual(10);
// expect(results.length).toBeLessThanOrEqual(15);
// }, 50000);// 15 seconds timeout to account for network delays
});
describe("GET /is-production", () => {
it("should return the production status", async () => {
const response = await request(TEST_URL).get("/is-production");

1
apps/api/src/lib/entities.ts
View File

@ -44,6 +44,7 @@ export type WebScraperOptions = {
limit?: number;
generateImgAltText?: boolean;
replaceAllPathsWithAbsolutePaths?: boolean;
mode?: "default" | "fast"; // have a mode of some sort
};
pageOptions?: PageOptions;
extractorOptions?: ExtractorOptions;

43
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import async from "async";
import { Progress } from "../../lib/entities";
import { scrapWithScrapingBee } from "./single_url";
import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url";
import robotsParser from "robots-parser";
export class WebCrawler {
@ -15,7 +15,7 @@ export class WebCrawler {
private maxCrawledLinks: number;
private maxCrawledDepth: number;
private visited: Set<string> = new Set();
private crawledUrls: Set<string> = new Set();
private crawledUrls: Map<string, string> = new Map();
private limit: number;
private robotsTxtUrl: string;
private robots: any;
@ -51,7 +51,6 @@ export class WebCrawler {
this.generateImgAltText = generateImgAltText ?? false;
}
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks
.filter((link) => {
@ -99,7 +98,7 @@ export class WebCrawler {
concurrencyLimit: number = 5,
limit: number = 10000,
maxDepth: number = 10
): Promise<string[]> {
): Promise<{ url: string, html: string }[]> {
// Fetch and parse robots.txt
try {
const response = await axios.get(this.robotsTxtUrl);
@ -111,7 +110,7 @@ export class WebCrawler {
const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);
if (sitemapLinks.length > 0) {
const filteredLinks = this.filterLinks(sitemapLinks, limit, maxDepth);
return filteredLinks;
return filteredLinks.map(link => ({ url: link, html: "" }));
}
const urls = await this.crawlUrls(
@ -123,18 +122,19 @@ export class WebCrawler {
urls.length === 0 &&
this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0
) {
return [this.initialUrl];
return [{ url: this.initialUrl, html: "" }];
}
// make sure to run include exclude here again
return this.filterLinks(urls, limit, this.maxCrawledDepth);
const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth);
return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" }));
}
private async crawlUrls(
urls: string[],
concurrencyLimit: number,
inProgress?: (progress: Progress) => void
): Promise<string[]> {
): Promise<{ url: string, html: string }[]> {
const queue = async.queue(async (task: string, callback) => {
if (this.crawledUrls.size >= this.maxCrawledLinks) {
if (callback && typeof callback === "function") {
@ -143,13 +143,13 @@ export class WebCrawler {
return;
}
const newUrls = await this.crawl(task);
newUrls.forEach((url) => this.crawledUrls.add(url));
newUrls.forEach((page) => this.crawledUrls.set(page.url, page.html));
if (inProgress && newUrls.length > 0) {
inProgress({
current: this.crawledUrls.size,
total: this.maxCrawledLinks,
status: "SCRAPING",
currentDocumentUrl: newUrls[newUrls.length - 1],
currentDocumentUrl: newUrls[newUrls.length - 1].url,
});
} else if (inProgress) {
inProgress({
@ -159,7 +159,7 @@ export class WebCrawler {
currentDocumentUrl: task,
});
}
await this.crawlUrls(newUrls, concurrencyLimit, inProgress);
await this.crawlUrls(newUrls.map((p) => p.url), concurrencyLimit, inProgress);
if (callback && typeof callback === "function") {
callback();
}
@ -175,10 +175,10 @@ export class WebCrawler {
}
);
await queue.drain();
return Array.from(this.crawledUrls);
return Array.from(this.crawledUrls.entries()).map(([url, html]) => ({ url, html }));
}
async crawl(url: string): Promise<string[]> {
async crawl(url: string): Promise<{url: string, html: string}[]> {
if (this.visited.has(url) || !this.robots.isAllowed(url, "FireCrawlAgent"))
return [];
this.visited.add(url);
@ -193,16 +193,17 @@ export class WebCrawler {
}
try {
let content;
// If it is the first link, fetch with scrapingbee
let content : string = "";
// If it is the first link, fetch with single url
if (this.visited.size === 1) {
content = await scrapWithScrapingBee(url, "load");
const page = await scrapSingleUrl(url, {includeHtml: true});
content = page.html ?? ""
} else {
const response = await axios.get(url);
content = response.data;
content = response.data ?? "";
}
const $ = load(content);
let links: string[] = [];
let links: {url: string, html: string}[] = [];
$("a").each((_, element) => {
const href = $(element).attr("href");
@ -215,7 +216,6 @@ export class WebCrawler {
const path = url.pathname;
if (
// fullUrl.startsWith(this.initialUrl) && // this condition makes it stop crawling back the url
this.isInternalLink(fullUrl) &&
this.matchesPattern(fullUrl) &&
this.noSections(fullUrl) &&
@ -223,12 +223,13 @@ export class WebCrawler {
!this.matchesExcludes(path) &&
this.robots.isAllowed(fullUrl, "FireCrawlAgent")
) {
links.push(fullUrl);
links.push({url: fullUrl, html: content});
}
}
});
return links.filter((link) => !this.visited.has(link));
// Create a new list to return to avoid modifying the visited list
return links.filter((link) => !this.visited.has(link.url));
} catch (error) {
return [];
}

34
apps/api/src/scraper/WebScraper/index.ts
View File

@ -35,6 +35,7 @@ export class WebScraperDataProvider {
private replaceAllPathsWithAbsolutePaths?: boolean = false;
private generateImgAltTextModel: "gpt-4-turbo" | "claude-3-opus" =
"gpt-4-turbo";
private crawlerMode: string = "default";
authorize(): void {
throw new Error("Method not implemented.");
@ -46,7 +47,8 @@ export class WebScraperDataProvider {
private async convertUrlsToDocuments(
urls: string[],
inProgress?: (progress: Progress) => void
inProgress?: (progress: Progress) => void,
allHtmls?: string[]
): Promise<Document[]> {
const totalUrls = urls.length;
let processedUrls = 0;
@ -56,7 +58,8 @@ export class WebScraperDataProvider {
const batchUrls = urls.slice(i, i + this.concurrentRequests);
await Promise.all(
batchUrls.map(async (url, index) => {
const result = await scrapSingleUrl(url, this.pageOptions);
const existingHTML = allHtmls ? allHtmls[i + index] : "";
const result = await scrapSingleUrl(url, this.pageOptions, existingHTML);
processedUrls++;
if (inProgress) {
inProgress({
@ -139,13 +142,26 @@ export class WebScraperDataProvider {
limit: this.limit,
generateImgAltText: this.generateImgAltText,
});
let links = await crawler.start(inProgress, 5, this.limit, this.maxCrawledDepth);
const allLinks = links.map((e) => e.url);
const allHtmls = links.map((e)=> e.html);
if (this.returnOnlyUrls) {
return this.returnOnlyUrlsResponse(links, inProgress);
return this.returnOnlyUrlsResponse(allLinks , inProgress);
}
let documents = await this.processLinks(links, inProgress);
return this.cacheAndFinalizeDocuments(documents, links);
let documents = [];
// check if fast mode is enabled and there is html inside the links
if (this.crawlerMode === "fast" && links.some((link) => link.html)) {
console.log("Fast mode enabled");
documents = await this.processLinks(allLinks, inProgress, allHtmls);
}else{
documents = await this.processLinks(allLinks, inProgress);
}
return this.cacheAndFinalizeDocuments(documents, allLinks);
}
private async handleSingleUrlsMode(
@ -187,14 +203,17 @@ export class WebScraperDataProvider {
private async processLinks(
links: string[],
inProgress?: (progress: Progress) => void
inProgress?: (progress: Progress) => void,
allHtmls?: string[]
): Promise<Document[]> {
let pdfLinks = links.filter((link) => link.endsWith(".pdf"));
let pdfDocuments = await this.fetchPdfDocuments(pdfLinks);
links = links.filter((link) => !link.endsWith(".pdf"));
let documents = await this.convertUrlsToDocuments(links, inProgress);
let documents = await this.convertUrlsToDocuments(links, inProgress, allHtmls);
documents = await this.getSitemapData(this.urls[0], documents);
documents = this.applyPathReplacements(documents);
// documents = await this.applyImgAltText(documents);
@ -397,6 +416,7 @@ export class WebScraperDataProvider {
this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? false;
//! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check
this.excludes = this.excludes.filter((item) => item !== "");
this.crawlerMode = options.crawlerOptions?.mode ?? "default";
// make sure all urls start with https://
this.urls = this.urls.map((url) => {

12
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -106,7 +106,8 @@ export async function scrapWithPlaywright(url: string): Promise<string> {
export async function scrapSingleUrl(
urlToScrap: string,
pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false }
pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false },
existingHtml: string = ""
): Promise<Document> {
urlToScrap = urlToScrap.trim();
@ -197,8 +198,15 @@ export async function scrapSingleUrl(
: ["scrapingBee", "playwright", "scrapingBeeLoad", "fetch"];
for (const scraper of scrapersInOrder) {
// If exists text coming from crawler, use it
if (existingHtml && existingHtml.trim().length >= 100) {
let cleanedHtml = removeUnwantedElements(existingHtml, pageOptions);
text = await parseMarkdown(cleanedHtml);
html = existingHtml;
break;
}
[text, html] = await attemptScraping(urlToScrap, scraper);
if (text && text.length >= 100) break;
if (text && text.trim().length >= 100) break;
console.log(`Falling back to ${scraper}`);
}

2
apps/api/src/services/queue-worker.ts
View File

@ -26,7 +26,7 @@ getWebScraperQueue().process(
success: success,
result: {
links: docs.map((doc) => {
return { content: doc, source: doc.metadata.sourceURL };
return { content: doc, source: doc?.metadata?.sourceURL ?? doc?.url ?? "" };
}),
},
project_id: job.data.project_id,