Merge pull request #149 from mendableai/nsc/speed-up-crawl-4x

feat: 4x-5x faster crawler (fast mode)

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -511,6 +511,107 @@ describe("E2E Tests for API Routes", () => {
   //   }, 120000); // 120 secs
   // });
 
+  describe("POST /v0/crawl with fast mode", () => {
+    it("should complete the crawl under 20 seconds", async () => {
+      const startTime = Date.now();
+
+      const crawlResponse = await request(TEST_URL)
+        .post("/v0/crawl")
+        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+        .set("Content-Type", "application/json")
+        .send({
+          url: "https://flutterbricks.com",
+          crawlerOptions: {
+            mode: "fast"
+          }
+        });
+
+      expect(crawlResponse.statusCode).toBe(200);
+
+      const jobId = crawlResponse.body.jobId;
+      let statusResponse;
+      let isFinished = false;
+
+      while (!isFinished) {
+        statusResponse = await request(TEST_URL)
+          .get(`/v0/crawl/status/${jobId}`)
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+
+        expect(statusResponse.statusCode).toBe(200);
+        isFinished = statusResponse.body.status === "completed";
+
+        if (!isFinished) {
+          await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
+        }
+      }
+
+      const endTime = Date.now();
+      const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
+
+      console.log(`Time elapsed: ${timeElapsed} seconds`);
+
+      expect(statusResponse.body.status).toBe("completed");
+      expect(statusResponse.body).toHaveProperty("data");
+      expect(statusResponse.body.data[0]).toHaveProperty("content");
+      expect(statusResponse.body.data[0]).toHaveProperty("markdown");
+      const results = statusResponse.body.data;
+      // results.forEach((result, i) => {
+      //   console.log(result.metadata.sourceURL);
+      // });
+      expect(results.length).toBeGreaterThanOrEqual(10);
+      expect(results.length).toBeLessThanOrEqual(15);
+      
+    }, 20000);
+
+    // it("should complete the crawl in more than 10 seconds", async () => {
+    //   const startTime = Date.now();
+
+    //   const crawlResponse = await request(TEST_URL)
+    //     .post("/v0/crawl")
+    //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+    //     .set("Content-Type", "application/json")
+    //     .send({
+    //       url: "https://flutterbricks.com",
+    //     });
+
+    //   expect(crawlResponse.statusCode).toBe(200);
+
+    //   const jobId = crawlResponse.body.jobId;
+    //   let statusResponse;
+    //   let isFinished = false;
+
+    //   while (!isFinished) {
+    //     statusResponse = await request(TEST_URL)
+    //       .get(`/v0/crawl/status/${jobId}`)
+    //       .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+
+    //     expect(statusResponse.statusCode).toBe(200);
+    //     isFinished = statusResponse.body.status === "completed";
+
+    //     if (!isFinished) {
+    //       await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
+    //     }
+    //   }
+
+    //   const endTime = Date.now();
+    //   const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
+
+    //   console.log(`Time elapsed: ${timeElapsed} seconds`);
+
+    //   expect(statusResponse.body.status).toBe("completed");
+    //   expect(statusResponse.body).toHaveProperty("data");
+    //   expect(statusResponse.body.data[0]).toHaveProperty("content");
+    //   expect(statusResponse.body.data[0]).toHaveProperty("markdown");
+    //   const results = statusResponse.body.data;
+    //   // results.forEach((result, i) => {
+    //   //   console.log(result.metadata.sourceURL);
+    //   // });
+    //   expect(results.length).toBeGreaterThanOrEqual(10);
+    //   expect(results.length).toBeLessThanOrEqual(15);
+      
+    // }, 50000);// 15 seconds timeout to account for network delays
+  });
+
   describe("GET /is-production", () => {
     it("should return the production status", async () => {
       const response = await request(TEST_URL).get("/is-production");

apps/api/src/lib/entities.ts

@@ -44,6 +44,7 @@ export type WebScraperOptions = {
     limit?: number;
     generateImgAltText?: boolean;
     replaceAllPathsWithAbsolutePaths?: boolean;
+    mode?: "default" | "fast"; // have a mode of some sort
   };
   pageOptions?: PageOptions;
   extractorOptions?: ExtractorOptions;

apps/api/src/scraper/WebScraper/crawler.ts

@@ -4,7 +4,7 @@ import { URL } from "url";
 import { getLinksFromSitemap } from "./sitemap";
 import async from "async";
 import { Progress } from "../../lib/entities";
-import { scrapWithScrapingBee } from "./single_url";
+import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url";
 import robotsParser from "robots-parser";
 
 export class WebCrawler {
@@ -15,7 +15,7 @@ export class WebCrawler {
   private maxCrawledLinks: number;
   private maxCrawledDepth: number;
   private visited: Set<string> = new Set();
-  private crawledUrls: Set<string> = new Set();
+  private crawledUrls: Map<string, string> = new Map();
   private limit: number;
   private robotsTxtUrl: string;
   private robots: any;
@@ -51,7 +51,6 @@ export class WebCrawler {
     this.generateImgAltText = generateImgAltText ?? false;
   }
 
-
   private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
     return sitemapLinks
       .filter((link) => {
@@ -99,7 +98,7 @@ export class WebCrawler {
     concurrencyLimit: number = 5,
     limit: number = 10000,
     maxDepth: number = 10
-  ): Promise<string[]> {
+  ): Promise<{ url: string, html: string }[]> {
     // Fetch and parse robots.txt
     try {
       const response = await axios.get(this.robotsTxtUrl);
@@ -111,7 +110,7 @@ export class WebCrawler {
     const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);
     if (sitemapLinks.length > 0) {
       const filteredLinks = this.filterLinks(sitemapLinks, limit, maxDepth);
-      return filteredLinks;
+      return filteredLinks.map(link => ({ url: link, html: "" }));
     }
 
     const urls = await this.crawlUrls(
@@ -123,18 +122,19 @@ export class WebCrawler {
       urls.length === 0 &&
       this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0
     ) {
-      return [this.initialUrl];
+      return [{ url: this.initialUrl, html: "" }];
     }
 
     // make sure to run include exclude here again
-    return this.filterLinks(urls, limit, this.maxCrawledDepth);
+    const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth);
+    return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" }));
   }
 
   private async crawlUrls(
     urls: string[],
     concurrencyLimit: number,
     inProgress?: (progress: Progress) => void
-  ): Promise<string[]> {
+  ): Promise<{ url: string, html: string }[]> {
     const queue = async.queue(async (task: string, callback) => {
       if (this.crawledUrls.size >= this.maxCrawledLinks) {
         if (callback && typeof callback === "function") {
@@ -143,13 +143,13 @@ export class WebCrawler {
         return;
       }
       const newUrls = await this.crawl(task);
-      newUrls.forEach((url) => this.crawledUrls.add(url));
+      newUrls.forEach((page) => this.crawledUrls.set(page.url, page.html));
       if (inProgress && newUrls.length > 0) {
         inProgress({
           current: this.crawledUrls.size,
           total: this.maxCrawledLinks,
           status: "SCRAPING",
-          currentDocumentUrl: newUrls[newUrls.length - 1],
+          currentDocumentUrl: newUrls[newUrls.length - 1].url,
         });
       } else if (inProgress) {
         inProgress({
@@ -159,7 +159,7 @@ export class WebCrawler {
           currentDocumentUrl: task,
         });
       }
-      await this.crawlUrls(newUrls, concurrencyLimit, inProgress);
+      await this.crawlUrls(newUrls.map((p) => p.url), concurrencyLimit, inProgress);
       if (callback && typeof callback === "function") {
         callback();
       }
@@ -175,10 +175,10 @@ export class WebCrawler {
       }
     );
     await queue.drain();
-    return Array.from(this.crawledUrls);
+    return Array.from(this.crawledUrls.entries()).map(([url, html]) => ({ url, html }));
   }
 
-  async crawl(url: string): Promise<string[]> {
+  async crawl(url: string): Promise<{url: string, html: string}[]> {
     if (this.visited.has(url) || !this.robots.isAllowed(url, "FireCrawlAgent"))
       return [];
     this.visited.add(url);
@@ -193,16 +193,17 @@ export class WebCrawler {
     }
 
     try {
-      let content;
+      let content : string = "";
-      // If it is the first link, fetch with scrapingbee
+      // If it is the first link, fetch with single url
       if (this.visited.size === 1) {
-        content = await scrapWithScrapingBee(url, "load");
+        const page = await scrapSingleUrl(url, {includeHtml: true});
+        content = page.html ?? ""
       } else {
         const response = await axios.get(url);
-        content = response.data;
+        content = response.data ?? "";
       }
       const $ = load(content);
-      let links: string[] = [];
+      let links: {url: string, html: string}[] = [];
 
       $("a").each((_, element) => {
         const href = $(element).attr("href");
@@ -215,7 +216,6 @@ export class WebCrawler {
           const path = url.pathname;
 
           if (
-            // fullUrl.startsWith(this.initialUrl) && // this condition makes it stop crawling back the url
             this.isInternalLink(fullUrl) &&
             this.matchesPattern(fullUrl) &&
             this.noSections(fullUrl) &&
@@ -223,12 +223,13 @@ export class WebCrawler {
             !this.matchesExcludes(path) &&
             this.robots.isAllowed(fullUrl, "FireCrawlAgent")
           ) {
-            links.push(fullUrl);
+            links.push({url: fullUrl, html: content});
           }
         }
       });
 
-      return links.filter((link) => !this.visited.has(link));
+      // Create a new list to return to avoid modifying the visited list
+      return links.filter((link) => !this.visited.has(link.url));
     } catch (error) {
       return [];
     }

apps/api/src/scraper/WebScraper/index.ts

@@ -35,6 +35,7 @@ export class WebScraperDataProvider {
   private replaceAllPathsWithAbsolutePaths?: boolean = false;
   private generateImgAltTextModel: "gpt-4-turbo" | "claude-3-opus" =
     "gpt-4-turbo";
+  private crawlerMode: string = "default";
 
   authorize(): void {
     throw new Error("Method not implemented.");
@@ -46,7 +47,8 @@ export class WebScraperDataProvider {
 
   private async convertUrlsToDocuments(
     urls: string[],
-    inProgress?: (progress: Progress) => void
+    inProgress?: (progress: Progress) => void,
+    allHtmls?: string[]
   ): Promise<Document[]> {
     const totalUrls = urls.length;
     let processedUrls = 0;
@@ -56,7 +58,8 @@ export class WebScraperDataProvider {
       const batchUrls = urls.slice(i, i + this.concurrentRequests);
       await Promise.all(
         batchUrls.map(async (url, index) => {
-          const result = await scrapSingleUrl(url, this.pageOptions);
+          const existingHTML = allHtmls ? allHtmls[i + index] : "";
+          const result = await scrapSingleUrl(url, this.pageOptions, existingHTML);
           processedUrls++;
           if (inProgress) {
             inProgress({
@@ -139,13 +142,26 @@ export class WebScraperDataProvider {
       limit: this.limit,
       generateImgAltText: this.generateImgAltText,
     });
+
     let links = await crawler.start(inProgress, 5, this.limit, this.maxCrawledDepth);
+
+    const allLinks = links.map((e) => e.url);
+    const allHtmls = links.map((e)=> e.html);
+
     if (this.returnOnlyUrls) {
-      return this.returnOnlyUrlsResponse(links, inProgress);
+      return this.returnOnlyUrlsResponse(allLinks , inProgress);
+    }
+    
+    let documents = [];
+    // check if fast mode is enabled and there is html inside the links
+    if (this.crawlerMode === "fast" && links.some((link) => link.html)) {
+      console.log("Fast mode enabled");
+      documents = await this.processLinks(allLinks, inProgress, allHtmls);
+    }else{
+      documents = await this.processLinks(allLinks, inProgress);
     }
 
-    let documents = await this.processLinks(links, inProgress);
+    return this.cacheAndFinalizeDocuments(documents, allLinks);
-    return this.cacheAndFinalizeDocuments(documents, links);
   }
 
   private async handleSingleUrlsMode(
@@ -187,14 +203,17 @@ export class WebScraperDataProvider {
 
   private async processLinks(
     links: string[],
-    inProgress?: (progress: Progress) => void
+    inProgress?: (progress: Progress) => void,
+    allHtmls?: string[]
   ): Promise<Document[]> {
     let pdfLinks = links.filter((link) => link.endsWith(".pdf"));
     let pdfDocuments = await this.fetchPdfDocuments(pdfLinks);
     links = links.filter((link) => !link.endsWith(".pdf"));
-
+    
-    let documents = await this.convertUrlsToDocuments(links, inProgress);
+    let documents = await this.convertUrlsToDocuments(links, inProgress, allHtmls);
     documents = await this.getSitemapData(this.urls[0], documents);
+
+
     documents = this.applyPathReplacements(documents);
     // documents = await this.applyImgAltText(documents);
 
@@ -397,6 +416,7 @@ export class WebScraperDataProvider {
     this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? false;
     //! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check
     this.excludes = this.excludes.filter((item) => item !== "");
+    this.crawlerMode = options.crawlerOptions?.mode ?? "default";
 
     // make sure all urls start with https://
     this.urls = this.urls.map((url) => {

apps/api/src/scraper/WebScraper/single_url.ts

@@ -106,7 +106,8 @@ export async function scrapWithPlaywright(url: string): Promise<string> {
 
 export async function scrapSingleUrl(
   urlToScrap: string,
-  pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false }
+  pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false },
+  existingHtml: string = ""
 ): Promise<Document> {
   urlToScrap = urlToScrap.trim();
 
@@ -197,8 +198,15 @@ export async function scrapSingleUrl(
       : ["scrapingBee", "playwright", "scrapingBeeLoad", "fetch"];
 
     for (const scraper of scrapersInOrder) {
+      // If exists text coming from crawler, use it
+      if (existingHtml && existingHtml.trim().length >= 100) {
+        let cleanedHtml = removeUnwantedElements(existingHtml, pageOptions);
+        text = await parseMarkdown(cleanedHtml);
+        html = existingHtml;
+        break;
+      }
       [text, html] = await attemptScraping(urlToScrap, scraper);
-      if (text && text.length >= 100) break;
+      if (text && text.trim().length >= 100) break;
       console.log(`Falling back to ${scraper}`);
     }
 

apps/api/src/services/queue-worker.ts

@@ -26,7 +26,7 @@ getWebScraperQueue().process(
         success: success,
         result: {
           links: docs.map((doc) => {
-            return { content: doc, source: doc.metadata.sourceURL };
+            return { content: doc, source: doc?.metadata?.sourceURL ?? doc?.url ?? "" };
           }),
         },
         project_id: job.data.project_id,