diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 0e2caeb..35ae746 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -511,6 +511,107 @@ describe("E2E Tests for API Routes", () => { // }, 120000); // 120 secs // }); + describe("POST /v0/crawl with fast mode", () => { + it("should complete the crawl under 20 seconds", async () => { + const startTime = Date.now(); + + const crawlResponse = await request(TEST_URL) + .post("/v0/crawl") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ + url: "https://flutterbricks.com", + crawlerOptions: { + mode: "fast" + } + }); + + expect(crawlResponse.statusCode).toBe(200); + + const jobId = crawlResponse.body.jobId; + let statusResponse; + let isFinished = false; + + while (!isFinished) { + statusResponse = await request(TEST_URL) + .get(`/v0/crawl/status/${jobId}`) + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); + + expect(statusResponse.statusCode).toBe(200); + isFinished = statusResponse.body.status === "completed"; + + if (!isFinished) { + await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again + } + } + + const endTime = Date.now(); + const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds + + console.log(`Time elapsed: ${timeElapsed} seconds`); + + expect(statusResponse.body.status).toBe("completed"); + expect(statusResponse.body).toHaveProperty("data"); + expect(statusResponse.body.data[0]).toHaveProperty("content"); + expect(statusResponse.body.data[0]).toHaveProperty("markdown"); + const results = statusResponse.body.data; + // results.forEach((result, i) => { + // console.log(result.metadata.sourceURL); + // }); + expect(results.length).toBeGreaterThanOrEqual(10); + expect(results.length).toBeLessThanOrEqual(15); + + }, 20000); + + // it("should complete the crawl in more than 10 seconds", async () => { + // const startTime = Date.now(); + + // const crawlResponse = await request(TEST_URL) + // .post("/v0/crawl") + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + // .set("Content-Type", "application/json") + // .send({ + // url: "https://flutterbricks.com", + // }); + + // expect(crawlResponse.statusCode).toBe(200); + + // const jobId = crawlResponse.body.jobId; + // let statusResponse; + // let isFinished = false; + + // while (!isFinished) { + // statusResponse = await request(TEST_URL) + // .get(`/v0/crawl/status/${jobId}`) + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); + + // expect(statusResponse.statusCode).toBe(200); + // isFinished = statusResponse.body.status === "completed"; + + // if (!isFinished) { + // await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again + // } + // } + + // const endTime = Date.now(); + // const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds + + // console.log(`Time elapsed: ${timeElapsed} seconds`); + + // expect(statusResponse.body.status).toBe("completed"); + // expect(statusResponse.body).toHaveProperty("data"); + // expect(statusResponse.body.data[0]).toHaveProperty("content"); + // expect(statusResponse.body.data[0]).toHaveProperty("markdown"); + // const results = statusResponse.body.data; + // // results.forEach((result, i) => { + // // console.log(result.metadata.sourceURL); + // // }); + // expect(results.length).toBeGreaterThanOrEqual(10); + // expect(results.length).toBeLessThanOrEqual(15); + + // }, 50000);// 15 seconds timeout to account for network delays + }); + describe("GET /is-production", () => { it("should return the production status", async () => { const response = await request(TEST_URL).get("/is-production"); diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index a387b54..15550be 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -44,6 +44,7 @@ export type WebScraperOptions = { limit?: number; generateImgAltText?: boolean; replaceAllPathsWithAbsolutePaths?: boolean; + mode?: "default" | "fast"; // have a mode of some sort }; pageOptions?: PageOptions; extractorOptions?: ExtractorOptions; diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 0248df2..521b1e1 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -4,7 +4,7 @@ import { URL } from "url"; import { getLinksFromSitemap } from "./sitemap"; import async from "async"; import { Progress } from "../../lib/entities"; -import { scrapWithScrapingBee } from "./single_url"; +import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url"; import robotsParser from "robots-parser"; export class WebCrawler { @@ -15,7 +15,7 @@ export class WebCrawler { private maxCrawledLinks: number; private maxCrawledDepth: number; private visited: Set = new Set(); - private crawledUrls: Set = new Set(); + private crawledUrls: Map = new Map(); private limit: number; private robotsTxtUrl: string; private robots: any; @@ -51,7 +51,6 @@ export class WebCrawler { this.generateImgAltText = generateImgAltText ?? false; } - private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] { return sitemapLinks .filter((link) => { @@ -99,7 +98,7 @@ export class WebCrawler { concurrencyLimit: number = 5, limit: number = 10000, maxDepth: number = 10 - ): Promise { + ): Promise<{ url: string, html: string }[]> { // Fetch and parse robots.txt try { const response = await axios.get(this.robotsTxtUrl); @@ -111,7 +110,7 @@ export class WebCrawler { const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl); if (sitemapLinks.length > 0) { const filteredLinks = this.filterLinks(sitemapLinks, limit, maxDepth); - return filteredLinks; + return filteredLinks.map(link => ({ url: link, html: "" })); } const urls = await this.crawlUrls( @@ -123,18 +122,19 @@ export class WebCrawler { urls.length === 0 && this.filterLinks([this.initialUrl], limit, this.maxCrawledDepth).length > 0 ) { - return [this.initialUrl]; + return [{ url: this.initialUrl, html: "" }]; } // make sure to run include exclude here again - return this.filterLinks(urls, limit, this.maxCrawledDepth); + const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth); + return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" })); } private async crawlUrls( urls: string[], concurrencyLimit: number, inProgress?: (progress: Progress) => void - ): Promise { + ): Promise<{ url: string, html: string }[]> { const queue = async.queue(async (task: string, callback) => { if (this.crawledUrls.size >= this.maxCrawledLinks) { if (callback && typeof callback === "function") { @@ -143,13 +143,13 @@ export class WebCrawler { return; } const newUrls = await this.crawl(task); - newUrls.forEach((url) => this.crawledUrls.add(url)); + newUrls.forEach((page) => this.crawledUrls.set(page.url, page.html)); if (inProgress && newUrls.length > 0) { inProgress({ current: this.crawledUrls.size, total: this.maxCrawledLinks, status: "SCRAPING", - currentDocumentUrl: newUrls[newUrls.length - 1], + currentDocumentUrl: newUrls[newUrls.length - 1].url, }); } else if (inProgress) { inProgress({ @@ -159,7 +159,7 @@ export class WebCrawler { currentDocumentUrl: task, }); } - await this.crawlUrls(newUrls, concurrencyLimit, inProgress); + await this.crawlUrls(newUrls.map((p) => p.url), concurrencyLimit, inProgress); if (callback && typeof callback === "function") { callback(); } @@ -175,10 +175,10 @@ export class WebCrawler { } ); await queue.drain(); - return Array.from(this.crawledUrls); + return Array.from(this.crawledUrls.entries()).map(([url, html]) => ({ url, html })); } - async crawl(url: string): Promise { + async crawl(url: string): Promise<{url: string, html: string}[]> { if (this.visited.has(url) || !this.robots.isAllowed(url, "FireCrawlAgent")) return []; this.visited.add(url); @@ -193,16 +193,17 @@ export class WebCrawler { } try { - let content; - // If it is the first link, fetch with scrapingbee + let content : string = ""; + // If it is the first link, fetch with single url if (this.visited.size === 1) { - content = await scrapWithScrapingBee(url, "load"); + const page = await scrapSingleUrl(url, {includeHtml: true}); + content = page.html ?? "" } else { const response = await axios.get(url); - content = response.data; + content = response.data ?? ""; } const $ = load(content); - let links: string[] = []; + let links: {url: string, html: string}[] = []; $("a").each((_, element) => { const href = $(element).attr("href"); @@ -215,7 +216,6 @@ export class WebCrawler { const path = url.pathname; if ( - // fullUrl.startsWith(this.initialUrl) && // this condition makes it stop crawling back the url this.isInternalLink(fullUrl) && this.matchesPattern(fullUrl) && this.noSections(fullUrl) && @@ -223,12 +223,13 @@ export class WebCrawler { !this.matchesExcludes(path) && this.robots.isAllowed(fullUrl, "FireCrawlAgent") ) { - links.push(fullUrl); + links.push({url: fullUrl, html: content}); } } }); - return links.filter((link) => !this.visited.has(link)); + // Create a new list to return to avoid modifying the visited list + return links.filter((link) => !this.visited.has(link.url)); } catch (error) { return []; } diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 7ef0a10..c95e889 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -35,6 +35,7 @@ export class WebScraperDataProvider { private replaceAllPathsWithAbsolutePaths?: boolean = false; private generateImgAltTextModel: "gpt-4-turbo" | "claude-3-opus" = "gpt-4-turbo"; + private crawlerMode: string = "default"; authorize(): void { throw new Error("Method not implemented."); @@ -46,7 +47,8 @@ export class WebScraperDataProvider { private async convertUrlsToDocuments( urls: string[], - inProgress?: (progress: Progress) => void + inProgress?: (progress: Progress) => void, + allHtmls?: string[] ): Promise { const totalUrls = urls.length; let processedUrls = 0; @@ -56,7 +58,8 @@ export class WebScraperDataProvider { const batchUrls = urls.slice(i, i + this.concurrentRequests); await Promise.all( batchUrls.map(async (url, index) => { - const result = await scrapSingleUrl(url, this.pageOptions); + const existingHTML = allHtmls ? allHtmls[i + index] : ""; + const result = await scrapSingleUrl(url, this.pageOptions, existingHTML); processedUrls++; if (inProgress) { inProgress({ @@ -139,13 +142,26 @@ export class WebScraperDataProvider { limit: this.limit, generateImgAltText: this.generateImgAltText, }); + let links = await crawler.start(inProgress, 5, this.limit, this.maxCrawledDepth); + + const allLinks = links.map((e) => e.url); + const allHtmls = links.map((e)=> e.html); + if (this.returnOnlyUrls) { - return this.returnOnlyUrlsResponse(links, inProgress); + return this.returnOnlyUrlsResponse(allLinks , inProgress); + } + + let documents = []; + // check if fast mode is enabled and there is html inside the links + if (this.crawlerMode === "fast" && links.some((link) => link.html)) { + console.log("Fast mode enabled"); + documents = await this.processLinks(allLinks, inProgress, allHtmls); + }else{ + documents = await this.processLinks(allLinks, inProgress); } - let documents = await this.processLinks(links, inProgress); - return this.cacheAndFinalizeDocuments(documents, links); + return this.cacheAndFinalizeDocuments(documents, allLinks); } private async handleSingleUrlsMode( @@ -187,14 +203,17 @@ export class WebScraperDataProvider { private async processLinks( links: string[], - inProgress?: (progress: Progress) => void + inProgress?: (progress: Progress) => void, + allHtmls?: string[] ): Promise { let pdfLinks = links.filter((link) => link.endsWith(".pdf")); let pdfDocuments = await this.fetchPdfDocuments(pdfLinks); links = links.filter((link) => !link.endsWith(".pdf")); - - let documents = await this.convertUrlsToDocuments(links, inProgress); + + let documents = await this.convertUrlsToDocuments(links, inProgress, allHtmls); documents = await this.getSitemapData(this.urls[0], documents); + + documents = this.applyPathReplacements(documents); // documents = await this.applyImgAltText(documents); @@ -397,6 +416,7 @@ export class WebScraperDataProvider { this.replaceAllPathsWithAbsolutePaths = options.crawlerOptions?.replaceAllPathsWithAbsolutePaths ?? false; //! @nicolas, for some reason this was being injected and breakign everything. Don't have time to find source of the issue so adding this check this.excludes = this.excludes.filter((item) => item !== ""); + this.crawlerMode = options.crawlerOptions?.mode ?? "default"; // make sure all urls start with https:// this.urls = this.urls.map((url) => { diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index c43ea40..4bbaee7 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -106,7 +106,8 @@ export async function scrapWithPlaywright(url: string): Promise { export async function scrapSingleUrl( urlToScrap: string, - pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false } + pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false }, + existingHtml: string = "" ): Promise { urlToScrap = urlToScrap.trim(); @@ -197,8 +198,15 @@ export async function scrapSingleUrl( : ["scrapingBee", "playwright", "scrapingBeeLoad", "fetch"]; for (const scraper of scrapersInOrder) { + // If exists text coming from crawler, use it + if (existingHtml && existingHtml.trim().length >= 100) { + let cleanedHtml = removeUnwantedElements(existingHtml, pageOptions); + text = await parseMarkdown(cleanedHtml); + html = existingHtml; + break; + } [text, html] = await attemptScraping(urlToScrap, scraper); - if (text && text.length >= 100) break; + if (text && text.trim().length >= 100) break; console.log(`Falling back to ${scraper}`); } diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 78ea030..ef7bb1f 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -26,7 +26,7 @@ getWebScraperQueue().process( success: success, result: { links: docs.map((doc) => { - return { content: doc, source: doc.metadata.sourceURL }; + return { content: doc, source: doc?.metadata?.sourceURL ?? doc?.url ?? "" }; }), }, project_id: job.data.project_id,