github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #285 from mendableai/feat/maxDepthRelative

Browse Source 
Feat/max depth relative
This commit is contained in:
Eric Ciarla 2024-06-15 17:23:20 -04:00 committed by GitHub
commit d236fe9a1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 240 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -619,12 +619,13 @@ describe("E2E Tests for API Routes", () => {
}, 180000);
it.concurrent("should return a successful response with relative max depth option for a valid crawl job with maxDepths equals to zero", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
url: "https://www.mendable.ai",
crawlerOptions: { maxDepth: 0 },
});
expect(crawlResponse.statusCode).toBe(200);
@ -651,6 +652,11 @@ describe("E2E Tests for API Routes", () => {
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
const testurls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL
);
//console.log(testurls)
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
@ -671,58 +677,9 @@ describe("E2E Tests for API Routes", () => {
});
}, 180000);
it.concurrent("should return a successful response with relative max depth option for a valid crawl job with maxDepth equals to 2", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "proxyway.com",
crawlerOptions: { maxDepth: 2, limit: 5 },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
// wait for 60 seconds
let isCompleted = false;
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL
);
expect(urls.length).toBeGreaterThanOrEqual(1);
// Check if all URLs have an absolute maximum depth of 3 after the base URL depth was 2 and the maxDepth was 1
urls.forEach((url: string) => {
const pathSplits = new URL(url).pathname.split('/');
const depth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
expect(depth).toBeLessThanOrEqual(3);
});
}, 180000);
// it.concurrent("should return a successful response with a valid API key and valid limit option", async () => {
// const crawlResponse = await request(TEST_URL)
@ -1009,54 +966,6 @@ describe("E2E Tests for API Routes", () => {
}, 180000); // 120 seconds
it.concurrent("should return a successful response with max depth option for a valid crawl job", async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
crawlerOptions: { maxDepth: 1 },
});
expect(crawlResponse.statusCode).toBe(200);
let isCompleted = false;
let completedResponse;
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
if (response.body.status === "completed") {
isCompleted = true;
completedResponse = response;
}
}
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL
);
expect(urls.length).toBeGreaterThan(1);
// Check if all URLs have a maximum depth of 1
urls.forEach((url) => {
const pathSplits = new URL(url).pathname.split('/');
const depth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
expect(depth).toBeLessThanOrEqual(2);
});
}, 180000);
it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option (2)", async () => {
const crawlResponse = await request(TEST_URL)

163
apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts Normal file
View File

@ -0,0 +1,163 @@
// crawler.test.ts
import { WebCrawler } from '../crawler';
import axios from 'axios';
import robotsParser from 'robots-parser';
import { getAdjustedMaxDepth } from '../utils/maxDepthUtils';
jest.mock('axios');
jest.mock('robots-parser');
describe('WebCrawler maxDepth and filterLinks', () => {
let crawler: WebCrawler;
const mockAxios = axios as jest.Mocked<typeof axios>;
const mockRobotsParser = robotsParser as jest.MockedFunction<typeof robotsParser>;
let maxCrawledDepth: number;
beforeEach(() => {
// Setup default mocks
mockAxios.get.mockImplementation((url) => {
if (url.includes('robots.txt')) {
return Promise.resolve({ data: 'User-agent: *\nAllow: /' });
} else if (url.includes('sitemap.xml')) {
return Promise.resolve({ data: 'sitemap content' }); // You would normally parse this to URLs
}
return Promise.resolve({ data: '<html></html>' });
});
mockRobotsParser.mockReturnValue({
isAllowed: jest.fn().mockReturnValue(true),
isDisallowed: jest.fn().mockReturnValue(false),
getMatchingLineNumber: jest.fn().mockReturnValue(0),
getCrawlDelay: jest.fn().mockReturnValue(0),
getSitemaps: jest.fn().mockReturnValue([]),
getPreferredHost: jest.fn().mockReturnValue('example.com')
});
});
it('should filter out links that exceed maxDepth param of 2 based on enterURL depth of 0 ', async () => {
const initialUrl = 'http://example.com'; // Set initial URL for this test
const enteredMaxCrawledDepth = 2;
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
crawler = new WebCrawler({
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: 100,
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
});
// Mock sitemap fetching function to return controlled links
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
initialUrl, // depth 0
initialUrl + '/page1', // depth 1
initialUrl + '/page1/page2', // depth 2
initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
]);
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
expect(results).toEqual([
{ url: initialUrl, html: '' },
{ url: initialUrl + '/page1', html: '' },
{ url: initialUrl + '/page1/page2', html: '' }
]);
// Ensure that the link with depth 3 is not included
expect(results.some(r => r.url === initialUrl + '/page1/page2/page3')).toBe(false);
});
it('should filter out links that exceed maxDepth param of 0 based on enterURL depth of 0 ', async () => {
const initialUrl = 'http://example.com'; // Set initial URL for this test
const enteredMaxCrawledDepth = 0;
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
crawler = new WebCrawler({
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: 100,
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
});
// Mock sitemap fetching function to return controlled links
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
initialUrl, // depth 0
initialUrl + '/page1', // depth 1
initialUrl + '/page1/page2', // depth 2
initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
]);
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
expect(results).toEqual([
{ url: initialUrl, html: '' },
]);
});
it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 1 ', async () => {
const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
const enteredMaxCrawledDepth = 1;
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
crawler = new WebCrawler({
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: 100,
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
});
// Mock sitemap fetching function to return controlled links
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
initialUrl, // depth 0
initialUrl + '/page2', // depth 1
initialUrl + '/page2/page3', // depth 2
initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
]);
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
expect(results).toEqual([
{ url: initialUrl, html: '' },
{ url: initialUrl + '/page2', html: '' }
]);
});
it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 2 ', async () => {
const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
const enteredMaxCrawledDepth = 2;
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
crawler = new WebCrawler({
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: 100,
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
});
// Mock sitemap fetching function to return controlled links
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
initialUrl, // depth 0
initialUrl + '/page2', // depth 1
initialUrl + '/page2/page3', // depth 2
initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
]);
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
expect(results).toEqual([
{ url: initialUrl, html: '' },
{ url: initialUrl + '/page2', html: '' },
{ url: initialUrl + '/page2/page3', html: '' }
]);
});
// Add more tests to cover other scenarios, such as checking includes and excludes
});

8
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -6,6 +6,7 @@ import async from "async";
import { CrawlerOptions, PageOptions, Progress } from "../../lib/entities";
import { scrapSingleUrl, scrapWithScrapingBee } from "./single_url";
import robotsParser from "robots-parser";
import { getURLDepth } from "./utils/maxDepthUtils";
export class WebCrawler {
private initialUrl: string;
@ -60,8 +61,10 @@ export class WebCrawler {
.filter((link) => {
const url = new URL(link);
const path = url.pathname;
const depth = url.pathname.split('/').length - 1;
const depth = getURLDepth(url.toString());
// Check if the link exceeds the maximum depth allowed
if (depth > maxDepth) {
return false;
@ -136,8 +139,10 @@ export class WebCrawler {
if(!crawlerOptions?.ignoreSitemap){
const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);
if (sitemapLinks.length > 0) {
let filteredLinks = this.filterLinks(sitemapLinks, limit, maxDepth);
return filteredLinks.map(link => ({ url: link, html: "" }));
}
}
@ -148,6 +153,7 @@ export class WebCrawler {
concurrencyLimit,
inProgress
);
if (
urls.length === 0 &&

10
apps/api/src/scraper/WebScraper/index.ts
View File

@ -18,6 +18,7 @@ import {
import { generateCompletions } from "../../lib/LLM-extraction";
import { getWebScraperQueue } from "../../../src/services/queue-service";
import { fetchAndProcessDocx } from "./utils/docxProcessor";
import { getAdjustedMaxDepth, getURLDepth } from "./utils/maxDepthUtils";
export class WebScraperDataProvider {
private bullJobId: string;
@ -163,16 +164,12 @@ export class WebScraperDataProvider {
inProgress?: (progress: Progress) => void
): Promise<Document[]> {
const pathSplits = new URL(this.urls[0]).pathname.split('/');
const baseURLDepth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth;
const crawler = new WebCrawler({
initialUrl: this.urls[0],
includes: this.includes,
excludes: this.excludes,
maxCrawledLinks: this.maxCrawledLinks,
maxCrawledDepth: adjustedMaxDepth,
maxCrawledDepth: getAdjustedMaxDepth(this.urls[0], this.maxCrawledDepth),
limit: this.limit,
generateImgAltText: this.generateImgAltText,
allowBackwardCrawling: this.allowBackwardCrawling,
@ -580,8 +577,7 @@ export class WebScraperDataProvider {
filterDepth(documents: Document[]): Document[] {
return documents.filter((document) => {
const url = new URL(document.metadata.sourceURL);
const path = url.pathname;
return path.split("/").length <= this.maxCrawledDepth;
return getURLDepth(url.toString()) <= this.maxCrawledDepth;
});
}
}

47
apps/api/src/scraper/WebScraper/utils/__tests__/maxDepthUtils.test.ts Normal file
View File

@ -0,0 +1,47 @@
import { getURLDepth, getAdjustedMaxDepth } from '../maxDepthUtils';
describe('Testing getURLDepth and getAdjustedMaxDepth', () => {
it('should return 0 for root - mendable.ai', () => {
const enteredURL = "https://www.mendable.ai/"
expect(getURLDepth(enteredURL)).toBe(0);
});
it('should return 0 for root - scrapethissite.com', () => {
const enteredURL = "https://scrapethissite.com/"
expect(getURLDepth(enteredURL)).toBe(0);
});
it('should return 1 for scrapethissite.com/pages', () => {
const enteredURL = "https://scrapethissite.com/pages"
expect(getURLDepth(enteredURL)).toBe(1);
});
it('should return 2 for scrapethissite.com/pages/articles', () => {
const enteredURL = "https://scrapethissite.com/pages/articles"
expect(getURLDepth(enteredURL)).toBe(2);
});
it('Adjusted maxDepth should return 1 for scrapethissite.com and max depth param of 1', () => {
const enteredURL = "https://scrapethissite.com"
expect(getAdjustedMaxDepth(enteredURL, 1)).toBe(1);
});
it('Adjusted maxDepth should return 0 for scrapethissite.com and max depth param of 0', () => {
const enteredURL = "https://scrapethissite.com"
expect(getAdjustedMaxDepth(enteredURL, 0)).toBe(0);
});
it('Adjusted maxDepth should return 0 for mendable.ai and max depth param of 0', () => {
const enteredURL = "https://mendable.ai"
expect(getAdjustedMaxDepth(enteredURL, 0)).toBe(0);
});
it('Adjusted maxDepth should return 4 for scrapethissite.com/pages/articles and max depth param of 2', () => {
const enteredURL = "https://scrapethissite.com/pages/articles"
expect(getAdjustedMaxDepth(enteredURL, 2)).toBe(4);
});
});

12
apps/api/src/scraper/WebScraper/utils/maxDepthUtils.ts Normal file
View File

@ -0,0 +1,12 @@
export function getAdjustedMaxDepth(url: string, maxCrawlDepth: number): number {
const baseURLDepth = getURLDepth(url);
const adjustedMaxDepth = maxCrawlDepth + baseURLDepth;
return adjustedMaxDepth;
}
export function getURLDepth(url: string): number {
const pathSplits = new URL(url).pathname.split('/');
return pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0) - 1;
}