github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update maxDepth tests

Browse Source
This commit is contained in:
Eric Ciarla 2024-06-13 18:46:30 -04:00
parent 393bd45237
commit ab9de0f5ab
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -390,7 +390,8 @@ describe("E2E Tests for API Routes", () => {
// Check if all URLs have a maximum depth of 1 // Check if all URLs have a maximum depth of 1
urls.forEach((url: string) => { urls.forEach((url: string) => {
const depth = new URL(url).pathname.split("/").filter(Boolean).length; const pathSplits = new URL(url).pathname.split('/');
const depth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
expect(depth).toBeLessThanOrEqual(2); expect(depth).toBeLessThanOrEqual(2);
}); });
}, 120000); }, 120000);
@ -432,7 +433,8 @@ describe("E2E Tests for API Routes", () => {
// Check if all URLs have an absolute maximum depth of 3 after the base URL depth was 2 and the maxDepth was 1 // Check if all URLs have an absolute maximum depth of 3 after the base URL depth was 2 and the maxDepth was 1
urls.forEach((url: string) => { urls.forEach((url: string) => {
const depth = new URL(url).pathname.split("/").filter(Boolean).length; const pathSplits = new URL(url).pathname.split('/');
const depth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
expect(depth).toBeLessThanOrEqual(3); expect(depth).toBeLessThanOrEqual(3);
}); });
}, 120000); }, 120000);
@ -749,7 +751,8 @@ describe("E2E Tests for API Routes", () => {
// Check if all URLs have a maximum depth of 1 // Check if all URLs have a maximum depth of 1
urls.forEach((url) => { urls.forEach((url) => {
const depth = new URL(url).pathname.split("/").filter(Boolean).length; const pathSplits = new URL(url).pathname.split('/');
const depth = pathSplits.length - (pathSplits[0].length === 0 && pathSplits[pathSplits.length - 1].length === 0 ? 1 : 0);
expect(depth).toBeLessThanOrEqual(2); expect(depth).toBeLessThanOrEqual(2);
}); });
}, 180000); }, 180000);

1
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -158,7 +158,6 @@ export class WebCrawler {
// make sure to run include exclude here again // make sure to run include exclude here again
const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth); const filteredUrls = this.filterLinks(urls.map(urlObj => urlObj.url), limit, this.maxCrawledDepth);
return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" })); return filteredUrls.map(url => ({ url, html: urls.find(urlObj => urlObj.url === url)?.html || "" }));
} }