github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update test

Browse Source
This commit is contained in:
Eric Ciarla 2024-06-13 17:40:00 -04:00
parent 5e8aa92788
commit 095951aa4d
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -401,7 +401,7 @@ describe("E2E Tests for API Routes", () => {
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ .send({
url: "https://docs.dify.ai/v/zh-hans", url: "https://www.scrapethissite.com/pages/",
crawlerOptions: { maxDepth: 1 }, crawlerOptions: { maxDepth: 1 },
}); });
expect(crawlResponse.statusCode).toBe(200); expect(crawlResponse.statusCode).toBe(200);
@ -433,7 +433,7 @@ describe("E2E Tests for API Routes", () => {
// Check if all URLs have a maximum depth of 1 // Check if all URLs have a maximum depth of 1
urls.forEach((url: string) => { urls.forEach((url: string) => {
const depth = new URL(url).pathname.split("/").filter(Boolean).length; const depth = new URL(url).pathname.split("/").filter(Boolean).length;
expect(depth).toBeLessThanOrEqual(4); expect(depth).toBeLessThanOrEqual(3);
}); });
}, 120000); }, 120000);

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -163,7 +163,7 @@ export class WebScraperDataProvider {
inProgress?: (progress: Progress) => void inProgress?: (progress: Progress) => void
): Promise<Document[]> { ): Promise<Document[]> {
const baseURLDepth = new URL(this.urls[0]).pathname.split('/').length - 1; const baseURLDepth = new URL(this.urls[0]).pathname.split('/').length - 2;
const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth; const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth;