From 095951aa4df5a47a55fc5fdc0c599f1ecfdb2b95 Mon Sep 17 00:00:00 2001
From: Eric Ciarla <ericciarla@yahoo.com>
Date: Thu, 13 Jun 2024 17:40:00 -0400
Subject: [PATCH] Update test

---
 apps/api/src/__tests__/e2e_withAuth/index.test.ts | 4 ++--
 apps/api/src/scraper/WebScraper/index.ts          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 39635cb..b0642cf 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -401,7 +401,7 @@ describe("E2E Tests for API Routes", () => {
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
         .send({
-          url: "https://docs.dify.ai/v/zh-hans",
+          url: "https://www.scrapethissite.com/pages/",
           crawlerOptions: { maxDepth: 1 },
         });
       expect(crawlResponse.statusCode).toBe(200);
@@ -433,7 +433,7 @@ describe("E2E Tests for API Routes", () => {
       // Check if all URLs have a maximum depth of 1
       urls.forEach((url: string) => {
         const depth = new URL(url).pathname.split("/").filter(Boolean).length;
-        expect(depth).toBeLessThanOrEqual(4);
+        expect(depth).toBeLessThanOrEqual(3);
       });
     }, 120000);
 
diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts
index ef0c839..e8ce813 100644
--- a/apps/api/src/scraper/WebScraper/index.ts
+++ b/apps/api/src/scraper/WebScraper/index.ts
@@ -163,7 +163,7 @@ export class WebScraperDataProvider {
     inProgress?: (progress: Progress) => void
   ): Promise<Document[]> {
 
-    const baseURLDepth = new URL(this.urls[0]).pathname.split('/').length - 1;
+    const baseURLDepth = new URL(this.urls[0]).pathname.split('/').length - 2;
     const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth;