diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index f432f43..dfb52c4 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -163,12 +163,15 @@ export class WebScraperDataProvider { inProgress?: (progress: Progress) => void ): Promise { + const baseURLDepth = new URL(this.urls[0]).pathname.split('/').length - 1; + const adjustedMaxDepth = this.maxCrawledDepth + baseURLDepth; + const crawler = new WebCrawler({ initialUrl: this.urls[0], includes: this.includes, excludes: this.excludes, maxCrawledLinks: this.maxCrawledLinks, - maxCrawledDepth: this.maxCrawledDepth, + maxCrawledDepth: adjustedMaxDepth, limit: this.limit, generateImgAltText: this.generateImgAltText, allowBackwardCrawling: this.allowBackwardCrawling,