Merge pull request #77 from tractorjuice/patch-1

Add additional file extensions to crawler.ts
2024-05-22 10:16:49 -07:00 · 2024-05-22 10:16:49 -07:00 · 0d187f0425
commit 0d187f0425
parent f915b08016 1dc6458c6a
1 changed files with 6 additions and 0 deletions
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@ -324,6 +324,12 @@ export class WebCrawler {
      // ".docx",
      ".xlsx",
      ".xml",
+      ".avi",
+      ".flv",
+      ".woff",
+      ".ttf",
+      ".woff2",
+      ".webp"
    ];
    return fileExtensions.some((ext) => url.endsWith(ext));
  }