Update blocklist.ts

2024-05-20 17:26:01 -07:00 · 2024-05-20 17:26:01 -07:00 · 7f64fe884a
commit 7f64fe884a
parent 756f54466d
1 changed files with 8 additions and 1 deletions
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@ -22,7 +22,14 @@ const allowedKeywords = [
  'policy',
  'user-agreement',
  'legal',
-  'help'
+  'help',
+  'support',
+  'contact',
+  'about',
+  'careers',
+  'blog',
+  'press',
+  'conditions',
 ];

 export function isUrlBlocked(url: string): boolean {