diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts index ededfc7..c3d37c4 100644 --- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts +++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts @@ -22,7 +22,14 @@ const allowedKeywords = [ 'policy', 'user-agreement', 'legal', - 'help' + 'help', + 'support', + 'contact', + 'about', + 'careers', + 'blog', + 'press', + 'conditions', ]; export function isUrlBlocked(url: string): boolean {