github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update blocklist.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-20 17:26:01 -07:00
parent 756f54466d
commit 7f64fe884a
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -22,7 +22,14 @@ const allowedKeywords = [
'policy', 'policy',
'user-agreement', 'user-agreement',
'legal', 'legal',
'help' 'help',
'support',
'contact',
'about',
'careers',
'blog',
'press',
'conditions',
]; ];
export function isUrlBlocked(url: string): boolean { export function isUrlBlocked(url: string): boolean {