github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #168 from mendableai/nsc/allowed-keywords-in-blocklist

Browse Source 
feat: Allow privacy/legal/ other pages in social media websites
This commit is contained in:
Nicolas 2024-05-24 09:43:15 -07:00 committed by GitHub
commit 53a214cefb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -1,6 +1,7 @@
const socialMediaBlocklist = [ const socialMediaBlocklist = [
'facebook.com', 'facebook.com',
'twitter.com', 'twitter.com',
'x.com',
'instagram.com', 'instagram.com',
'linkedin.com', 'linkedin.com',
'pinterest.com', 'pinterest.com',
@ -14,12 +15,25 @@ const socialMediaBlocklist = [
'telegram.org', 'telegram.org',
]; ];
const allowedUrls = [ const allowedKeywords = [
'linkedin.com/pulse' 'pulse',
'privacy',
'terms',
'policy',
'user-agreement',
'legal',
'help',
'support',
'contact',
'about',
'careers',
'blog',
'press',
'conditions',
]; ];
export function isUrlBlocked(url: string): boolean { export function isUrlBlocked(url: string): boolean {
if (allowedUrls.some(allowedUrl => url.includes(allowedUrl))) { if (allowedKeywords.some(keyword => url.includes(keyword))) {
return false; return false;
} }