github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: allowed keywords for now

Browse Source
This commit is contained in:
Nicolas 2024-05-20 17:24:21 -07:00
parent 01783dc336
commit 756f54466d
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -1,6 +1,7 @@
const socialMediaBlocklist = [
'facebook.com',
'twitter.com',
'x.com',
'instagram.com',
'linkedin.com',
'pinterest.com',
@ -14,12 +15,18 @@ const socialMediaBlocklist = [
'telegram.org',
];
const allowedUrls = [
'linkedin.com/pulse'
const allowedKeywords = [
'pulse',
'privacy',
'terms',
'policy',
'user-agreement',
'legal',
'help'
];
export function isUrlBlocked(url: string): boolean {
if (allowedUrls.some(allowedUrl => url.includes(allowedUrl))) {
if (allowedKeywords.some(keyword => url.includes(keyword))) {
return false;
}