github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update blocklist.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-24 15:04:15 -07:00
parent e5c8719554
commit e98434606d
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -1,5 +1,6 @@
const socialMediaBlocklist = [
'facebook.com',
'x.com',
'twitter.com',
'instagram.com',
'linkedin.com',
@ -32,9 +33,17 @@ const allowedKeywords = [
];
export function isUrlBlocked(url: string): boolean {
// Check if the URL contains any allowed keywords
if (allowedKeywords.some(keyword => url.includes(keyword))) {
return false;
}
return socialMediaBlocklist.some(domain => url.includes(domain));
// Check if the URL matches any domain in the blocklist
return socialMediaBlocklist.some(domain => {
// Create a regular expression to match the exact domain
const domainPattern = new RegExp(`(^|\\.)${domain.replace('.', '\\.')}$`);
// Test the hostname of the URL against the pattern
return domainPattern.test(new URL(url).hostname);
});
}