github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-06-03 16:42:42 -07:00
parent 48d1ec05b2
commit c69c89f838
2 changed files with 75 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts Normal file
View File

@ -0,0 +1,63 @@
import { isUrlBlocked } from '../blocklist';
describe('isUrlBlocked', () => {
it('should return true for blocked social media URLs', () => {
const blockedUrls = [
'https://www.facebook.com',
'https://twitter.com/someuser',
'https://instagram.com/someuser',
'https://www.linkedin.com/in/someuser',
'https://pinterest.com/someuser',
'https://snapchat.com/someuser',
'https://tiktok.com/@someuser',
'https://reddit.com/r/somesubreddit',
'https://flickr.com/photos/someuser',
'https://whatsapp.com/someuser',
'https://wechat.com/someuser',
'https://telegram.org/someuser',
];
blockedUrls.forEach(url => {
if (!isUrlBlocked(url)) {
console.log(`URL not blocked: ${url}`);
}
expect(isUrlBlocked(url)).toBe(true);
});
});
it('should return false for URLs containing allowed keywords', () => {
const allowedUrls = [
'https://www.facebook.com/privacy',
'https://twitter.com/terms',
'https://instagram.com/legal',
'https://www.linkedin.com/help',
'https://pinterest.com/about',
'https://snapchat.com/support',
'https://tiktok.com/contact',
'https://reddit.com/user-agreement',
'https://tumblr.com/policy',
'https://flickr.com/blog',
'https://whatsapp.com/press',
'https://wechat.com/careers',
'https://telegram.org/conditions',
];
allowedUrls.forEach(url => {
expect(isUrlBlocked(url)).toBe(false);
});
});
it('should return false for non-blocked URLs', () => {
const nonBlockedUrls = [
'https://www.example.com',
'https://www.somewebsite.org',
'https://subdomain.example.com',
'firecrawl.dev',
'amazon.com',
];
nonBlockedUrls.forEach(url => {
expect(isUrlBlocked(url)).toBe(false);
});
});
});

5
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -38,6 +38,7 @@ export function isUrlBlocked(url: string): boolean {
return false; return false;
} }
try {
// Check if the URL matches any domain in the blocklist // Check if the URL matches any domain in the blocklist
return socialMediaBlocklist.some(domain => { return socialMediaBlocklist.some(domain => {
// Create a regular expression to match the exact domain // Create a regular expression to match the exact domain
@ -45,5 +46,9 @@ export function isUrlBlocked(url: string): boolean {
// Test the hostname of the URL against the pattern // Test the hostname of the URL against the pattern
return domainPattern.test(new URL(url).hostname); return domainPattern.test(new URL(url).hostname);
}); });
} catch (e) {
// If an error occurs (e.g., invalid URL), return false
return false;
}
} }