github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge branch 'nsc/improved-blocklist'

Browse Source
This commit is contained in:
Nicolas 2024-06-03 16:44:33 -07:00
commit 93bb53271e
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts
View File

@ -40,6 +40,7 @@ describe('isUrlBlocked', () => {
'https://whatsapp.com/press', 'https://whatsapp.com/press',
'https://wechat.com/careers', 'https://wechat.com/careers',
'https://telegram.org/conditions', 'https://telegram.org/conditions',
'https://wix.com/careers',
]; ];
allowedUrls.forEach(url => { allowedUrls.forEach(url => {
@ -54,6 +55,8 @@ describe('isUrlBlocked', () => {
'https://subdomain.example.com', 'https://subdomain.example.com',
'firecrawl.dev', 'firecrawl.dev',
'amazon.com', 'amazon.com',
'wix.com',
'https://wix.com'
]; ];
nonBlockedUrls.forEach(url => { nonBlockedUrls.forEach(url => {