Merge pull request #64 from mendableai/feat/allowed-urls

[Feat] Added allowed urls

apps/api/src/scraper/WebScraper/utils/blocklist.ts

@@ -14,6 +14,14 @@ const socialMediaBlocklist = [
   'telegram.org',
 ];
 
+const allowedUrls = [
+  'linkedin.com/pulse'
+];
+
 export function isUrlBlocked(url: string): boolean {
+  if (allowedUrls.some(allowedUrl => url.includes(allowedUrl))) {
+    return false;
+  }
+
   return socialMediaBlocklist.some(domain => url.includes(domain));
 }