Merge pull request #64 from mendableai/feat/allowed-urls

[Feat] Added allowed urls
2024-04-25 08:43:16 -03:00 · 2024-04-25 08:43:16 -03:00 · abf6918642
commit abf6918642
parent 75e82869ed 75597f72a1
1 changed files with 8 additions and 0 deletions
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@ -14,6 +14,14 @@ const socialMediaBlocklist = [
  'telegram.org',
 ];

+const allowedUrls = [
+  'linkedin.com/pulse'
+];
+
 export function isUrlBlocked(url: string): boolean {
+  if (allowedUrls.some(allowedUrl => url.includes(allowedUrl))) {
+    return false;
+  }
+
  return socialMediaBlocklist.some(domain => url.includes(domain));
 }