From 53cc4c396fea229ac87004e822f2228a090feb5c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 23 Apr 2024 17:05:58 -0700 Subject: [PATCH] Update search.ts --- apps/api/src/controllers/search.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/search.ts b/apps/api/src/controllers/search.ts index 6a1c7b4..4c03644 100644 --- a/apps/api/src/controllers/search.ts +++ b/apps/api/src/controllers/search.ts @@ -6,6 +6,7 @@ import { RateLimiterMode } from "../types"; import { logJob } from "../services/logging/log_job"; import { PageOptions, SearchOptions } from "../lib/entities"; import { search } from "../search"; +import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; export async function searchHelper( req: Request, @@ -28,7 +29,7 @@ export async function searchHelper( const tbs = searchOptions.tbs ?? null; const filter = searchOptions.filter ?? null; - const res = await search({query: query, advanced: advanced, num_results: searchOptions.limit ?? 7, tbs: tbs, filter: filter}); + let res = await search({query: query, advanced: advanced, num_results: searchOptions.limit ?? 7, tbs: tbs, filter: filter}); let justSearch = pageOptions.fetchPageContent === false; @@ -40,6 +41,9 @@ export async function searchHelper( return { success: true, error: "No search results found", returnCode: 200 }; } + // filter out social media links + res = res.filter((r) => !isUrlBlocked(r)); + const a = new WebScraperDataProvider(); await a.setOptions({ mode: "single_urls",