github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update search.ts

Browse Source
This commit is contained in:
Nicolas 2024-04-23 17:05:58 -07:00
parent 734c76fc56
commit 53cc4c396f
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/controllers/search.ts
View File

@ -6,6 +6,7 @@ import { RateLimiterMode } from "../types";
import { logJob } from "../services/logging/log_job"; import { logJob } from "../services/logging/log_job";
import { PageOptions, SearchOptions } from "../lib/entities"; import { PageOptions, SearchOptions } from "../lib/entities";
import { search } from "../search"; import { search } from "../search";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
export async function searchHelper( export async function searchHelper(
req: Request, req: Request,
@ -28,7 +29,7 @@ export async function searchHelper(
const tbs = searchOptions.tbs ?? null; const tbs = searchOptions.tbs ?? null;
const filter = searchOptions.filter ?? null; const filter = searchOptions.filter ?? null;
const res = await search({query: query, advanced: advanced, num_results: searchOptions.limit ?? 7, tbs: tbs, filter: filter}); let res = await search({query: query, advanced: advanced, num_results: searchOptions.limit ?? 7, tbs: tbs, filter: filter});
let justSearch = pageOptions.fetchPageContent === false; let justSearch = pageOptions.fetchPageContent === false;
@ -40,6 +41,9 @@ export async function searchHelper(
return { success: true, error: "No search results found", returnCode: 200 }; return { success: true, error: "No search results found", returnCode: 200 };
} }
// filter out social media links
res = res.filter((r) => !isUrlBlocked(r));
const a = new WebScraperDataProvider(); const a = new WebScraperDataProvider();
await a.setOptions({ await a.setOptions({
mode: "single_urls", mode: "single_urls",