diff --git a/apps/api/src/controllers/search.ts b/apps/api/src/controllers/search.ts index 6839d8a..bc81f69 100644 --- a/apps/api/src/controllers/search.ts +++ b/apps/api/src/controllers/search.ts @@ -29,7 +29,16 @@ export async function searchHelper( const tbs = searchOptions.tbs ?? null; const filter = searchOptions.filter ?? null; - let res = await search({query: query, advanced: advanced, num_results: searchOptions.limit ?? 7, tbs: tbs, filter: filter}); + let res = await search({ + query: query, + advanced: advanced, + num_results: searchOptions.limit ?? 7, + tbs: tbs, + filter: filter, + lang: searchOptions.lang ?? "en", + country: searchOptions.country ?? "us", + location: searchOptions.location, + }); let justSearch = pageOptions.fetchPageContent === false; diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index bda7448..7b46305 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -20,6 +20,9 @@ export type SearchOptions = { limit?: number; tbs?: string; filter?: string; + lang?: string; + country?: string; + location?: string; }; export type WebScraperOptions = { diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.ts index 0f7c72f..a6d09ed 100644 --- a/apps/api/src/search/googlesearch.ts +++ b/apps/api/src/search/googlesearch.ts @@ -17,11 +17,12 @@ function get_useragent(): string { return _useragent_list[Math.floor(Math.random() * _useragent_list.length)]; } -async function _req(term: string, results: number, lang: string, start: number, proxies: any, timeout: number, tbs: string = null, filter: string = null) { +async function _req(term: string, results: number, lang: string, country: string, start: number, proxies: any, timeout: number, tbs: string = null, filter: string = null) { const params = { "q": term, "num": results, // Number of results to return "hl": lang, + "gl": country, "start": start, }; if (tbs) { @@ -50,7 +51,7 @@ async function _req(term: string, results: number, lang: string, start: number, -export async function google_search(term: string, advanced = false, num_results = 7, tbs = null, filter = null, lang = "en", proxy = null, sleep_interval = 0, timeout = 5000, ) :Promise { +export async function google_search(term: string, advanced = false, num_results = 7, tbs = null, filter = null, lang = "en", country = "us", proxy = null, sleep_interval = 0, timeout = 5000, ) :Promise { const escaped_term = querystring.escape(term); let proxies = null; @@ -70,7 +71,7 @@ export async function google_search(term: string, advanced = false, num_results const maxAttempts = 20; // Define a maximum number of attempts to prevent infinite loop while (start < num_results && attempts < maxAttempts) { try { - const resp = await _req(escaped_term, num_results - start, lang, start, proxies, timeout, tbs, filter); + const resp = await _req(escaped_term, num_results - start, lang, country, start, proxies, timeout, tbs, filter); const $ = cheerio.load(resp.data); const result_block = $("div.g"); if (result_block.length === 0) { diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts index 5a6a3d8..d3b66aa 100644 --- a/apps/api/src/search/index.ts +++ b/apps/api/src/search/index.ts @@ -9,6 +9,8 @@ export async function search({ tbs = null, filter = null, lang = "en", + country = "us", + location = undefined, proxy = null, sleep_interval = 0, timeout = 5000, @@ -19,13 +21,15 @@ export async function search({ tbs?: string; filter?: string; lang?: string; + country?: string; + location?: string; proxy?: string; sleep_interval?: number; timeout?: number; }) : Promise { try { - if (process.env.SERPER_API_KEY && !tbs) { - return await serper_search(query, num_results); + if (process.env.SERPER_API_KEY ) { + return await serper_search(query, {num_results, tbs, filter, lang, country, location}); } return await google_search( query, @@ -34,6 +38,7 @@ export async function search({ tbs, filter, lang, + country, proxy, sleep_interval, timeout diff --git a/apps/api/src/search/serper.ts b/apps/api/src/search/serper.ts index f8806b7..be71636 100644 --- a/apps/api/src/search/serper.ts +++ b/apps/api/src/search/serper.ts @@ -4,10 +4,23 @@ import { SearchResult } from "../../src/lib/entities"; dotenv.config(); -export async function serper_search(q, num_results): Promise { +export async function serper_search(q, options: { + tbs?: string; + filter?: string; + lang?: string; + country?: string; + location?: string; + num_results: number; + page?: number; +}): Promise { let data = JSON.stringify({ q: q, - num: num_results, + hl: options.lang, + gl: options.country, + location: options.location, + tbs: options.tbs, + num: options.num_results, + page: options.page ?? 1, }); let config = {