diff --git a/apps/api/src/controllers/search.ts b/apps/api/src/controllers/search.ts new file mode 100644 index 0000000..7cd5209 --- /dev/null +++ b/apps/api/src/controllers/search.ts @@ -0,0 +1,136 @@ +import { Request, Response } from "express"; +import { WebScraperDataProvider } from "../scraper/WebScraper"; +import { billTeam, checkTeamCredits } from "../services/billing/credit_billing"; +import { authenticateUser } from "./auth"; +import { RateLimiterMode } from "../types"; +import { logJob } from "../services/logging/log_job"; +import { PageOptions } from "../lib/entities"; +import { search } from "../search/googlesearch"; + +export async function searchHelper( + req: Request, + team_id: string, + crawlerOptions: any, + pageOptions: PageOptions +): Promise<{ + success: boolean; + error?: string; + data?: any; + returnCode: number; +}> { + const query = req.body.query; + if (!query) { + return { success: false, error: "Query is required", returnCode: 400 }; + } + + const res = await search(query, true, 7); + + let justSearch = pageOptions.fetchPageContent === false; + + if(justSearch){ + return { success: true, data: res, returnCode: 200 }; + } + + if (res.results.length === 0) { + return { success: true, error: "No search results found", returnCode: 200 }; + } + + const a = new WebScraperDataProvider(); + await a.setOptions({ + mode: "single_urls", + urls: res.results.map((r) => r.url), + crawlerOptions: { + ...crawlerOptions, + }, + pageOptions: {...pageOptions, onlyMainContent: pageOptions?.onlyMainContent ?? true, fetchPageContent: pageOptions?.fetchPageContent ?? true, fallback:false}, + }); + + const docs = await a.getDocuments(true); + if (docs.length === 0) + { + return { success: true, error: "No search results found", returnCode: 200 }; + } + + + // make sure doc.content is not empty + const filteredDocs = docs.filter( + (doc: { content?: string }) => doc.content && doc.content.trim().length > 0 + ); + + if (filteredDocs.length === 0) { + return { success: true, error: "No page found", returnCode: 200 }; + } + + const { success, credit_usage } = await billTeam( + team_id, + filteredDocs.length + ); + if (!success) { + return { + success: false, + error: + "Failed to bill team. Insufficient credits or subscription not found.", + returnCode: 402, + }; + } + + return { + success: true, + data: filteredDocs, + returnCode: 200, + }; +} + +export async function searchController(req: Request, res: Response) { + try { + // make sure to authenticate user first, Bearer + const { success, team_id, error, status } = await authenticateUser( + req, + res, + RateLimiterMode.Search + ); + if (!success) { + return res.status(status).json({ error }); + } + const crawlerOptions = req.body.crawlerOptions ?? {}; + const pageOptions = req.body.pageOptions ?? { onlyMainContent: true, fetchPageContent: true, fallback: false}; + const origin = req.body.origin ?? "api"; + + try { + const { success: creditsCheckSuccess, message: creditsCheckMessage } = + await checkTeamCredits(team_id, 1); + if (!creditsCheckSuccess) { + return res.status(402).json({ error: "Insufficient credits" }); + } + } catch (error) { + console.error(error); + return res.status(500).json({ error: "Internal server error" }); + } + const startTime = new Date().getTime(); + const result = await searchHelper( + req, + team_id, + crawlerOptions, + pageOptions + ); + const endTime = new Date().getTime(); + const timeTakenInSeconds = (endTime - startTime) / 1000; + logJob({ + success: result.success, + message: result.error, + num_docs: 1, + docs: [result.data], + time_taken: timeTakenInSeconds, + team_id: team_id, + mode: "search", + url: req.body.url, + crawlerOptions: crawlerOptions, + pageOptions: pageOptions, + origin: origin, + }); + return res.status(result.returnCode).json(result); + } catch (error) { + console.error(error); + return res.status(500).json({ error: error.message }); + } +} diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index e261dd4..07f07e4 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -11,6 +11,8 @@ export interface Progress { export type PageOptions = { onlyMainContent?: boolean; + fallback?: boolean; + fetchPageContent?: boolean; }; export type WebScraperOptions = { urls: string[]; diff --git a/apps/api/src/routes/v0.ts b/apps/api/src/routes/v0.ts index 023282a..f84b974 100644 --- a/apps/api/src/routes/v0.ts +++ b/apps/api/src/routes/v0.ts @@ -4,6 +4,7 @@ import { crawlStatusController } from "../../src/controllers/crawl-status"; import { scrapeController } from "../../src/controllers/scrape"; import { crawlPreviewController } from "../../src/controllers/crawlPreview"; import { crawlJobStatusPreviewController } from "../../src/controllers/status"; +import { searchController } from "../../src/controllers/search"; export const v0Router = express.Router(); @@ -12,3 +13,7 @@ v0Router.post("/v0/crawl", crawlController); v0Router.post("/v0/crawlWebsitePreview", crawlPreviewController); v0Router.get("/v0/crawl/status/:jobId", crawlStatusController); v0Router.get("/v0/checkJobStatus/:jobId", crawlJobStatusPreviewController); + +// Search routes +v0Router.post("/v0/search", searchController); + diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 0f3cc38..fcbb688 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -4,9 +4,7 @@ import { extractMetadata } from "./utils/metadata"; import dotenv from "dotenv"; import { Document, PageOptions } from "../../lib/entities"; import { parseMarkdown } from "../../lib/html-to-markdown"; -import { parseTablesToMarkdown } from "./utils/parseTable"; import { excludeNonMainTags } from "./utils/excludeTags"; -// import puppeteer from "puppeteer"; dotenv.config(); @@ -155,6 +153,15 @@ export async function scrapSingleUrl( // } let [text, html] = await attemptScraping(urlToScrap, "scrapingBee"); + if(pageOptions.fallback === false){ + const soup = cheerio.load(html); + const metadata = extractMetadata(soup, urlToScrap); + return { + content: text, + markdown: text, + metadata: { ...metadata, sourceURL: urlToScrap }, + } as Document; + } if (!text || text.length < 100) { console.log("Falling back to playwright"); [text, html] = await attemptScraping(urlToScrap, "playwright"); diff --git a/apps/api/src/scraper/WebScraper/utils/metadata.ts b/apps/api/src/scraper/WebScraper/utils/metadata.ts index ef883c3..ddaf1e8 100644 --- a/apps/api/src/scraper/WebScraper/utils/metadata.ts +++ b/apps/api/src/scraper/WebScraper/utils/metadata.ts @@ -1,4 +1,3 @@ -// import * as cheerio from 'cheerio'; import { CheerioAPI } from "cheerio"; interface Metadata { title?: string; @@ -8,6 +7,14 @@ interface Metadata { robots?: string; ogTitle?: string; ogDescription?: string; + ogUrl?: string; + ogImage?: string; + ogAudio?: string; + ogDeterminer?: string; + ogLocale?: string; + ogLocaleAlternate?: string[]; + ogSiteName?: string; + ogVideo?: string; dctermsCreated?: string; dcDateCreated?: string; dcDate?: string; @@ -17,7 +24,6 @@ interface Metadata { dctermsSubject?: string; dcSubject?: string; dcDescription?: string; - ogImage?: string; dctermsKeywords?: string; modifiedTime?: string; publishedTime?: string; @@ -33,6 +39,14 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata { let robots: string | null = null; let ogTitle: string | null = null; let ogDescription: string | null = null; + let ogUrl: string | null = null; + let ogImage: string | null = null; + let ogAudio: string | null = null; + let ogDeterminer: string | null = null; + let ogLocale: string | null = null; + let ogLocaleAlternate: string[] | null = null; + let ogSiteName: string | null = null; + let ogVideo: string | null = null; let dctermsCreated: string | null = null; let dcDateCreated: string | null = null; let dcDate: string | null = null; @@ -42,7 +56,6 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata { let dctermsSubject: string | null = null; let dcSubject: string | null = null; let dcDescription: string | null = null; - let ogImage: string | null = null; let dctermsKeywords: string | null = null; let modifiedTime: string | null = null; let publishedTime: string | null = null; @@ -62,11 +75,18 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata { robots = soup('meta[name="robots"]').attr("content") || null; ogTitle = soup('meta[property="og:title"]').attr("content") || null; ogDescription = soup('meta[property="og:description"]').attr("content") || null; + ogUrl = soup('meta[property="og:url"]').attr("content") || null; + ogImage = soup('meta[property="og:image"]').attr("content") || null; + ogAudio = soup('meta[property="og:audio"]').attr("content") || null; + ogDeterminer = soup('meta[property="og:determiner"]').attr("content") || null; + ogLocale = soup('meta[property="og:locale"]').attr("content") || null; + ogLocaleAlternate = soup('meta[property="og:locale:alternate"]').map((i, el) => soup(el).attr("content")).get() || null; + ogSiteName = soup('meta[property="og:site_name"]').attr("content") || null; + ogVideo = soup('meta[property="og:video"]').attr("content") || null; articleSection = soup('meta[name="article:section"]').attr("content") || null; articleTag = soup('meta[name="article:tag"]').attr("content") || null; publishedTime = soup('meta[property="article:published_time"]').attr("content") || null; modifiedTime = soup('meta[property="article:modified_time"]').attr("content") || null; - ogImage = soup('meta[property="og:image"]').attr("content") || null; dctermsKeywords = soup('meta[name="dcterms.keywords"]').attr("content") || null; dcDescription = soup('meta[name="dc.description"]').attr("content") || null; dcSubject = soup('meta[name="dc.subject"]').attr("content") || null; @@ -90,6 +110,14 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata { ...(robots ? { robots } : {}), ...(ogTitle ? { ogTitle } : {}), ...(ogDescription ? { ogDescription } : {}), + ...(ogUrl ? { ogUrl } : {}), + ...(ogImage ? { ogImage } : {}), + ...(ogAudio ? { ogAudio } : {}), + ...(ogDeterminer ? { ogDeterminer } : {}), + ...(ogLocale ? { ogLocale } : {}), + ...(ogLocaleAlternate ? { ogLocaleAlternate } : {}), + ...(ogSiteName ? { ogSiteName } : {}), + ...(ogVideo ? { ogVideo } : {}), ...(dctermsCreated ? { dctermsCreated } : {}), ...(dcDateCreated ? { dcDateCreated } : {}), ...(dcDate ? { dcDate } : {}), @@ -99,7 +127,6 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata { ...(dctermsSubject ? { dctermsSubject } : {}), ...(dcSubject ? { dcSubject } : {}), ...(dcDescription ? { dcDescription } : {}), - ...(ogImage ? { ogImage } : {}), ...(dctermsKeywords ? { dctermsKeywords } : {}), ...(modifiedTime ? { modifiedTime } : {}), ...(publishedTime ? { publishedTime } : {}), diff --git a/apps/api/src/search/googlesearch.ts b/apps/api/src/search/googlesearch.ts new file mode 100644 index 0000000..fd3b645 --- /dev/null +++ b/apps/api/src/search/googlesearch.ts @@ -0,0 +1,134 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; +import * as querystring from 'querystring'; +import { ScrapingBeeClient } from 'scrapingbee'; + +const _useragent_list = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0' +]; + +function get_useragent(): string { + return _useragent_list[Math.floor(Math.random() * _useragent_list.length)]; +} + +async function _req(term: string, results: number, lang: string, start: number, proxies: any, timeout: number) { + const resp = await axios.get("https://www.google.com/search", { + headers: { + "User-Agent": get_useragent() + }, + params: { + "q": term, + "num": results + 2, // Prevents multiple requests + "hl": lang, + }, + proxy: proxies, + timeout: timeout, + }); + return resp; +} + +class SearchResult { + url: string; + title: string; + description: string; + + constructor(url: string, title: string, description: string) { + this.url = url; + this.title = title; + this.description = description; + } + + toString(): string { + return `SearchResult(url=${this.url}, title=${this.title}, description=${this.description})`; + } +} + +export async function search(term: string, advanced = false, num_results = 7, lang = "en", proxy = null, sleep_interval = 0, timeout = 5000) { + const escaped_term = querystring.escape(term); + + let proxies = null; + if (proxy) { + if (proxy.startsWith("https")) { + proxies = {"https": proxy}; + } else { + proxies = {"http": proxy}; + } + } + + // const response = await _req_scraping_bee(escaped_term, num_results, lang); + // const $ = cheerio.load(response); + + // const knowledgeGraphElement = $("div.kno-rdesc"); + // console.log(knowledgeGraphElement); + // console.log(knowledgeGraphElement.html()); + + // let knowledgeGraph = null; + // if (knowledgeGraphElement.length > 0) { + // console.log("Knowledge Graph found"); + // const title = knowledgeGraphElement.find("h2").text(); + // const type = knowledgeGraphElement.find("div[data-attrid='subtitle']").text(); + // const website = knowledgeGraphElement.find("a[data-ved]").attr("href"); + // const imageUrl = knowledgeGraphElement.find("g-img img").attr("src"); + // const description = knowledgeGraphElement.find("div[data-attrid='description'] span").text(); + // const descriptionSource = knowledgeGraphElement.find("div[data-attrid='description'] a").text(); + // const descriptionLink = knowledgeGraphElement.find("div[data-attrid='description'] a").attr("href"); + // const attributes = {}; + // knowledgeGraphElement.find("div[data-attrid='kc:/common:sideways']").each((index, element) => { + // const attributeKey = $(element).find("span[data-attrid]").text(); + // const attributeValue = $(element).find("span[data-log-string]").text(); + // attributes[attributeKey] = attributeValue; + // }); + // knowledgeGraph = { + // "title": title, + // "type": type, + // "website": website, + // "imageUrl": imageUrl, + // "description": description, + // "descriptionSource": descriptionSource, + // "descriptionLink": descriptionLink, + // "attributes": attributes + // }; + // } + + let start = 0; + let results = []; + while (start < num_results) { + const resp = await _req(escaped_term, num_results - start, lang, start, proxies, timeout); + const $ = cheerio.load(resp.data); + const result_block = $("div.g"); + if (result_block.length === 0) { + start += 1; + } + result_block.each((index, element) => { + const linkElement = $(element).find("a"); + const link = linkElement && linkElement.attr("href") ? linkElement.attr("href") : null; + const title = $(element).find("h3"); + const ogImage = $(element).find("img").eq(1).attr("src"); + const description_box = $(element).find("div[style='-webkit-line-clamp:2']"); + const answerBox = $(element).find(".mod").text(); + if (description_box) { + const description = description_box.text(); + if (link && title && description) { + start += 1; + if (advanced) { + results.push(new SearchResult(link, title.text(), description)); + } else { + results.push(link); + } + } + } + }); + await new Promise(resolve => setTimeout(resolve, sleep_interval * 1000)); + + if (start === 0) { + return {results: []}; + } + } + return {results: results}; +} diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 5d778a2..c65140c 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -44,6 +44,8 @@ export enum RateLimiterMode { CrawlStatus = "crawl-status", Scrape = "scrape", Preview = "preview", + Search = "search", + } export interface AuthResponse {