From d8ee4e90d6e64eff8cb5bb0ab557360e08dcba75 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 28 Apr 2024 11:47:25 -0700 Subject: [PATCH 1/2] Update website_params.ts --- .../WebScraper/utils/custom/website_params.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts index 164b074..dd9f20e 100644 --- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts +++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts @@ -21,4 +21,22 @@ export const urlSpecificParams = { "mC1On8P2GWT3A5UeSYH6z_MP94xcTAdZ5jfNi9IT2U0-1714327136-1.0.1.1-ILAP5pSX_Oo9PPo2iHEYCYX.p9a0yRBNLr58GHyrzYNDJ537xYpG50MXxUYVdfrD.h3FV5O7oMlRKGA0scbxaQ", }, }, + "support.greenpay.me":{ + params: { + wait_browser: "networkidle2", + block_resources: false, + }, + headers: { + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "sec-fetch-site": "same-origin", + "sec-fetch-mode": "cors", + "sec-fetch-dest": "empty", + referer: "https://www.google.com/", + "accept-language": "en-US,en;q=0.9", + "accept-encoding": "gzip, deflate, br", + accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + }, + } }; From 68838c9e0da8c74f9921e4d89e459275f9d235ce Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 28 Apr 2024 12:44:00 -0700 Subject: [PATCH 2/2] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 262a90c..ff73e95 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -20,10 +20,15 @@ export async function generateRequestParams( headers: { "ScrapingService-Request": "TRUE" }, }; - const urlKey = new URL(url).hostname; - if (urlSpecificParams.hasOwnProperty(urlKey)) { - return { ...defaultParams, ...urlSpecificParams[urlKey] }; - } else { + try { + const urlKey = new URL(url).hostname; + if (urlSpecificParams.hasOwnProperty(urlKey)) { + return { ...defaultParams, ...urlSpecificParams[urlKey] }; + } else { + return defaultParams; + } + } catch (error) { + console.error(`Error generating URL key: ${error}`); return defaultParams; } }