From 9ab4cb478846e216c3e7bf310a0ec6013af17a71 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Wed, 17 Apr 2024 17:13:30 -0300
Subject: [PATCH] [Bugfix] Trim and Lowercase all urls

---
 apps/api/src/index.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 7198988..437c967 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -103,10 +103,11 @@ app.post("/v0/scrape", async (req, res) => {
     }
 
     // authenticate on supabase
-    const url = req.body.url;
+    let url = req.body.url;
     if (!url) {
       return res.status(400).json({ error: "Url is required" });
     }
+    url = url.trim().toLowerCase();
 
     try {
       const a = new WebScraperDataProvider();
@@ -164,10 +165,12 @@ app.post("/v0/crawl", async (req, res) => {
     }
 
     // authenticate on supabase
-    const url = req.body.url;
+    let url = req.body.url;
     if (!url) {
       return res.status(400).json({ error: "Url is required" });
     }
+
+    url = url.trim().toLowerCase();
     const mode = req.body.mode ?? "crawl";
     const crawlerOptions = req.body.crawlerOptions ?? {};
 
@@ -225,10 +228,11 @@ app.post("/v0/crawlWebsitePreview", async (req, res) => {
     }
 
     // authenticate on supabase
-    const url = req.body.url;
+    let url = req.body.url;
     if (!url) {
       return res.status(400).json({ error: "Url is required" });
     }
+    url = url.trim().toLowerCase();
     const mode = req.body.mode ?? "crawl";
     const crawlerOptions = req.body.crawlerOptions ?? {};
     const job = await addWebScraperJob({