github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

[Bugfix] Trim and Lowercase all urls

Browse Source
This commit is contained in:
rafaelsideguide 2024-04-17 17:13:30 -03:00
parent ee8a097252
commit 9ab4cb4788
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/index.ts
View File

@ -103,10 +103,11 @@ app.post("/v0/scrape", async (req, res) => {
} }
// authenticate on supabase // authenticate on supabase
const url = req.body.url; let url = req.body.url;
if (!url) { if (!url) {
return res.status(400).json({ error: "Url is required" }); return res.status(400).json({ error: "Url is required" });
} }
url = url.trim().toLowerCase();
try { try {
const a = new WebScraperDataProvider(); const a = new WebScraperDataProvider();
@ -164,10 +165,12 @@ app.post("/v0/crawl", async (req, res) => {
} }
// authenticate on supabase // authenticate on supabase
const url = req.body.url; let url = req.body.url;
if (!url) { if (!url) {
return res.status(400).json({ error: "Url is required" }); return res.status(400).json({ error: "Url is required" });
} }
url = url.trim().toLowerCase();
const mode = req.body.mode ?? "crawl"; const mode = req.body.mode ?? "crawl";
const crawlerOptions = req.body.crawlerOptions ?? {}; const crawlerOptions = req.body.crawlerOptions ?? {};
@ -225,10 +228,11 @@ app.post("/v0/crawlWebsitePreview", async (req, res) => {
} }
// authenticate on supabase // authenticate on supabase
const url = req.body.url; let url = req.body.url;
if (!url) { if (!url) {
return res.status(400).json({ error: "Url is required" }); return res.status(400).json({ error: "Url is required" });
} }
url = url.trim().toLowerCase();
const mode = req.body.mode ?? "crawl"; const mode = req.body.mode ?? "crawl";
const crawlerOptions = req.body.crawlerOptions ?? {}; const crawlerOptions = req.body.crawlerOptions ?? {};
const job = await addWebScraperJob({ const job = await addWebScraperJob({