diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 820b40c..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 2a7c2a5..cbfb076 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ +.DS_Store /node_modules/ /dist/ .env *.csv dump.rdb /mongo-data -apps/js-sdk/node_modules/ \ No newline at end of file +apps/js-sdk/node_modules/ diff --git a/README.md b/README.md index 101fd6c..b35adfe 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ Crawl and convert any website into LLM-ready markdown. Build by [Mendable.ai](ht ## How to use it? -We provide an easy to use API with our hosted version. You can find the playground and documentation [here](https://firecrawl.com/playground). You can also self host the backend if you'd like. +We provide an easy to use API with our hosted version. You can find the playground and documentation [here](https://firecrawl.dev/playground). You can also self host the backend if you'd like. -- [x] [API](https://firecrawl.com/playground) +- [x] [API](https://firecrawl.dev/playground) - [x] [Python SDK](https://github.com/mendableai/firecrawl/tree/main/apps/python-sdk) - [x] [Langchain Integration 🦜🔗](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/) - [x] [Llama Index Integration 🦙](https://docs.llamaindex.ai/en/stable/) @@ -25,7 +25,7 @@ Self-host. To self-host refer to guide [here](https://github.com/mendableai/fire ### API Key -To use the API, you need to sign up on [Firecrawl](https://firecrawl.com) and get an API key. +To use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and get an API key. ### Crawling diff --git a/apps/.DS_Store b/apps/.DS_Store deleted file mode 100644 index 61e35a2..0000000 Binary files a/apps/.DS_Store and /dev/null differ diff --git a/apps/api/src/.DS_Store b/apps/api/src/.DS_Store deleted file mode 100644 index bbd93bc..0000000 Binary files a/apps/api/src/.DS_Store and /dev/null differ diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 26fb2a9..8cd928b 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -45,14 +45,14 @@ app.get("/test", async (req, res) => { res.send("Hello, world!"); }); -async function authenticateUser(req, res, mode?: string): Promise { +async function authenticateUser(req, res, mode?: string): Promise<{ success: boolean, team_id?: string, error?: string, status?: number }> { const authHeader = req.headers.authorization; if (!authHeader) { - return res.status(401).json({ error: "Unauthorized" }); + return { success: false, error: "Unauthorized", status: 401 }; } const token = authHeader.split(" ")[1]; // Extract the token from "Bearer " if (!token) { - return res.status(401).json({ error: "Unauthorized: Token missing" }); + return { success: false, error: "Unauthorized: Token missing", status: 401 }; } try { @@ -64,13 +64,11 @@ async function authenticateUser(req, res, mode?: string): Promise { ).consume(iptoken); } catch (rateLimiterRes) { console.error(rateLimiterRes); - return res.status(429).json({ - error: "Rate limit exceeded. Too many requests, try again in 1 minute.", - }); + return { success: false, error: "Rate limit exceeded. Too many requests, try again in 1 minute.", status: 429 }; } if (token === "this_is_just_a_preview_token" && mode === "scrape") { - return "preview"; + return { success: true, team_id: "preview" }; } const normalizedApi = parseApi(token); @@ -80,16 +78,19 @@ async function authenticateUser(req, res, mode?: string): Promise { .select("*") .eq("key", normalizedApi); if (error || !data || data.length === 0) { - return res.status(401).json({ error: "Unauthorized: Invalid token" }); + return { success: false, error: "Unauthorized: Invalid token", status: 401 }; } - return data[0].team_id; + return { success: true, team_id: data[0].team_id }; } app.post("/v0/scrape", async (req, res) => { try { // make sure to authenticate user first, Bearer - const team_id = await authenticateUser(req, res, "scrape"); + const { success, team_id, error, status } = await authenticateUser(req, res, "crawl"); + if (!success) { + return res.status(status).json({ error }); + } const crawlerOptions = req.body.crawlerOptions ?? {}; try { @@ -159,7 +160,10 @@ app.post("/v0/scrape", async (req, res) => { app.post("/v0/crawl", async (req, res) => { try { - const team_id = await authenticateUser(req, res); + const { success, team_id, error, status } = await authenticateUser(req, res, "scrape"); + if (!success) { + return res.status(status).json({ error }); + } const { success: creditsCheckSuccess, message: creditsCheckMessage } = await checkTeamCredits(team_id, 1); @@ -251,22 +255,9 @@ app.post("/v0/crawlWebsitePreview", async (req, res) => { app.get("/v0/crawl/status/:jobId", async (req, res) => { try { - const authHeader = req.headers.authorization; - if (!authHeader) { - return res.status(401).json({ error: "Unauthorized" }); - } - const token = authHeader.split(" ")[1]; // Extract the token from "Bearer " - if (!token) { - return res.status(401).json({ error: "Unauthorized: Token missing" }); - } - - // make sure api key is valid, based on the api_keys table in supabase - const { data, error } = await supabase_service - .from("api_keys") - .select("*") - .eq("key", token); - if (error || !data || data.length === 0) { - return res.status(401).json({ error: "Unauthorized: Invalid token" }); + const { success, team_id, error, status } = await authenticateUser(req, res, "scrape"); + if (!success) { + return res.status(status).json({ error }); } const job = await getWebScraperQueue().getJob(req.params.jobId); if (!job) { diff --git a/apps/playwright-service/.DS_Store b/apps/playwright-service/.DS_Store deleted file mode 100644 index 5008ddf..0000000 Binary files a/apps/playwright-service/.DS_Store and /dev/null differ