From 40ad97dee80599a8c8c3b22332e878b43bc35d05 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 14 May 2024 18:08:31 -0300 Subject: [PATCH 01/11] added rate limits --- apps/api/.env.example | 2 + apps/api/src/controllers/auth.ts | 94 ++++++++++++++++++++++----- apps/api/src/services/rate-limiter.ts | 46 +++++++++---- 3 files changed, 111 insertions(+), 31 deletions(-) diff --git a/apps/api/.env.example b/apps/api/.env.example index b025326..d91799a 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -27,3 +27,5 @@ SLACK_WEBHOOK_URL= # set if you'd like to send slack server health status messag POSTHOG_API_KEY= # set if you'd like to send posthog events like job logs POSTHOG_HOST= # set if you'd like to send posthog events like job logs +STRIPE_PRICE_ID_STANDARD= +STRIPE_PRICE_ID_SCALE= diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 77aa52f..fb3a813 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -1,9 +1,9 @@ import { parseApi } from "../../src/lib/parseApi"; -import { getRateLimiter } from "../../src/services/rate-limiter"; +import { getRateLimiter, crawlRateLimit, scrapeRateLimit } from "../../src/services/rate-limiter"; import { AuthResponse, RateLimiterMode } from "../../src/types"; import { supabase_service } from "../../src/services/supabase"; import { withAuth } from "../../src/lib/withAuth"; - +import { RateLimiterRedis } from "rate-limiter-flexible"; export async function authenticateUser(req, res, mode?: RateLimiterMode) : Promise { return withAuth(supaAuthenticateUser)(req, res, mode); @@ -19,7 +19,6 @@ export async function supaAuthenticateUser( error?: string; status?: number; }> { - const authHeader = req.headers.authorization; if (!authHeader) { return { success: false, error: "Unauthorized", status: 401 }; @@ -33,13 +32,55 @@ export async function supaAuthenticateUser( }; } + const incomingIP = (req.headers["x-forwarded-for"] || + req.socket.remoteAddress) as string; + const iptoken = incomingIP + token; + + let rateLimiter: RateLimiterRedis; + let subscriptionData: { team_id: string, plan: string } | null = null; + let normalizedApi: string; + + if (token == "this_is_just_a_preview_token") { + rateLimiter = await getRateLimiter(RateLimiterMode.Preview, token); + } else { + normalizedApi = parseApi(token); + + const { data, error } = await supabase_service.rpc( + 'get_key_and_price_id', { api_key: normalizedApi }); + + if (error) { + console.error('Error fetching key and price_id:', error); + } else { + console.log('Key and Price ID:', data); + } + + if (error || !data || data.length === 0) { + return { + success: false, + error: "Unauthorized: Invalid token", + status: 401, + }; + } + + subscriptionData = { + team_id: data[0].team_id, + plan: getPlanByPriceId(data[0].price_id) + } + switch (mode) { + case RateLimiterMode.Crawl: + rateLimiter = crawlRateLimit(subscriptionData.plan); + break; + case RateLimiterMode.Scrape: + rateLimiter = scrapeRateLimit(subscriptionData.plan); + break; + // case RateLimiterMode.Search: + // rateLimiter = await searchRateLimiter(RateLimiterMode.Search, token); + // break; + } + } + try { - const incomingIP = (req.headers["x-forwarded-for"] || - req.socket.remoteAddress) as string; - const iptoken = incomingIP + token; - await getRateLimiter( - token === "this_is_just_a_preview_token" ? RateLimiterMode.Preview : mode, token - ).consume(iptoken); + rateLimiter.consume(iptoken); } catch (rateLimiterRes) { console.error(rateLimiterRes); return { @@ -66,19 +107,36 @@ export async function supaAuthenticateUser( // return { success: false, error: "Unauthorized: Invalid token", status: 401 }; } - const normalizedApi = parseApi(token); // make sure api key is valid, based on the api_keys table in supabase - const { data, error } = await supabase_service + if (!subscriptionData) { + normalizedApi = parseApi(token); + + const { data, error } = await supabase_service .from("api_keys") .select("*") .eq("key", normalizedApi); - if (error || !data || data.length === 0) { - return { - success: false, - error: "Unauthorized: Invalid token", - status: 401, - }; + + if (error || !data || data.length === 0) { + return { + success: false, + error: "Unauthorized: Invalid token", + status: 401, + }; + } + + subscriptionData = data[0]; } - return { success: true, team_id: data[0].team_id }; + return { success: true, team_id: subscriptionData.team_id }; } + +function getPlanByPriceId(price_id: string) { + switch (price_id) { + case process.env.STRIPE_PRICE_ID_STANDARD: + return 'standard'; + case process.env.STRIPE_PRICE_ID_SCALE: + return 'scale'; + default: + return 'starter'; + } +} \ No newline at end of file diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 5bc9acb..c20f67a 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -2,18 +2,18 @@ import { RateLimiterRedis } from "rate-limiter-flexible"; import * as redis from "redis"; import { RateLimiterMode } from "../../src/types"; -const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5; const MAX_CRAWLS_PER_MINUTE_STARTER = 2; const MAX_CRAWLS_PER_MINUTE_STANDARD = 4; const MAX_CRAWLS_PER_MINUTE_SCALE = 20; +const MAX_SCRAPES_PER_MINUTE_STARTER = 10; +const MAX_SCRAPES_PER_MINUTE_STANDARD = 15; +const MAX_SCRAPES_PER_MINUTE_SCALE = 30; + +const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5; const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 20; - const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 120; - - - export const redisClient = redis.createClient({ url: process.env.REDIS_URL, legacyMode: true, @@ -48,15 +48,15 @@ export const testSuiteRateLimiter = new RateLimiterRedis({ }); -export function crawlRateLimit(plan: string){ - if(plan === "standard"){ +export function crawlRateLimit (plan: string){ + if (plan === "standard"){ return new RateLimiterRedis({ storeClient: redisClient, keyPrefix: "middleware", points: MAX_CRAWLS_PER_MINUTE_STANDARD, duration: 60, // Duration in seconds }); - }else if(plan === "scale"){ + } else if (plan === "scale"){ return new RateLimiterRedis({ storeClient: redisClient, keyPrefix: "middleware", @@ -70,18 +70,38 @@ export function crawlRateLimit(plan: string){ points: MAX_CRAWLS_PER_MINUTE_STARTER, duration: 60, // Duration in seconds }); - } - - +export function scrapeRateLimit (plan: string){ + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "middleware", + points: MAX_SCRAPES_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "middleware", + points: MAX_SCRAPES_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "middleware", + points: MAX_SCRAPES_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); +} export function getRateLimiter(mode: RateLimiterMode, token: string){ // Special test suite case. TODO: Change this later. - if(token.includes("5089cefa58")){ + if (token.includes("5089cefa58")){ return testSuiteRateLimiter; } - switch(mode) { + switch (mode) { case RateLimiterMode.Preview: return previewRateLimiter; case RateLimiterMode.CrawlStatus: From 4761ea510b1dc3deec56c842ba0c787f80e8a265 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 14 May 2024 14:26:42 -0700 Subject: [PATCH 02/11] Update rate-limiter.ts --- apps/api/src/services/rate-limiter.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index c20f67a..6139702 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -21,28 +21,28 @@ export const redisClient = redis.createClient({ export const previewRateLimiter = new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "preview", points: MAX_REQUESTS_PER_MINUTE_PREVIEW, duration: 60, // Duration in seconds }); export const serverRateLimiter = new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "server", points: MAX_REQUESTS_PER_MINUTE_ACCOUNT, duration: 60, // Duration in seconds }); export const crawlStatusRateLimiter = new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "crawl-status", points: MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS, duration: 60, // Duration in seconds }); export const testSuiteRateLimiter = new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "test-suite", points: 1000, duration: 60, // Duration in seconds }); @@ -52,21 +52,21 @@ export function crawlRateLimit (plan: string){ if (plan === "standard"){ return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "crawl-standard", points: MAX_CRAWLS_PER_MINUTE_STANDARD, duration: 60, // Duration in seconds }); } else if (plan === "scale"){ return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "crawl-scale", points: MAX_CRAWLS_PER_MINUTE_SCALE, duration: 60, // Duration in seconds }); } return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "crawl-starter", points: MAX_CRAWLS_PER_MINUTE_STARTER, duration: 60, // Duration in seconds }); @@ -76,21 +76,21 @@ export function scrapeRateLimit (plan: string){ if (plan === "standard"){ return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "scrape-standard", points: MAX_SCRAPES_PER_MINUTE_STANDARD, duration: 60, // Duration in seconds }); } else if (plan === "scale"){ return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "scrape-scale", points: MAX_SCRAPES_PER_MINUTE_SCALE, duration: 60, // Duration in seconds }); } return new RateLimiterRedis({ storeClient: redisClient, - keyPrefix: "middleware", + keyPrefix: "scrape-starter", points: MAX_SCRAPES_PER_MINUTE_STARTER, duration: 60, // Duration in seconds }); From 672eddb999270676854e890798f6931411d6be04 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 14 May 2024 18:47:21 -0300 Subject: [PATCH 03/11] updated rpc --- apps/api/src/controllers/auth.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index fb3a813..74c62b8 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -46,7 +46,8 @@ export async function supaAuthenticateUser( normalizedApi = parseApi(token); const { data, error } = await supabase_service.rpc( - 'get_key_and_price_id', { api_key: normalizedApi }); + 'get_key_and_price_id_2', { api_key: normalizedApi } + ); if (error) { console.error('Error fetching key and price_id:', error); From 0e0faa28b3d2b6ce4db5229d526a278d3dbf7a6f Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 14 May 2024 14:47:36 -0700 Subject: [PATCH 04/11] Update auth.ts --- apps/api/src/controllers/auth.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index fb3a813..3f343cb 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -45,8 +45,10 @@ export async function supaAuthenticateUser( } else { normalizedApi = parseApi(token); + console.log('normalizedApi:', normalizedApi); + const { data, error } = await supabase_service.rpc( - 'get_key_and_price_id', { api_key: normalizedApi }); + 'get_key_and_price_id_2', { api_key: normalizedApi }); if (error) { console.error('Error fetching key and price_id:', error); @@ -73,6 +75,12 @@ export async function supaAuthenticateUser( case RateLimiterMode.Scrape: rateLimiter = scrapeRateLimit(subscriptionData.plan); break; + case RateLimiterMode.CrawlStatus: + rateLimiter = await getRateLimiter(RateLimiterMode.CrawlStatus, token); + break; + default: + rateLimiter = await getRateLimiter(RateLimiterMode.Crawl, token); + break; // case RateLimiterMode.Search: // rateLimiter = await searchRateLimiter(RateLimiterMode.Search, token); // break; From 47c20c80ab693f5307ef67f54184dd64bab8ee18 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 15 May 2024 08:34:49 -0300 Subject: [PATCH 05/11] Update auth.ts --- apps/api/src/controllers/auth.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 43be489..aff628c 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -75,10 +75,10 @@ export async function supaAuthenticateUser( rateLimiter = scrapeRateLimit(subscriptionData.plan); break; case RateLimiterMode.CrawlStatus: - rateLimiter = await getRateLimiter(RateLimiterMode.CrawlStatus, token); + rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); break; default: - rateLimiter = await getRateLimiter(RateLimiterMode.Crawl, token); + rateLimiter = getRateLimiter(RateLimiterMode.Crawl, token); break; // case RateLimiterMode.Search: // rateLimiter = await searchRateLimiter(RateLimiterMode.Search, token); From d4574851becc714b1bc4ea7aac7c2686ff274623 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Wed, 15 May 2024 08:40:21 -0300 Subject: [PATCH 06/11] Added rpc definition --- apps/api/src/controllers/auth.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index aff628c..524f440 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -48,6 +48,22 @@ export async function supaAuthenticateUser( const { data, error } = await supabase_service.rpc( 'get_key_and_price_id_2', { api_key: normalizedApi } ); + // get_key_and_price_id_2 rpc definition: + // create or replace function get_key_and_price_id_2(api_key uuid) + // returns table(key uuid, team_id uuid, price_id text) as $$ + // begin + // if api_key is null then + // return query + // select null::uuid as key, null::uuid as team_id, null::text as price_id; + // end if; + + // return query + // select ak.key, ak.team_id, s.price_id + // from api_keys ak + // left join subscriptions s on ak.team_id = s.team_id + // where ak.key = api_key; + // end; + // $$ language plpgsql; if (error) { console.error('Error fetching key and price_id:', error); From 54049be539986cac01207aab07785e1e26e9cb76 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Fri, 17 May 2024 15:37:47 -0300 Subject: [PATCH 07/11] Added e2e tests --- apps/api/.env.example | 2 + .../src/__tests__/e2e_withAuth/index.test.ts | 61 +++++++++++++++++++ apps/api/src/controllers/auth.ts | 6 +- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/apps/api/.env.example b/apps/api/.env.example index d91799a..1ba5ffe 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -16,6 +16,8 @@ SUPABASE_SERVICE_TOKEN= # Other Optionals TEST_API_KEY= # use if you've set up authentication and want to test with a real API key +RATE_LIMIT_TEST_API_KEY_SCRAPE= # set if you'd like to test the scraping rate limit +RATE_LIMIT_TEST_API_KEY_CRAWL= # set if you'd like to test the crawling rate limit SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.) BULL_AUTH_KEY= # diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 0e2caeb..352d762 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -518,4 +518,65 @@ describe("E2E Tests for API Routes", () => { expect(response.body).toHaveProperty("isProduction"); }); }); + + describe("Rate Limiter", () => { + it("should return 429 when rate limit is exceeded for preview token", async () => { + for (let i = 0; i < 5; i++) { + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer this_is_just_a_preview_token`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); + }); + + it("should return 429 when rate limit is exceeded for API key", async () => { + for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) { + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + + const response = await request(TEST_URL) + .post("/v0/scrape") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); + + it("should return 429 when rate limit is exceeded for API key", async () => { + for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) { + const response = await request(TEST_URL) + .post("/v0/crawl") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(200); + } + + const response = await request(TEST_URL) + .post("/v0/crawl") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send({ url: "https://firecrawl.dev" }); + + expect(response.statusCode).toBe(429); + }, 60000); }); diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 524f440..ff751ef 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -41,8 +41,8 @@ export async function supaAuthenticateUser( let normalizedApi: string; if (token == "this_is_just_a_preview_token") { - rateLimiter = await getRateLimiter(RateLimiterMode.Preview, token); - } else { + rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); + } else { normalizedApi = parseApi(token); const { data, error } = await supabase_service.rpc( @@ -103,7 +103,7 @@ export async function supaAuthenticateUser( } try { - rateLimiter.consume(iptoken); + await rateLimiter.consume(iptoken); } catch (rateLimiterRes) { console.error(rateLimiterRes); return { From a480595aa76185b9ef7f1cf749ca19a8421815cc Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Fri, 17 May 2024 15:41:27 -0300 Subject: [PATCH 08/11] Update index.test.ts --- apps/api/src/__tests__/e2e_withAuth/index.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 352d762..fcf3284 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -526,7 +526,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/scrape") .set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(200); } @@ -534,7 +534,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/scrape") .set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(429); }, 60000); @@ -546,7 +546,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/scrape") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(200); } @@ -555,7 +555,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/scrape") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(429); }, 60000); @@ -566,7 +566,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/crawl") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(200); } @@ -575,7 +575,7 @@ describe("E2E Tests for API Routes", () => { .post("/v0/crawl") .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) .set("Content-Type", "application/json") - .send({ url: "https://firecrawl.dev" }); + .send({ url: "https://www.scrapethissite.com" }); expect(response.statusCode).toBe(429); }, 60000); From 614c073af06095157fc48c8e03a0eb3bcbc3f673 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 19 May 2024 12:45:46 -0700 Subject: [PATCH 09/11] Nick: improvements --- apps/api/src/controllers/auth.ts | 15 +++- apps/api/src/services/rate-limiter.ts | 120 +++++++++++++++----------- 2 files changed, 82 insertions(+), 53 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index ff751ef..4009d69 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -1,5 +1,5 @@ import { parseApi } from "../../src/lib/parseApi"; -import { getRateLimiter, crawlRateLimit, scrapeRateLimit } from "../../src/services/rate-limiter"; +import { getRateLimiter, } from "../../src/services/rate-limiter"; import { AuthResponse, RateLimiterMode } from "../../src/types"; import { supabase_service } from "../../src/services/supabase"; import { withAuth } from "../../src/lib/withAuth"; @@ -68,7 +68,7 @@ export async function supaAuthenticateUser( if (error) { console.error('Error fetching key and price_id:', error); } else { - console.log('Key and Price ID:', data); + // console.log('Key and Price ID:', data); } if (error || !data || data.length === 0) { @@ -79,20 +79,27 @@ export async function supaAuthenticateUser( }; } + subscriptionData = { team_id: data[0].team_id, plan: getPlanByPriceId(data[0].price_id) } switch (mode) { case RateLimiterMode.Crawl: - rateLimiter = crawlRateLimit(subscriptionData.plan); + rateLimiter = getRateLimiter(RateLimiterMode.Crawl, token, subscriptionData.plan); break; case RateLimiterMode.Scrape: - rateLimiter = scrapeRateLimit(subscriptionData.plan); + rateLimiter = getRateLimiter(RateLimiterMode.Scrape, token, subscriptionData.plan); break; case RateLimiterMode.CrawlStatus: rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); break; + case RateLimiterMode.Search: + rateLimiter = getRateLimiter(RateLimiterMode.Search, token); + break; + case RateLimiterMode.Preview: + rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); + break; default: rateLimiter = getRateLimiter(RateLimiterMode.Crawl, token); break; diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 0c42fc5..d4834a1 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -10,6 +10,10 @@ const MAX_SCRAPES_PER_MINUTE_STARTER = 10; const MAX_SCRAPES_PER_MINUTE_STANDARD = 15; const MAX_SCRAPES_PER_MINUTE_SCALE = 30; +const MAX_SEARCHES_PER_MINUTE_STARTER = 10; +const MAX_SEARCHES_PER_MINUTE_STANDARD = 15; +const MAX_SEARCHES_PER_MINUTE_SCALE = 30; + const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5; const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 20; const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 120; @@ -48,55 +52,7 @@ export const testSuiteRateLimiter = new RateLimiterRedis({ }); -export function crawlRateLimit (plan: string){ - if (plan === "standard"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-standard", - points: MAX_CRAWLS_PER_MINUTE_STANDARD, - duration: 60, // Duration in seconds - }); - } else if (plan === "scale"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-scale", - points: MAX_CRAWLS_PER_MINUTE_SCALE, - duration: 60, // Duration in seconds - }); - } - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-starter", - points: MAX_CRAWLS_PER_MINUTE_STARTER, - duration: 60, // Duration in seconds - }); -} - -export function scrapeRateLimit (plan: string){ - if (plan === "standard"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-standard", - points: MAX_SCRAPES_PER_MINUTE_STANDARD, - duration: 60, // Duration in seconds - }); - } else if (plan === "scale"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-scale", - points: MAX_SCRAPES_PER_MINUTE_SCALE, - duration: 60, // Duration in seconds - }); - } - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-starter", - points: MAX_SCRAPES_PER_MINUTE_STARTER, - duration: 60, // Duration in seconds - }); -} - -export function getRateLimiter(mode: RateLimiterMode, token: string){ +export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string){ // Special test suite case. TODO: Change this later. if (token.includes("5089cefa58")){ return testSuiteRateLimiter; @@ -106,6 +62,72 @@ export function getRateLimiter(mode: RateLimiterMode, token: string){ return previewRateLimiter; case RateLimiterMode.CrawlStatus: return crawlStatusRateLimiter; + case RateLimiterMode.Crawl: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-standard", + points: MAX_CRAWLS_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-scale", + points: MAX_CRAWLS_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-starter", + points: MAX_CRAWLS_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); + case RateLimiterMode.Scrape: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-standard", + points: MAX_SCRAPES_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-scale", + points: MAX_SCRAPES_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-starter", + points: MAX_SCRAPES_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); + case RateLimiterMode.Search: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-standard", + points: MAX_SEARCHES_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-scale", + points: MAX_SEARCHES_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-starter", + points: MAX_SEARCHES_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); default: return serverRateLimiter; } From 18fa15df25da6b16900586ab63983eb0e39e0176 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 19 May 2024 12:50:06 -0700 Subject: [PATCH 10/11] Update index.test.ts --- .../src/__tests__/e2e_withAuth/index.test.ts | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts index 1078cb6..e9082ca 100644 --- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts +++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts @@ -977,43 +977,43 @@ describe("E2E Tests for API Routes", () => { }, 60000); }); - it("should return 429 when rate limit is exceeded for API key", async () => { - for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) { - const response = await request(TEST_URL) - .post("/v0/scrape") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ url: "https://www.scrapethissite.com" }); + // it("should return 429 when rate limit is exceeded for API key", async () => { + // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) { + // const response = await request(TEST_URL) + // .post("/v0/scrape") + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + // .set("Content-Type", "application/json") + // .send({ url: "https://www.scrapethissite.com" }); - expect(response.statusCode).toBe(200); - } + // expect(response.statusCode).toBe(200); + // } - const response = await request(TEST_URL) - .post("/v0/scrape") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ url: "https://www.scrapethissite.com" }); + // const response = await request(TEST_URL) + // .post("/v0/scrape") + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + // .set("Content-Type", "application/json") + // .send({ url: "https://www.scrapethissite.com" }); - expect(response.statusCode).toBe(429); - }, 60000); + // expect(response.statusCode).toBe(429); + // }, 60000); - it("should return 429 when rate limit is exceeded for API key", async () => { - for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) { - const response = await request(TEST_URL) - .post("/v0/crawl") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ url: "https://www.scrapethissite.com" }); + // it("should return 429 when rate limit is exceeded for API key", async () => { + // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) { + // const response = await request(TEST_URL) + // .post("/v0/crawl") + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + // .set("Content-Type", "application/json") + // .send({ url: "https://www.scrapethissite.com" }); - expect(response.statusCode).toBe(200); - } + // expect(response.statusCode).toBe(200); + // } - const response = await request(TEST_URL) - .post("/v0/crawl") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ url: "https://www.scrapethissite.com" }); + // const response = await request(TEST_URL) + // .post("/v0/crawl") + // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + // .set("Content-Type", "application/json") + // .send({ url: "https://www.scrapethissite.com" }); - expect(response.statusCode).toBe(429); - }, 60000); + // expect(response.statusCode).toBe(429); + // }, 60000); }); From 98a39b39ab5cd62affa8caab3f079d81b9f23dab Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 19 May 2024 12:59:29 -0700 Subject: [PATCH 11/11] Nick: increased rate limits --- apps/api/src/services/rate-limiter.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index d4834a1..29b14f8 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -2,21 +2,21 @@ import { RateLimiterRedis } from "rate-limiter-flexible"; import * as redis from "redis"; import { RateLimiterMode } from "../../src/types"; -const MAX_CRAWLS_PER_MINUTE_STARTER = 2; -const MAX_CRAWLS_PER_MINUTE_STANDARD = 4; +const MAX_CRAWLS_PER_MINUTE_STARTER = 3; +const MAX_CRAWLS_PER_MINUTE_STANDARD = 5; const MAX_CRAWLS_PER_MINUTE_SCALE = 20; -const MAX_SCRAPES_PER_MINUTE_STARTER = 10; -const MAX_SCRAPES_PER_MINUTE_STANDARD = 15; -const MAX_SCRAPES_PER_MINUTE_SCALE = 30; +const MAX_SCRAPES_PER_MINUTE_STARTER = 20; +const MAX_SCRAPES_PER_MINUTE_STANDARD = 30; +const MAX_SCRAPES_PER_MINUTE_SCALE = 50; -const MAX_SEARCHES_PER_MINUTE_STARTER = 10; -const MAX_SEARCHES_PER_MINUTE_STANDARD = 15; -const MAX_SEARCHES_PER_MINUTE_SCALE = 30; +const MAX_SEARCHES_PER_MINUTE_STARTER = 20; +const MAX_SEARCHES_PER_MINUTE_STANDARD = 30; +const MAX_SEARCHES_PER_MINUTE_SCALE = 50; const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5; const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 20; -const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 120; +const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 150; export const redisClient = redis.createClient({ url: process.env.REDIS_URL,