From 614c073af06095157fc48c8e03a0eb3bcbc3f673 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 19 May 2024 12:45:46 -0700 Subject: [PATCH] Nick: improvements --- apps/api/src/controllers/auth.ts | 15 +++- apps/api/src/services/rate-limiter.ts | 120 +++++++++++++++----------- 2 files changed, 82 insertions(+), 53 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index ff751ef..4009d69 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -1,5 +1,5 @@ import { parseApi } from "../../src/lib/parseApi"; -import { getRateLimiter, crawlRateLimit, scrapeRateLimit } from "../../src/services/rate-limiter"; +import { getRateLimiter, } from "../../src/services/rate-limiter"; import { AuthResponse, RateLimiterMode } from "../../src/types"; import { supabase_service } from "../../src/services/supabase"; import { withAuth } from "../../src/lib/withAuth"; @@ -68,7 +68,7 @@ export async function supaAuthenticateUser( if (error) { console.error('Error fetching key and price_id:', error); } else { - console.log('Key and Price ID:', data); + // console.log('Key and Price ID:', data); } if (error || !data || data.length === 0) { @@ -79,20 +79,27 @@ export async function supaAuthenticateUser( }; } + subscriptionData = { team_id: data[0].team_id, plan: getPlanByPriceId(data[0].price_id) } switch (mode) { case RateLimiterMode.Crawl: - rateLimiter = crawlRateLimit(subscriptionData.plan); + rateLimiter = getRateLimiter(RateLimiterMode.Crawl, token, subscriptionData.plan); break; case RateLimiterMode.Scrape: - rateLimiter = scrapeRateLimit(subscriptionData.plan); + rateLimiter = getRateLimiter(RateLimiterMode.Scrape, token, subscriptionData.plan); break; case RateLimiterMode.CrawlStatus: rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); break; + case RateLimiterMode.Search: + rateLimiter = getRateLimiter(RateLimiterMode.Search, token); + break; + case RateLimiterMode.Preview: + rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); + break; default: rateLimiter = getRateLimiter(RateLimiterMode.Crawl, token); break; diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index 0c42fc5..d4834a1 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -10,6 +10,10 @@ const MAX_SCRAPES_PER_MINUTE_STARTER = 10; const MAX_SCRAPES_PER_MINUTE_STANDARD = 15; const MAX_SCRAPES_PER_MINUTE_SCALE = 30; +const MAX_SEARCHES_PER_MINUTE_STARTER = 10; +const MAX_SEARCHES_PER_MINUTE_STANDARD = 15; +const MAX_SEARCHES_PER_MINUTE_SCALE = 30; + const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5; const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 20; const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 120; @@ -48,55 +52,7 @@ export const testSuiteRateLimiter = new RateLimiterRedis({ }); -export function crawlRateLimit (plan: string){ - if (plan === "standard"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-standard", - points: MAX_CRAWLS_PER_MINUTE_STANDARD, - duration: 60, // Duration in seconds - }); - } else if (plan === "scale"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-scale", - points: MAX_CRAWLS_PER_MINUTE_SCALE, - duration: 60, // Duration in seconds - }); - } - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "crawl-starter", - points: MAX_CRAWLS_PER_MINUTE_STARTER, - duration: 60, // Duration in seconds - }); -} - -export function scrapeRateLimit (plan: string){ - if (plan === "standard"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-standard", - points: MAX_SCRAPES_PER_MINUTE_STANDARD, - duration: 60, // Duration in seconds - }); - } else if (plan === "scale"){ - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-scale", - points: MAX_SCRAPES_PER_MINUTE_SCALE, - duration: 60, // Duration in seconds - }); - } - return new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: "scrape-starter", - points: MAX_SCRAPES_PER_MINUTE_STARTER, - duration: 60, // Duration in seconds - }); -} - -export function getRateLimiter(mode: RateLimiterMode, token: string){ +export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string){ // Special test suite case. TODO: Change this later. if (token.includes("5089cefa58")){ return testSuiteRateLimiter; @@ -106,6 +62,72 @@ export function getRateLimiter(mode: RateLimiterMode, token: string){ return previewRateLimiter; case RateLimiterMode.CrawlStatus: return crawlStatusRateLimiter; + case RateLimiterMode.Crawl: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-standard", + points: MAX_CRAWLS_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-scale", + points: MAX_CRAWLS_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "crawl-starter", + points: MAX_CRAWLS_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); + case RateLimiterMode.Scrape: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-standard", + points: MAX_SCRAPES_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-scale", + points: MAX_SCRAPES_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "scrape-starter", + points: MAX_SCRAPES_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); + case RateLimiterMode.Search: + if (plan === "standard"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-standard", + points: MAX_SEARCHES_PER_MINUTE_STANDARD, + duration: 60, // Duration in seconds + }); + } else if (plan === "scale"){ + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-scale", + points: MAX_SEARCHES_PER_MINUTE_SCALE, + duration: 60, // Duration in seconds + }); + } + return new RateLimiterRedis({ + storeClient: redisClient, + keyPrefix: "search-starter", + points: MAX_SEARCHES_PER_MINUTE_STARTER, + duration: 60, // Duration in seconds + }); default: return serverRateLimiter; }