Nick: new limits, new pricing

apps/api/.env.example

@@ -31,6 +31,13 @@ POSTHOG_HOST= # set if you'd like to send posthog events like job logs
 
 STRIPE_PRICE_ID_STANDARD=
 STRIPE_PRICE_ID_SCALE=
+STRIPE_PRICE_ID_STARTER=
+STRIPE_PRICE_ID_HOBBY=
+STRIPE_PRICE_ID_HOBBY_YEARLY=
+STRIPE_PRICE_ID_STANDARD_NEW=
+STRIPE_PRICE_ID_STANDARD_NEW_YEARLY=
+STRIPE_PRICE_ID_GROWTH=
+STRIPE_PRICE_ID_GROWTH_YEARLY=
 
 HYPERDX_API_KEY=
 HDX_NODE_BETA_MODE=1

apps/api/src/controllers/auth.ts

@@ -177,11 +177,19 @@ export async function supaAuthenticateUser(
 
 function getPlanByPriceId(price_id: string) {
   switch (price_id) {
+    case process.env.STRIPE_PRICE_ID_STARTER:
+      return 'starter';
     case process.env.STRIPE_PRICE_ID_STANDARD:
       return 'standard';
     case process.env.STRIPE_PRICE_ID_SCALE:
       return 'scale';
+    case process.env.STRIPE_PRICE_ID_HOBBY || process.env.STRIPE_PRICE_ID_HOBBY_YEARLY:
+      return 'hobby';
+    case process.env.STRIPE_PRICE_ID_STANDARD_NEW || process.env.STRIPE_PRICE_ID_STANDARD_NEW_YEARLY:
+      return 'standard-new';
+    case process.env.STRIPE_PRICE_ID_GROWTH || process.env.STRIPE_PRICE_ID_GROWTH_YEARLY:
+      return 'growth';
     default:
-      return 'starter';
+      return 'free';
   }
 }

apps/api/src/controllers/scrape.ts

@@ -64,7 +64,7 @@ export async function scrapeHelper(
   }
 
   let creditsToBeBilled = filteredDocs.length;
-  const creditsPerLLMExtract = 5;
+  const creditsPerLLMExtract = 50;
 
   if (extractorOptions.mode === "llm-extraction") {
     creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);

apps/api/src/index.ts

@@ -168,3 +168,6 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
 app.get("/is-production", (req, res) => {
   res.send({ isProduction: global.isProduction });
 });
+
+
+// /workers health check, cant act as load balancer, just has to be a pre deploy thing

apps/api/src/services/billing/credit_billing.ts

@@ -1,7 +1,7 @@
 import { withAuth } from "../../lib/withAuth";
 import { supabase_service } from "../supabase";
 
-const FREE_CREDITS = 300;
+const FREE_CREDITS = 500;
 
 export async function billTeam(team_id: string, credits: number) {
   return withAuth(supaBillTeam)(team_id, credits);

apps/api/src/services/rate-limiter.ts

@@ -2,133 +2,67 @@ import { RateLimiterRedis } from "rate-limiter-flexible";
 import * as redis from "redis";
 import { RateLimiterMode } from "../../src/types";
 
-const MAX_CRAWLS_PER_MINUTE_STARTER = 3;
+const RATE_LIMITS = {
-const MAX_CRAWLS_PER_MINUTE_STANDARD = 5;
+  crawl: {
-const MAX_CRAWLS_PER_MINUTE_SCALE = 20;
+    free: 1,
-
+    starter: 3,
-const MAX_SCRAPES_PER_MINUTE_STARTER = 20;
+    standard: 5,
-const MAX_SCRAPES_PER_MINUTE_STANDARD = 40;
+    scale: 20,
-const MAX_SCRAPES_PER_MINUTE_SCALE = 50;
+    hobby: 3,
-
+    standardNew: 10,
-const MAX_SEARCHES_PER_MINUTE_STARTER = 20;
+    growth: 50,
-const MAX_SEARCHES_PER_MINUTE_STANDARD = 40;
+  },
-const MAX_SEARCHES_PER_MINUTE_SCALE = 50;
+  scrape: {
-
+    free: 5,
-const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5;
+    starter: 20,
-const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 20;
+    standardOld: 40,
-const MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS = 150;
+    scale: 50,
+    hobby: 10,
+    standardNew: 50,
+    growth: 500,
+  },
+  search: {
+    free: 5,
+    starter: 20,
+    standard: 40,
+    scale: 50,
+    hobby: 10,
+    standardNew: 50,
+    growth: 500,
+  },
+  preview: 5,
+  account: 20,
+  crawlStatus: 150,
+  testSuite: 10000,
+};
 
 export const redisClient = redis.createClient({
   url: process.env.REDIS_URL,
   legacyMode: true,
 });
 
-export const previewRateLimiter = new RateLimiterRedis({
+const createRateLimiter = (keyPrefix, points) => new RateLimiterRedis({
   storeClient: redisClient,
-  keyPrefix: "preview",
+  keyPrefix,
-  points: MAX_REQUESTS_PER_MINUTE_PREVIEW,
+  points,
   duration: 60, // Duration in seconds
 });
 
-export const serverRateLimiter = new RateLimiterRedis({
+export const previewRateLimiter = createRateLimiter("preview", RATE_LIMITS.preview);
-  storeClient: redisClient,
+export const serverRateLimiter = createRateLimiter("server", RATE_LIMITS.account);
-  keyPrefix: "server",
+export const crawlStatusRateLimiter = createRateLimiter("crawl-status", RATE_LIMITS.crawlStatus);
-  points: MAX_REQUESTS_PER_MINUTE_ACCOUNT,
+export const testSuiteRateLimiter = createRateLimiter("test-suite", RATE_LIMITS.testSuite);
-  duration: 60, // Duration in seconds
-});
 
-export const crawlStatusRateLimiter = new RateLimiterRedis({
+export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string) {
-  storeClient: redisClient,
+  if (token.includes("5089cefa58") || token.includes("6254cf9")) {
-  keyPrefix: "crawl-status",
-  points: MAX_REQUESTS_PER_MINUTE_CRAWL_STATUS,
-  duration: 60, // Duration in seconds
-});
-
-export const testSuiteRateLimiter = new RateLimiterRedis({
-  storeClient: redisClient,
-  keyPrefix: "test-suite",
-  points: 10000,
-  duration: 60, // Duration in seconds
-});
-
-
-export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string){
-  // Special test suite case. TODO: Change this later.
-  if (token.includes("5089cefa58") || token.includes("6254cf9")){
     return testSuiteRateLimiter;
   }
-  switch (mode) {
+
-    case RateLimiterMode.Preview:
+  const rateLimitConfig = RATE_LIMITS[mode];
-      return previewRateLimiter;
+  if (!rateLimitConfig) return serverRateLimiter;
-    case RateLimiterMode.CrawlStatus:
+
-      return crawlStatusRateLimiter;
+  const planKey = plan ? plan.replace("-", "") : "starter";
-    case RateLimiterMode.Crawl:
+  const points = rateLimitConfig[planKey] || rateLimitConfig.free;
-      if (plan === "standard"){
+
-        return new RateLimiterRedis({
+  return createRateLimiter(`${mode}-${planKey}`, points);
-          storeClient: redisClient,
-          keyPrefix: "crawl-standard",
-          points: MAX_CRAWLS_PER_MINUTE_STANDARD,
-          duration: 60, // Duration in seconds
-        });
-      } else if (plan === "scale"){
-        return new RateLimiterRedis({
-          storeClient: redisClient,
-          keyPrefix: "crawl-scale",
-          points: MAX_CRAWLS_PER_MINUTE_SCALE,
-          duration: 60, // Duration in seconds
-        });
-      }
-      return new RateLimiterRedis({
-        storeClient: redisClient,
-        keyPrefix: "crawl-starter",
-        points: MAX_CRAWLS_PER_MINUTE_STARTER,
-        duration: 60, // Duration in seconds
-      });
-    case RateLimiterMode.Scrape:
-      if (plan === "standard"){
-        return new RateLimiterRedis({
-          storeClient: redisClient,
-          keyPrefix: "scrape-standard",
-          points: MAX_SCRAPES_PER_MINUTE_STANDARD,
-          duration: 60, // Duration in seconds
-        });
-      } else if (plan === "scale"){
-        return new RateLimiterRedis({
-          storeClient: redisClient,
-          keyPrefix: "scrape-scale",
-          points: MAX_SCRAPES_PER_MINUTE_SCALE,
-          duration: 60, // Duration in seconds
-        });
-      }
-      return new RateLimiterRedis({
-        storeClient: redisClient,
-        keyPrefix: "scrape-starter",
-        points: MAX_SCRAPES_PER_MINUTE_STARTER,
-        duration: 60, // Duration in seconds
-      });
-    case RateLimiterMode.Search:
-      if (plan === "standard"){
-        return new RateLimiterRedis({
-          storeClient: redisClient,
-          keyPrefix: "search-standard",
-          points: MAX_SEARCHES_PER_MINUTE_STANDARD,
-          duration: 60, // Duration in seconds
-        });
-      } else if (plan === "scale"){
-        return new RateLimiterRedis({
-          storeClient: redisClient,
-          keyPrefix: "search-scale",
-          points: MAX_SEARCHES_PER_MINUTE_SCALE,
-          duration: 60, // Duration in seconds
-        });
-      }
-      return new RateLimiterRedis({
-        storeClient: redisClient,
-        keyPrefix: "search-starter",
-        points: MAX_SEARCHES_PER_MINUTE_STARTER,
-        duration: 60, // Duration in seconds
-      });
-    default:
-      return serverRateLimiter;
-  }
 }