Merge pull request #274 from mendableai/nsc/clusters

Clusters support to scale our API to # of CPUs running

apps/api/fly.toml

@@ -54,7 +54,7 @@ kill_timeout = '5s'
     soft_limit = 12
 
 [[vm]]
-  size = 'performance-8x'
+  size = 'performance-4x'
   processes = ['app']
 
 

apps/api/src/__tests__/e2e_noAuth/index.test.ts

@@ -1,5 +1,4 @@
 import request from "supertest";
-import { app } from "../../index";
 import dotenv from "dotenv";
 const fs = require("fs");
 const path = require("path");

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -1,5 +1,4 @@
 import request from "supertest";
-import { app } from "../../index";
 import dotenv from "dotenv";
 import { v4 as uuidv4 } from "uuid";
 
@@ -35,7 +34,7 @@ describe("E2E Tests for API Routes", () => {
 
   describe("POST /v0/scrape", () => {
     it.concurrent("should require authorization", async () => {
-      const response = await request(app).post("/v0/scrape");
+      const response = await request(TEST_URL).post("/v0/scrape");
       expect(response.statusCode).toBe(401);
     });
 

apps/api/src/index.ts

@@ -5,13 +5,32 @@ import "dotenv/config";
 import { getWebScraperQueue } from "./services/queue-service";
 import { redisClient } from "./services/rate-limiter";
 import { v0Router } from "./routes/v0";
-import { initSDK } from '@hyperdx/node-opentelemetry';
+import { initSDK } from "@hyperdx/node-opentelemetry";
+import cluster from "cluster";
+import os from "os";
 
 const { createBullBoard } = require("@bull-board/api");
 const { BullAdapter } = require("@bull-board/api/bullAdapter");
 const { ExpressAdapter } = require("@bull-board/express");
 
-export const app = express();
+const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
+console.log(`Number of CPUs: ${numCPUs} available`);
+
+if (cluster.isMaster) {
+  console.log(`Master ${process.pid} is running`);
+
+  // Fork workers.
+  for (let i = 0; i < numCPUs; i++) {
+    cluster.fork();
+  }
+
+  cluster.on("exit", (worker, code, signal) => {
+    console.log(`Worker ${worker.process.pid} exited`);
+    console.log("Starting a new worker");
+    cluster.fork();
+  });
+} else {
+  const app = express();
 
   global.isProduction = process.env.IS_PRODUCTION === "true";
 
@@ -50,14 +69,13 @@ const HOST = process.env.HOST ?? "localhost";
   redisClient.connect();
 
   // HyperDX OpenTelemetry
-if(process.env.ENV === 'production') {
+  if (process.env.ENV === "production") {
     initSDK({ consoleCapture: true, additionalInstrumentations: [] });
   }
 
-
+  function startServer(port = DEFAULT_PORT) {
-export function startServer(port = DEFAULT_PORT) {
     const server = app.listen(Number(port), HOST, () => {
-    console.log(`Server listening on port ${port}`);
+      console.log(`Worker ${process.pid} listening on port ${port}`);
       console.log(
         `For the UI, open http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
       );
@@ -112,7 +130,7 @@ app.get(`/serverHealthCheck`, async (req, res) => {
     }
   });
 
-app.get('/serverHealthCheck/notify', async (req, res) => {
+  app.get("/serverHealthCheck/notify", async (req, res) => {
     if (process.env.SLACK_WEBHOOK_URL) {
       const treshold = 1; // The treshold value for the active jobs
       const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
@@ -138,19 +156,21 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
               if (waitingJobsCount >= treshold) {
                 const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;
                 const message = {
-                text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${timeout/60000} minute(s).`,
+                  text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${
+                    timeout / 60000
+                  } minute(s).`,
                 };
 
                 const response = await fetch(slackWebhookUrl, {
-                method: 'POST',
+                  method: "POST",
                   headers: {
-                  'Content-Type': 'application/json',
+                    "Content-Type": "application/json",
                   },
                   body: JSON.stringify(message),
-              })
+                });
 
                 if (!response.ok) {
-                console.error('Failed to send Slack notification')
+                  console.error("Failed to send Slack notification");
                 }
               }
             }, timeout);
@@ -164,12 +184,16 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
     }
   });
 
-app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
+  app.get(
+    `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,
+    async (req, res) => {
       try {
         const webScraperQueue = getWebScraperQueue();
-    const completedJobs = await webScraperQueue.getJobs(['completed']);
+        const completedJobs = await webScraperQueue.getJobs(["completed"]);
-    const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
+        const before24hJobs = completedJobs.filter(
-    const jobIds = before24hJobs.map(job => job.id) as string[];
+          (job) => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000
+        );
+        const jobIds = before24hJobs.map((job) => job.id) as string[];
         let count = 0;
         for (const jobId of jobIds) {
           try {
@@ -181,14 +205,15 @@ app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, as
         }
         res.status(200).send(`Removed ${count} completed jobs.`);
       } catch (error) {
-    console.error('Failed to clean last 24h complete jobs:', error);
+        console.error("Failed to clean last 24h complete jobs:", error);
-    res.status(500).send('Failed to clean jobs');
+        res.status(500).send("Failed to clean jobs");
       }
-});
+    }
+  );
 
   app.get("/is-production", (req, res) => {
     res.send({ isProduction: global.isProduction });
   });
 
-
+  console.log(`Worker ${process.pid} started`);
-// /workers health check, cant act as load balancer, just has to be a pre deploy thing
+}

apps/api/src/services/redis.ts

@@ -1,8 +1,35 @@
-import Redis from 'ioredis';
+import Redis from "ioredis";
 
 // Initialize Redis client
 const redis = new Redis(process.env.REDIS_URL);
 
+// Listen to 'error' events to the Redis connection
+redis.on("error", (error) => {
+  try {
+    if (error.message === "ECONNRESET") {
+      console.log("Connection to Redis Session Store timed out.");
+    } else if (error.message === "ECONNREFUSED") {
+      console.log("Connection to Redis Session Store refused!");
+    } else console.log(error);
+  } catch (error) {}
+});
+
+// Listen to 'reconnecting' event to Redis
+redis.on("reconnecting", (err) => {
+  try {
+    if (redis.status === "reconnecting")
+      console.log("Reconnecting to Redis Session Store...");
+    else console.log("Error reconnecting to Redis Session Store.");
+  } catch (error) {}
+});
+
+// Listen to the 'connect' event to Redis
+redis.on("connect", (err) => {
+  try {
+    if (!err) console.log("Connected to Redis Session Store!");
+  } catch (error) {}
+});
+
 /**
  * Set a value in Redis with an optional expiration time.
  * @param {string} key The key under which to store the value.
@@ -11,7 +38,7 @@ const redis = new Redis(process.env.REDIS_URL);
  */
 const setValue = async (key: string, value: string, expire?: number) => {
   if (expire) {
-    await redis.set(key, value, 'EX', expire);
+    await redis.set(key, value, "EX", expire);
   } else {
     await redis.set(key, value);
   }