github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Merge pull request #274 from mendableai/nsc/clusters

Browse Source 
Clusters support to scale our API to # of CPUs running
This commit is contained in:
Nicolas 2024-06-12 18:09:26 -07:00 committed by GitHub
commit d48c0df6c5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 209 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/fly.toml
View File

@ -54,7 +54,7 @@ kill_timeout = '5s'
soft_limit = 12
[[vm]]
size = 'performance-8x'
size = 'performance-4x'
processes = ['app']

1
apps/api/src/__tests__/e2e_noAuth/index.test.ts
View File

@ -1,5 +1,4 @@
import request from "supertest";
import { app } from "../../index";
import dotenv from "dotenv";
const fs = require("fs");
const path = require("path");

3
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -1,5 +1,4 @@
import request from "supertest";
import { app } from "../../index";
import dotenv from "dotenv";
import { v4 as uuidv4 } from "uuid";
@ -35,7 +34,7 @@ describe("E2E Tests for API Routes", () => {
describe("POST /v0/scrape", () => {
it.concurrent("should require authorization", async () => {
const response = await request(app).post("/v0/scrape");
const response = await request(TEST_URL).post("/v0/scrape");
expect(response.statusCode).toBe(401);
});

135
apps/api/src/index.ts
View File

@ -5,59 +5,77 @@ import "dotenv/config";
import { getWebScraperQueue } from "./services/queue-service";
import { redisClient } from "./services/rate-limiter";
import { v0Router } from "./routes/v0";
import { initSDK } from '@hyperdx/node-opentelemetry';
import { initSDK } from "@hyperdx/node-opentelemetry";
import cluster from "cluster";
import os from "os";
const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter");
const { ExpressAdapter } = require("@bull-board/express");
export const app = express();
const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
console.log(`Number of CPUs: ${numCPUs} available`);
global.isProduction = process.env.IS_PRODUCTION === "true";
if (cluster.isMaster) {
console.log(`Master ${process.pid} is running`);
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json({ limit: "10mb" }));
// Fork workers.
for (let i = 0; i < numCPUs; i++) {
cluster.fork();
}
app.use(cors()); // Add this line to enable CORS
cluster.on("exit", (worker, code, signal) => {
console.log(`Worker ${worker.process.pid} exited`);
console.log("Starting a new worker");
cluster.fork();
});
} else {
const app = express();
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
global.isProduction = process.env.IS_PRODUCTION === "true";
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json({ limit: "10mb" }));
app.use(cors()); // Add this line to enable CORS
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [new BullAdapter(getWebScraperQueue())],
serverAdapter: serverAdapter,
});
});
app.use(
app.use(
`/admin/${process.env.BULL_AUTH_KEY}/queues`,
serverAdapter.getRouter()
);
);
app.get("/", (req, res) => {
app.get("/", (req, res) => {
res.send("SCRAPERS-JS: Hello, world! Fly.io");
});
});
//write a simple test function
app.get("/test", async (req, res) => {
//write a simple test function
app.get("/test", async (req, res) => {
res.send("Hello, world!");
});
});
// register router
app.use(v0Router);
// register router
app.use(v0Router);
const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost";
redisClient.connect();
const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost";
redisClient.connect();
// HyperDX OpenTelemetry
if(process.env.ENV === 'production') {
initSDK({ consoleCapture: true, additionalInstrumentations: []});
}
// HyperDX OpenTelemetry
if (process.env.ENV === "production") {
initSDK({ consoleCapture: true, additionalInstrumentations: [] });
}
export function startServer(port = DEFAULT_PORT) {
function startServer(port = DEFAULT_PORT) {
const server = app.listen(Number(port), HOST, () => {
console.log(`Server listening on port ${port}`);
console.log(`Worker ${process.pid} listening on port ${port}`);
console.log(
`For the UI, open http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
);
@ -68,14 +86,14 @@ export function startServer(port = DEFAULT_PORT) {
);
});
return server;
}
}
if (require.main === module) {
if (require.main === module) {
startServer();
}
}
// Use this as a "health check" that way we dont destroy the server
app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => {
// Use this as a "health check" that way we dont destroy the server
app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => {
try {
const webScraperQueue = getWebScraperQueue();
const [webScraperActive] = await Promise.all([
@ -92,9 +110,9 @@ app.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, async (req, res) => {
console.error(error);
return res.status(500).json({ error: error.message });
}
});
});
app.get(`/serverHealthCheck`, async (req, res) => {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
const webScraperQueue = getWebScraperQueue();
const [waitingJobs] = await Promise.all([
@ -110,9 +128,9 @@ app.get(`/serverHealthCheck`, async (req, res) => {
console.error(error);
return res.status(500).json({ error: error.message });
}
});
});
app.get('/serverHealthCheck/notify', async (req, res) => {
app.get("/serverHealthCheck/notify", async (req, res) => {
if (process.env.SLACK_WEBHOOK_URL) {
const treshold = 1; // The treshold value for the active jobs
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
@ -138,19 +156,21 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
if (waitingJobsCount >= treshold) {
const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;
const message = {
text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${timeout/60000} minute(s).`,
text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${
timeout / 60000
} minute(s).`,
};
const response = await fetch(slackWebhookUrl, {
method: 'POST',
method: "POST",
headers: {
'Content-Type': 'application/json',
"Content-Type": "application/json",
},
body: JSON.stringify(message),
})
});
if (!response.ok) {
console.error('Failed to send Slack notification')
console.error("Failed to send Slack notification");
}
}
}, timeout);
@ -162,14 +182,18 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
checkWaitingJobs();
}
});
});
app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
app.get(
`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,
async (req, res) => {
try {
const webScraperQueue = getWebScraperQueue();
const completedJobs = await webScraperQueue.getJobs(['completed']);
const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
const jobIds = before24hJobs.map(job => job.id) as string[];
const completedJobs = await webScraperQueue.getJobs(["completed"]);
const before24hJobs = completedJobs.filter(
(job) => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000
);
const jobIds = before24hJobs.map((job) => job.id) as string[];
let count = 0;
for (const jobId of jobIds) {
try {
@ -181,14 +205,15 @@ app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, as
}
res.status(200).send(`Removed ${count} completed jobs.`);
} catch (error) {
console.error('Failed to clean last 24h complete jobs:', error);
res.status(500).send('Failed to clean jobs');
console.error("Failed to clean last 24h complete jobs:", error);
res.status(500).send("Failed to clean jobs");
}
});
}
);
app.get("/is-production", (req, res) => {
app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction });
});
});
// /workers health check, cant act as load balancer, just has to be a pre deploy thing
console.log(`Worker ${process.pid} started`);
}

31
apps/api/src/services/redis.ts
View File

@ -1,8 +1,35 @@
import Redis from 'ioredis';
import Redis from "ioredis";
// Initialize Redis client
const redis = new Redis(process.env.REDIS_URL);
// Listen to 'error' events to the Redis connection
redis.on("error", (error) => {
try {
if (error.message === "ECONNRESET") {
console.log("Connection to Redis Session Store timed out.");
} else if (error.message === "ECONNREFUSED") {
console.log("Connection to Redis Session Store refused!");
} else console.log(error);
} catch (error) {}
});
// Listen to 'reconnecting' event to Redis
redis.on("reconnecting", (err) => {
try {
if (redis.status === "reconnecting")
console.log("Reconnecting to Redis Session Store...");
else console.log("Error reconnecting to Redis Session Store.");
} catch (error) {}
});
// Listen to the 'connect' event to Redis
redis.on("connect", (err) => {
try {
if (!err) console.log("Connected to Redis Session Store!");
} catch (error) {}
});
/**
* Set a value in Redis with an optional expiration time.
* @param {string} key The key under which to store the value.
@ -11,7 +38,7 @@ const redis = new Redis(process.env.REDIS_URL);
*/
const setValue = async (key: string, value: string, expire?: number) => {
if (expire) {
await redis.set(key, value, 'EX', expire);
await redis.set(key, value, "EX", expire);
} else {
await redis.set(key, value);
}