Merge pull request #167 from mendableai/nsc/hyper-dx-integration

feat: HyperDX Integration

apps/api/.env.example

@@ -31,3 +31,6 @@ POSTHOG_HOST= # set if you'd like to send posthog events like job logs
 
 STRIPE_PRICE_ID_STANDARD=
 STRIPE_PRICE_ID_SCALE=
+
+HYPERDX_API_KEY=
+HDX_NODE_BETA_MODE=1

apps/api/package.json

@@ -48,6 +48,7 @@
     "@bull-board/express": "^5.8.0",
     "@devil7softwares/pos": "^1.0.2",
     "@dqbd/tiktoken": "^1.0.13",
+    "@hyperdx/node-opentelemetry": "^0.7.0",
     "@logtail/node": "^0.4.12",
     "@nangohq/node": "^0.36.33",
     "@sentry/node": "^7.48.0",

apps/api/src/controllers/auth.ts

@@ -4,11 +4,22 @@ import { AuthResponse, RateLimiterMode } from "../../src/types";
 import { supabase_service } from "../../src/services/supabase";
 import { withAuth } from "../../src/lib/withAuth";
 import { RateLimiterRedis } from "rate-limiter-flexible";
+import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
 
 export async function authenticateUser(req, res, mode?: RateLimiterMode) : Promise<AuthResponse> {
   return withAuth(supaAuthenticateUser)(req, res, mode);
 }
-
+function setTrace(team_id: string, api_key: string) {
+  try {
+    setTraceAttributes({
+      team_id,
+      api_key
+    });
+  } catch (error) {
+    console.error('Error setting trace attributes:', error);
+  }
+  
+}
 export async function supaAuthenticateUser(
   req,
   res,
@@ -78,11 +89,13 @@ export async function supaAuthenticateUser(
         status: 401,
       };
     }
-    
-    
+    const team_id = data[0].team_id;
+    const plan = getPlanByPriceId(data[0].price_id);
+    // HyperDX Logging
+    setTrace(team_id, normalizedApi);
     subscriptionData = {
-      team_id: data[0].team_id,
-      plan: getPlanByPriceId(data[0].price_id)
+      team_id: team_id,
+      plan: plan
     }
     switch (mode) { 
       case RateLimiterMode.Crawl:

apps/api/src/index.ts

@@ -5,6 +5,8 @@ import "dotenv/config";
 import { getWebScraperQueue } from "./services/queue-service";
 import { redisClient } from "./services/rate-limiter";
 import { v0Router } from "./routes/v0";
+import { initSDK } from '@hyperdx/node-opentelemetry';
+
 const { createBullBoard } = require("@bull-board/api");
 const { BullAdapter } = require("@bull-board/api/bullAdapter");
 const { ExpressAdapter } = require("@bull-board/express");
@@ -47,6 +49,9 @@ const DEFAULT_PORT = process.env.PORT ?? 3002;
 const HOST = process.env.HOST ?? "localhost";
 redisClient.connect();
 
+// HyperDX OpenTelemetry
+initSDK({ consoleCapture: true, additionalInstrumentations: []});
+
 
 export function startServer(port = DEFAULT_PORT) {
   const server = app.listen(Number(port), HOST, () => {

apps/api/src/scraper/WebScraper/crawler.ts

@@ -117,7 +117,7 @@ export class WebCrawler {
       const response = await axios.get(this.robotsTxtUrl);
       this.robots = robotsParser(this.robotsTxtUrl, response.data);
     } catch (error) {
-      console.error(`Failed to fetch robots.txt from ${this.robotsTxtUrl}`);
+      console.log(`Failed to fetch robots.txt from ${this.robotsTxtUrl}`);
 
     }
 

apps/api/src/services/billing/credit_billing.ts

@@ -227,10 +227,11 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
 
     if (creditUsages && creditUsages.length > 0) {
         totalCreditsUsed = creditUsages[0].total_credits_used;
-        console.log("Total Credits Used:", totalCreditsUsed);
+        // console.log("Total Credits Used:", totalCreditsUsed);
     }
   } catch (error) {
     console.error("Error calculating credit usage:", error);
+    
   }
   // Adjust total credits used by subtracting coupon value
   const adjustedCreditsUsed = Math.max(0, totalCreditsUsed - couponCredits);

apps/api/src/services/queue-worker.ts

@@ -5,6 +5,9 @@ import { logtail } from "./logtail";
 import { startWebScraperPipeline } from "../main/runWebScraper";
 import { callWebhook } from "./webhook";
 import { logJob } from "./logging/log_job";
+import { initSDK } from '@hyperdx/node-opentelemetry';
+
+initSDK({ consoleCapture: true, additionalInstrumentations: []});
 
 getWebScraperQueue().process(
   Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),