From a9f93c2f1e9d02303b24dd49862602d0fd5828dd Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Tue, 11 Jun 2024 14:18:05 -0300
Subject: [PATCH 1/2] Added route to clean completed jobs and a github action
 cron that triggers every 24h

---
 .../clean-before-24h-complete-jobs.yml        | 17 +++++++++++++++
 apps/api/src/index.ts                         | 21 +++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 .github/workflows/clean-before-24h-complete-jobs.yml

diff --git a/.github/workflows/clean-before-24h-complete-jobs.yml b/.github/workflows/clean-before-24h-complete-jobs.yml
new file mode 100644
index 0000000..2fd3b22
--- /dev/null
+++ b/.github/workflows/clean-before-24h-complete-jobs.yml
@@ -0,0 +1,17 @@
+name: Clean Before 24h Completed Jobs
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  clean-jobs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send GET request to clean jobs
+        run: |
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/clean-before-24h-complete-jobs)
+          if [ "$response" -ne 200 ]; then
+            echo "Failed to clean jobs. Response: $response"
+            exit 1
+          fi
+          echo "Successfully cleaned jobs. Response: $response"
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 0246a1e..eac8204 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -164,6 +164,27 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
   }
 });
 
+app.get('/clean-before-24h-complete-jobs', async (req, res) => {
+  try {
+    const webScraperQueue = getWebScraperQueue();
+    const completedJobs = await webScraperQueue.getJobs(['completed']);
+    const before24hJobs = completedJobs.filter(job => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000);
+    const jobIds = before24hJobs.map(job => job.id) as string[];
+    let count = 0;
+    for (const jobId of jobIds) {
+      try {
+        await webScraperQueue.removeJobs(jobId);
+        count++;
+      } catch (jobError) {
+        console.error(`Failed to remove job with ID ${jobId}:`, jobError);
+      }
+    }
+    res.status(200).send(`Removed ${count} completed jobs.`);
+  } catch (error) {
+    console.error('Failed to clean last 24h complete jobs:', error);
+    res.status(500).send('Failed to clean jobs');
+  }
+});
 
 app.get("/is-production", (req, res) => {
   res.send({ isProduction: global.isProduction });

From 157fbe4a1ea67e4807426696b5f9b3de446641c8 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Tue, 11 Jun 2024 17:52:01 -0300
Subject: [PATCH 2/2] added bull auth key

---
 .github/workflows/clean-before-24h-complete-jobs.yml | 5 ++++-
 apps/api/src/index.ts                                | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/clean-before-24h-complete-jobs.yml b/.github/workflows/clean-before-24h-complete-jobs.yml
index 2fd3b22..2ced537 100644
--- a/.github/workflows/clean-before-24h-complete-jobs.yml
+++ b/.github/workflows/clean-before-24h-complete-jobs.yml
@@ -3,13 +3,16 @@ on:
   schedule:
     - cron: '0 0 * * *'
 
+env:
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+    
 jobs:
   clean-jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Send GET request to clean jobs
         run: |
-          response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/clean-before-24h-complete-jobs)
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null https://api.firecrawl.dev/admin/${{ secrets.BULL_AUTH_KEY }}/clean-before-24h-complete-jobs)
           if [ "$response" -ne 200 ]; then
             echo "Failed to clean jobs. Response: $response"
             exit 1
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index eac8204..cc8376b 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -164,7 +164,7 @@ app.get('/serverHealthCheck/notify', async (req, res) => {
   }
 });
 
-app.get('/clean-before-24h-complete-jobs', async (req, res) => {
+app.get(`/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, async (req, res) => {
   try {
     const webScraperQueue = getWebScraperQueue();
     const completedJobs = await webScraperQueue.getJobs(['completed']);