From 8b3c3aae911c0d6bef8e65ceae26f9fac15af288 Mon Sep 17 00:00:00 2001
From: AndyMik90 <andre@mikalsenutvikling.no>
Date: Tue, 18 Jun 2024 07:31:46 +0200
Subject: [PATCH 1/4] Added support for RegEx in removeTags

---
 apps/api/src/scraper/WebScraper/single_url.ts | 31 +++++++++++++------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index db8f0ae..354a5cb 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -316,15 +316,26 @@ export async function scrapSingleUrl(
   const removeUnwantedElements = (html: string, pageOptions: PageOptions) => {
     const soup = cheerio.load(html);
     soup("script, style, iframe, noscript, meta, head").remove();
-
+    
     if (pageOptions.removeTags) {
       if (typeof pageOptions.removeTags === 'string') {
-        pageOptions.removeTags.split(',').forEach((tag) => {
-          soup(tag.trim()).remove();
-        });
-      } else if (Array.isArray(pageOptions.removeTags)) {
+        pageOptions.removeTags = [pageOptions.removeTags];
+      }
+    
+      if (Array.isArray(pageOptions.removeTags)) {
         pageOptions.removeTags.forEach((tag) => {
-          soup(tag).remove();
+          let elementsToRemove;
+          if (tag.startsWith("*") && tag.endsWith("*")) {
+            const regexPattern = new RegExp(`\\b${tag.slice(1, -1)}\\b`);
+            elementsToRemove = soup('*').filter((index, element) => {
+              const classNames = soup(element).attr('class');
+              return classNames && classNames.split(/\s+/).some(className => regexPattern.test(className));
+            });
+          } else {
+            elementsToRemove = soup(tag);
+          }
+    
+          elementsToRemove.remove();
         });
       }
     }
@@ -332,11 +343,13 @@ export async function scrapSingleUrl(
     if (pageOptions.onlyMainContent) {
       // remove any other tags that are not in the main content
       excludeNonMainTags.forEach((tag) => {
-        soup(tag).remove();
+        const elementsToRemove = soup(tag);
+        elementsToRemove.remove();
       });
     }
-    return soup.html();
-  };
+    const cleanedHtml = soup.html();
+    return cleanedHtml;
+};
 
   const attemptScraping = async (
     url: string,

From 6c726a02eb64df41f64011d7bd87e5b6ccb6c844 Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Tue, 18 Jun 2024 09:46:42 -0300
Subject: [PATCH 2/4] Moved to utils/removeUnwantedElements, added unit tests

---
 apps/api/src/scraper/WebScraper/single_url.ts | 40 +-----------
 .../__tests__/removeUnwantedElements.test.ts  | 63 +++++++++++++++++++
 .../utils/removeUnwantedElements.ts           | 41 ++++++++++++
 3 files changed, 105 insertions(+), 39 deletions(-)
 create mode 100644 apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
 create mode 100644 apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 354a5cb..e112cd4 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -4,10 +4,10 @@ import { extractMetadata } from "./utils/metadata";
 import dotenv from "dotenv";
 import { Document, PageOptions, FireEngineResponse } from "../../lib/entities";
 import { parseMarkdown } from "../../lib/html-to-markdown";
-import { excludeNonMainTags } from "./utils/excludeTags";
 import { urlSpecificParams } from "./utils/custom/website_params";
 import { fetchAndProcessPdf } from "./utils/pdfProcessor";
 import { handleCustomScraping } from "./custom/handleCustomScraping";
+import { removeUnwantedElements } from "./utils/removeUnwantedElements";
 import axios from "axios";
 
 dotenv.config();
@@ -313,44 +313,6 @@ export async function scrapSingleUrl(
 ): Promise<Document> {
   urlToScrap = urlToScrap.trim();
 
-  const removeUnwantedElements = (html: string, pageOptions: PageOptions) => {
-    const soup = cheerio.load(html);
-    soup("script, style, iframe, noscript, meta, head").remove();
-    
-    if (pageOptions.removeTags) {
-      if (typeof pageOptions.removeTags === 'string') {
-        pageOptions.removeTags = [pageOptions.removeTags];
-      }
-    
-      if (Array.isArray(pageOptions.removeTags)) {
-        pageOptions.removeTags.forEach((tag) => {
-          let elementsToRemove;
-          if (tag.startsWith("*") && tag.endsWith("*")) {
-            const regexPattern = new RegExp(`\\b${tag.slice(1, -1)}\\b`);
-            elementsToRemove = soup('*').filter((index, element) => {
-              const classNames = soup(element).attr('class');
-              return classNames && classNames.split(/\s+/).some(className => regexPattern.test(className));
-            });
-          } else {
-            elementsToRemove = soup(tag);
-          }
-    
-          elementsToRemove.remove();
-        });
-      }
-    }
-    
-    if (pageOptions.onlyMainContent) {
-      // remove any other tags that are not in the main content
-      excludeNonMainTags.forEach((tag) => {
-        const elementsToRemove = soup(tag);
-        elementsToRemove.remove();
-      });
-    }
-    const cleanedHtml = soup.html();
-    return cleanedHtml;
-};
-
   const attemptScraping = async (
     url: string,
     method: (typeof baseScrapers)[number]
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
new file mode 100644
index 0000000..cfa49e7
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
@@ -0,0 +1,63 @@
+import { removeUnwantedElements } from "../removeUnwantedElements";
+import { PageOptions } from "../../../../lib/entities";
+
+describe('removeUnwantedElements', () => {
+  it('should remove script, style, iframe, noscript, meta, and head tags', () => {
+    const html = `<html><head><title>Test</title></head><body><script>alert('test');</script><div>Content</div></body></html>`;
+    const options: PageOptions = {};
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('<script>');
+    expect(result).not.toContain('<head>');
+    expect(result).toContain('Content');
+  });
+
+  it('should remove specified tags passed as string', () => {
+    const html = `<div><span>Remove</span><p>Keep</p></div>`;
+    const options: PageOptions = { removeTags: 'span' };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('<span>');
+    expect(result).toContain('<p>Keep</p>');
+  });
+
+  it('should remove specified tags passed as array', () => {
+    const html = `<div><span>Remove</span><p>Remove</p><a>Keep</a></div>`;
+    const options: PageOptions = { removeTags: ['span', 'p'] };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('<span>');
+    expect(result).not.toContain('<p>');
+    expect(result).toContain('<a>Keep</a>');
+  });
+
+  it('should handle class selectors', () => {
+    const html = `<div class="test">Remove</div><div class="keep">Keep</div>`;
+    const options: PageOptions = { removeTags: '.test' };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('class="test"');
+    expect(result).toContain('class="keep"');
+  });
+
+  it('should handle id selectors', () => {
+    const html = `<div id="test">Remove</div><div id="keep">Keep</div>`;
+    const options: PageOptions = { removeTags: '#test' };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('id="test"');
+    expect(result).toContain('id="keep"');
+  });
+
+  it('should handle regex patterns in class names', () => {
+    const html = `<div class="test-123">Remove</div><div class="test-abc">Remove</div><div class="keep">Keep</div>`;
+    const options: PageOptions = { removeTags: ['*test*'] };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('class="test-123"');
+    expect(result).not.toContain('class="test-abc"');
+    expect(result).toContain('class="keep"');
+  });
+
+  it('should remove non-main content if onlyMainContent is true', () => {
+    const html = `<div><main>Main Content</main><aside>Remove</aside></div>`;
+    const options: PageOptions = { onlyMainContent: true };
+    const result = removeUnwantedElements(html, options);
+    expect(result).toContain('Main Content');
+    expect(result).not.toContain('<aside>');
+  });
+});
diff --git a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
new file mode 100644
index 0000000..0fd28a5
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
@@ -0,0 +1,41 @@
+import cheerio, { AnyNode, Cheerio } from "cheerio";
+import { PageOptions } from "../../../lib/entities";
+import { excludeNonMainTags } from "./excludeTags";
+
+export const removeUnwantedElements = (html: string, pageOptions: PageOptions) => {
+  const soup = cheerio.load(html);
+  soup("script, style, iframe, noscript, meta, head").remove();
+  
+  if (pageOptions.removeTags) {
+    if (typeof pageOptions.removeTags === 'string') {
+      pageOptions.removeTags = [pageOptions.removeTags];
+    }
+  
+    if (Array.isArray(pageOptions.removeTags)) {
+      pageOptions.removeTags.forEach((tag) => {
+        let elementsToRemove: Cheerio<AnyNode>;
+        if (tag.startsWith("*") && tag.endsWith("*")) {
+          const regexPattern = new RegExp(`\\b${tag.slice(1, -1)}\\b`);
+          elementsToRemove = soup('*').filter((index, element) => {
+            const classNames = soup(element).attr('class');
+            return classNames && classNames.split(/\s+/).some(className => regexPattern.test(className));
+          });
+        } else {
+          elementsToRemove = soup(tag);
+        }
+  
+        elementsToRemove.remove();
+      });
+    }
+  }
+  
+  if (pageOptions.onlyMainContent) {
+    // remove any other tags that are not in the main content
+    excludeNonMainTags.forEach((tag) => {
+      const elementsToRemove = soup(tag);
+      elementsToRemove.remove();
+    });
+  }
+  const cleanedHtml = soup.html();
+  return cleanedHtml;
+};
\ No newline at end of file

From c2fc69af1ca0b9fda655253446aa30b3744308ed Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Tue, 18 Jun 2024 09:57:05 -0300
Subject: [PATCH 3/4] removed some e2e tests that are making the ci get stuck

---
 .../src/__tests__/e2e_withAuth/index.test.ts  | 128 +++++++++---------
 1 file changed, 64 insertions(+), 64 deletions(-)

diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index 9f04093..3213245 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -70,22 +70,22 @@ describe("E2E Tests for API Routes", () => {
     //   expect(response.statusCode).toBe(200);
     // }, 30000); // 30 seconds timeout
 
-    it.concurrent("should return a successful response with a valid API key", async () => {
-      const response = await request(TEST_URL)
-        .post("/v0/scrape")
-        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
-        .set("Content-Type", "application/json")
-        .send({ url: "https://roastmywebsite.ai" });
-      expect(response.statusCode).toBe(200);
-      expect(response.body).toHaveProperty("data");
-      expect(response.body.data).toHaveProperty("content");
-      expect(response.body.data).toHaveProperty("markdown");
-      expect(response.body.data).toHaveProperty("metadata");
-      expect(response.body.data).not.toHaveProperty("html");
-      expect(response.body.data.content).toContain("_Roast_");
-      expect(response.body.data.metadata.pageStatusCode).toBe(200);
-      expect(response.body.data.metadata.pageError).toBeUndefined();
-    }, 30000); // 30 seconds timeout
+    // it.concurrent("should return a successful response with a valid API key", async () => {
+    //   const response = await request(TEST_URL)
+    //     .post("/v0/scrape")
+    //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+    //     .set("Content-Type", "application/json")
+    //     .send({ url: "https://roastmywebsite.ai" });
+    //   expect(response.statusCode).toBe(200);
+    //   expect(response.body).toHaveProperty("data");
+    //   expect(response.body.data).toHaveProperty("content");
+    //   expect(response.body.data).toHaveProperty("markdown");
+    //   expect(response.body.data).toHaveProperty("metadata");
+    //   expect(response.body.data).not.toHaveProperty("html");
+    //   expect(response.body.data.content).toContain("_Roast_");
+    //   expect(response.body.data.metadata.pageStatusCode).toBe(200);
+    //   expect(response.body.data.metadata.pageError).toBeUndefined();
+    // }, 30000); // 30 seconds timeout
 
     it.concurrent("should return a successful response with a valid API key and includeHtml set to true", async () => {
       const response = await request(TEST_URL)
@@ -726,60 +726,60 @@ describe("E2E Tests for API Routes", () => {
     //   expect(completedResponse.body.data[0].content).not.toContain("main menu");
     // }, 60000); // 60 seconds
 
-    it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
-      const crawlResponse = await request(TEST_URL)
-        .post("/v0/crawl")
-        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
-        .set("Content-Type", "application/json")
-        .send({
-          url: "https://roastmywebsite.ai",
-          pageOptions: { includeHtml: true },
-        });
-      expect(crawlResponse.statusCode).toBe(200);
+    // it.concurrent("should return a successful response for a valid crawl job with includeHtml set to true option", async () => {
+    //   const crawlResponse = await request(TEST_URL)
+    //     .post("/v0/crawl")
+    //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+    //     .set("Content-Type", "application/json")
+    //     .send({
+    //       url: "https://roastmywebsite.ai",
+    //       pageOptions: { includeHtml: true },
+    //     });
+    //   expect(crawlResponse.statusCode).toBe(200);
 
-      const response = await request(TEST_URL)
-        .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
-        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
-      expect(response.statusCode).toBe(200);
-      expect(response.body).toHaveProperty("status");
-      expect(["active", "waiting"]).toContain(response.body.status);
+    //   const response = await request(TEST_URL)
+    //     .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+    //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+    //   expect(response.statusCode).toBe(200);
+    //   expect(response.body).toHaveProperty("status");
+    //   expect(["active", "waiting"]).toContain(response.body.status);
 
-      let isCompleted = false;
-      while (!isCompleted) {
-        const statusCheckResponse = await request(TEST_URL)
-          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
-          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
-        expect(statusCheckResponse.statusCode).toBe(200);
-        isCompleted = statusCheckResponse.body.status === "completed";
-        if (!isCompleted) {
-          await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
-        }
-      }
+    //   let isCompleted = false;
+    //   while (!isCompleted) {
+    //     const statusCheckResponse = await request(TEST_URL)
+    //       .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+    //       .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+    //     expect(statusCheckResponse.statusCode).toBe(200);
+    //     isCompleted = statusCheckResponse.body.status === "completed";
+    //     if (!isCompleted) {
+    //       await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
+    //     }
+    //   }
 
-      const completedResponse = await request(TEST_URL)
-        .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
-        .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+    //   const completedResponse = await request(TEST_URL)
+    //     .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+    //     .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
 
-      expect(completedResponse.statusCode).toBe(200);
-      expect(completedResponse.body).toHaveProperty("status");
-      expect(completedResponse.body.status).toBe("completed");
-      expect(completedResponse.body).toHaveProperty("data");
-      expect(completedResponse.body.data[0]).toHaveProperty("content");
-      expect(completedResponse.body.data[0]).toHaveProperty("markdown");
-      expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-      expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
-      expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
+    //   expect(completedResponse.statusCode).toBe(200);
+    //   expect(completedResponse.body).toHaveProperty("status");
+    //   expect(completedResponse.body.status).toBe("completed");
+    //   expect(completedResponse.body).toHaveProperty("data");
+    //   expect(completedResponse.body.data[0]).toHaveProperty("content");
+    //   expect(completedResponse.body.data[0]).toHaveProperty("markdown");
+    //   expect(completedResponse.body.data[0]).toHaveProperty("metadata");
+    //   expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
+    //   expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
 
-      // 120 seconds  
-      expect(completedResponse.body.data[0]).toHaveProperty("html");
-      expect(completedResponse.body.data[0]).toHaveProperty("metadata");
-      expect(completedResponse.body.data[0].content).toContain("_Roast_");
-      expect(completedResponse.body.data[0].markdown).toContain("_Roast_");
-      expect(completedResponse.body.data[0].html).toContain("<h1");
+    //   // 120 seconds  
+    //   expect(completedResponse.body.data[0]).toHaveProperty("html");
+    //   expect(completedResponse.body.data[0]).toHaveProperty("metadata");
+    //   expect(completedResponse.body.data[0].content).toContain("_Roast_");
+    //   expect(completedResponse.body.data[0].markdown).toContain("_Roast_");
+    //   expect(completedResponse.body.data[0].html).toContain("<h1");
 
-      expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
-      expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
-    }, 180000);
+    //   expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
+    //   expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
+    // }, 180000);
 
   });
 

From 9f7afd1e8837db0ce7af5f7b965bdcb4b7d1eeec Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Tue, 18 Jun 2024 14:36:51 -0300
Subject: [PATCH 4/4] fix for some complex cases

---
 .../__tests__/removeUnwantedElements.test.ts  | 40 +++++++++++++++++++
 .../utils/removeUnwantedElements.ts           | 24 ++++++++---
 2 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
index cfa49e7..0dc24c8 100644
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
+++ b/apps/api/src/scraper/WebScraper/utils/__tests__/removeUnwantedElements.test.ts
@@ -60,4 +60,44 @@ describe('removeUnwantedElements', () => {
     expect(result).toContain('Main Content');
     expect(result).not.toContain('<aside>');
   });
+
+  it('should handle complex regex patterns for class names', () => {
+    const html = `<div class="test-123">Remove</div><div class="test-abc">Remove</div><div class="keep">Keep</div><div class="test-xyz">Remove</div>`;
+    const options: PageOptions = { removeTags: ['*.test-[a-z]+*'] };
+    const result = removeUnwantedElements(html, options);
+    expect(result).toContain('class="test-123"');
+    expect(result).not.toContain('class="test-abc"');
+    expect(result).not.toContain('class="test-xyz"');
+    expect(result).toContain('class="keep"');
+  });
+
+  it('should handle complex regex patterns for attributes', () => {
+    const html = `<div data-info="12345">Remove</div><div data-info="abcde">Keep</div><div data-info="67890">Remove</div>`;
+    const options: PageOptions = { removeTags: ['*data-info="\\d+"*'] }; // Matches data-info that starts with digits
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('data-info="12345"');
+    expect(result).not.toContain('data-info="67890"');
+    expect(result).toContain('data-info="abcde"');
+  });
+
+  it('should handle mixed selectors with regex', () => {
+    const html = `<div class="remove-this">Remove</div><div id="remove-this">Remove</div><div class="keep-this">Keep</div>`;
+    const options: PageOptions = { removeTags: ['.remove-this', '#remove-this'] };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('class="remove-this"');
+    expect(result).not.toContain('id="remove-this"');
+    expect(result).toContain('class="keep-this"');
+  });
+
+  it('should handle multiple regex patterns', () => {
+    const html = `<div attr="test-123">Remove</div><div class="class-remove">Remove</div><div class="keep">Keep</div><div class="remove-this">Remove</div><div id="remove-this">Remove</div>`;
+    const options: PageOptions = { removeTags: ['*test*', '.class-remove', '*.remove-[a-z]+*', '#remove-this'] };
+    const result = removeUnwantedElements(html, options);
+    expect(result).not.toContain('class="test-123"');
+    expect(result).not.toContain('class="test-abc"');
+    expect(result).not.toContain('class="remove"');
+    expect(result).not.toContain('class="remove-this"');
+    expect(result).not.toContain('id="remove-this"');
+    expect(result).toContain('class="keep"');
+  });
 });
diff --git a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
index 0fd28a5..b69e007 100644
--- a/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
@@ -15,22 +15,34 @@ export const removeUnwantedElements = (html: string, pageOptions: PageOptions) =
       pageOptions.removeTags.forEach((tag) => {
         let elementsToRemove: Cheerio<AnyNode>;
         if (tag.startsWith("*") && tag.endsWith("*")) {
-          const regexPattern = new RegExp(`\\b${tag.slice(1, -1)}\\b`);
-          elementsToRemove = soup('*').filter((index, element) => {
-            const classNames = soup(element).attr('class');
-            return classNames && classNames.split(/\s+/).some(className => regexPattern.test(className));
+          let classMatch = false;
+
+          const regexPattern = new RegExp(tag.slice(1, -1), 'i');
+          elementsToRemove = soup('*').filter((i, element) => {
+            if (element.type === 'tag') {
+              const attributes = element.attribs;
+              const tagNameMatches = regexPattern.test(element.name);
+              const attributesMatch = Object.keys(attributes).some(attr => 
+                regexPattern.test(`${attr}="${attributes[attr]}"`)
+              );
+              if (tag.startsWith('*.')) {
+                classMatch = Object.keys(attributes).some(attr => 
+                  regexPattern.test(`class="${attributes[attr]}"`)
+                );
+              }
+              return tagNameMatches || attributesMatch || classMatch;
+            }
+            return false;
           });
         } else {
           elementsToRemove = soup(tag);
         }
-  
         elementsToRemove.remove();
       });
     }
   }
   
   if (pageOptions.onlyMainContent) {
-    // remove any other tags that are not in the main content
     excludeNonMainTags.forEach((tag) => {
       const elementsToRemove = soup(tag);
       elementsToRemove.remove();