Merge branch 'main' into playwright-service-bug-222

.github/workflows/fly-direct.yml

@@ -0,0 +1,37 @@
+name: Fly Deploy Direct
+on:
+  schedule:
+    - cron: '0 * * * *'
+
+env:
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }}
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  HOST: ${{ secrets.HOST }}
+  LLAMAPARSE_API_KEY: ${{ secrets.LLAMAPARSE_API_KEY }}
+  LOGTAIL_KEY: ${{ secrets.LOGTAIL_KEY }}
+  POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+  POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }}
+  NUM_WORKERS_PER_QUEUE: ${{ secrets.NUM_WORKERS_PER_QUEUE }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }}
+  PORT: ${{ secrets.PORT }}
+  REDIS_URL: ${{ secrets.REDIS_URL }}
+  SCRAPING_BEE_API_KEY: ${{ secrets.SCRAPING_BEE_API_KEY }}
+  SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }}
+  SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }}
+  SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+  TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
+
+jobs:
+  deploy:
+    name: Deploy app
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Change directory
+        run: cd apps/api
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy ./apps/api --remote-only -a firecrawl-scraper-js
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

.github/workflows/fly.yml

@@ -3,8 +3,8 @@ on:
   push:
     branches:
       - main
-  # schedule:
+  schedule:
-  #   - cron: '0 */4 * * *'
+    - cron: '0 */2 * * *'
 
 env:
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

CONTRIBUTING.md

@@ -39,7 +39,7 @@ SUPABASE_SERVICE_TOKEN=
 TEST_API_KEY= # use if you've set up authentication and want to test with a real API key
 SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
 OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
-BULL_AUTH_KEY= #
+BULL_AUTH_KEY= @
 LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
 PLAYWRIGHT_MICROSERVICE_URL=  # set if you'd like to run a playwright fallback
 LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs

apps/api/.env.example

@@ -21,7 +21,7 @@ RATE_LIMIT_TEST_API_KEY_SCRAPE= # set if you'd like to test the scraping rate li
 RATE_LIMIT_TEST_API_KEY_CRAWL= # set if you'd like to test the crawling rate limit
 SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
 OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
-BULL_AUTH_KEY= #
+BULL_AUTH_KEY= @
 LOGTAIL_KEY= # Use if you're configuring basic logging with logtail
 LLAMAPARSE_API_KEY= #Set if you have a llamaparse key you'd like to use to parse pdfs
 SERPER_API_KEY= #Set if you have a serper key you'd like to use as a search api

apps/api/fly.toml

@@ -24,14 +24,14 @@ kill_timeout = '5s'
 
 [http_service.concurrency]
   type = "requests"
-  hard_limit = 200
+  hard_limit = 100
-  soft_limit = 100
+  soft_limit = 50
 
 [[http_service.checks]]
-  grace_period = "10s"
+  grace_period = "20s"
   interval = "30s"
   method = "GET"
-  timeout = "5s"
+  timeout = "15s"
   path = "/"
 
 [[services]]
@@ -50,11 +50,11 @@ kill_timeout = '5s'
 
   [services.concurrency]
     type = 'connections'
-    hard_limit = 75
+    hard_limit = 30
-    soft_limit = 30
+    soft_limit = 12
 
 [[vm]]
-  size = 'performance-4x'
+  size = 'performance-8x'
   processes = ['app']
 
 

apps/api/package.json

@@ -29,6 +29,7 @@
     "@types/cors": "^2.8.13",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.12",
+    "@types/node": "^20.14.1",
     "body-parser": "^1.20.1",
     "express": "^4.18.2",
     "jest": "^29.6.3",

apps/api/src/__tests__/e2e_withAuth/index.test.ts

@@ -66,9 +66,9 @@ describe("E2E Tests for API Routes", () => {
         .post("/v0/scrape")
         .set("Authorization", `Bearer this_is_just_a_preview_token`)
         .set("Content-Type", "application/json")
-        .send({ url: "https://firecrawl.dev" });
+        .send({ url: "https://roastmywebsite.ai" });
       expect(response.statusCode).toBe(200);
-    }, 10000); // 10 seconds timeout
+    }, 30000); // 30 seconds timeout
 
     it("should return a successful response with a valid API key", async () => {
       const response = await request(TEST_URL)

apps/api/src/lib/entities.ts

@@ -17,6 +17,7 @@ export type PageOptions = {
   fetchPageContent?: boolean;
   waitFor?: number;
   screenshot?: boolean;
+  headers?: Record<string, string>;
 };
 
 export type ExtractorOptions = {

apps/api/src/scraper/WebScraper/index.ts

@@ -84,13 +84,15 @@ export class WebScraperDataProvider {
           const job = await getWebScraperQueue().getJob(this.bullJobId);
           const jobStatus = await job.getState();
           if (jobStatus === "failed") {
-            throw new Error(
+            console.error(
               "Job has failed or has been cancelled by the user. Stopping the job..."
             );
+            return [] as Document[];
           }
         }
       } catch (error) {
         console.error(error);
+        return [] as Document[];
       }
     }
     return results.filter((result) => result !== null) as Document[];

apps/api/src/scraper/WebScraper/single_url.ts

@@ -18,7 +18,6 @@ const baseScrapers = [
   "fetch",
 ] as const;
 
-
 export async function generateRequestParams(
   url: string,
   wait_browser: string = "domcontentloaded",
@@ -46,6 +45,7 @@ export async function scrapWithFireEngine(
   url: string,
   waitFor: number = 0,
   screenshot: boolean = false,
+  headers?: Record<string, string>,
   options?: any
 ): Promise<FireEngineResponse> {
   try {
@@ -53,14 +53,21 @@ export async function scrapWithFireEngine(
     // If the user has passed a wait parameter in the request, use that
     const waitParam = reqParams["params"]?.wait ?? waitFor;
     const screenshotParam = reqParams["params"]?.screenshot ?? screenshot;
-    console.log(`[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}`);
+    console.log(
+      `[Fire-Engine] Scraping ${url} with wait: ${waitParam} and screenshot: ${screenshotParam}`
+    );
 
-    const response = await fetch(process.env.FIRE_ENGINE_BETA_URL+ "/scrape", {
+    const response = await fetch(process.env.FIRE_ENGINE_BETA_URL + "/scrape", {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
       },
-      body: JSON.stringify({ url: url, wait: waitParam, screenshot: screenshotParam }),
+      body: JSON.stringify({
+        url: url,
+        wait: waitParam,
+        screenshot: screenshotParam,
+        headers: headers,
+      }),
     });
 
     if (!response.ok) {
@@ -70,8 +77,8 @@ export async function scrapWithFireEngine(
       return { html: "", screenshot: "" };
     }
 
-    const contentType = response.headers['content-type'];
+    const contentType = response.headers["content-type"];
-    if (contentType && contentType.includes('application/pdf')) {
+    if (contentType && contentType.includes("application/pdf")) {
       return { html: await fetchAndProcessPdf(url), screenshot: "" };
     } else {
       const data = await response.json();
@@ -107,8 +114,8 @@ export async function scrapWithScrapingBee(
       return "";
     }
 
-    const contentType = response.headers['content-type'];
+    const contentType = response.headers["content-type"];
-    if (contentType && contentType.includes('application/pdf')) {
+    if (contentType && contentType.includes("application/pdf")) {
       return fetchAndProcessPdf(url);
     } else {
       const decoder = new TextDecoder();
@@ -121,7 +128,11 @@ export async function scrapWithScrapingBee(
   }
 }
 
-export async function scrapWithPlaywright(url: string, waitFor: number = 0): Promise<string> {
+export async function scrapWithPlaywright(
+  url: string,
+  waitFor: number = 0,
+  headers?: Record<string, string>
+): Promise<string> {
   try {
     const reqParams = await generateRequestParams(url);
     // If the user has passed a wait parameter in the request, use that
@@ -132,7 +143,7 @@ export async function scrapWithPlaywright(url: string, waitFor: number = 0): Pro
       headers: {
         "Content-Type": "application/json",
       },
-      body: JSON.stringify({ url: url, wait: waitParam }),
+      body: JSON.stringify({ url: url, wait: waitParam, headers: headers }),
     });
 
     if (!response.ok) {
@@ -142,8 +153,8 @@ export async function scrapWithPlaywright(url: string, waitFor: number = 0): Pro
       return "";
     }
 
-    const contentType = response.headers['content-type'];
+    const contentType = response.headers["content-type"];
-    if (contentType && contentType.includes('application/pdf')) {
+    if (contentType && contentType.includes("application/pdf")) {
       return fetchAndProcessPdf(url);
     } else {
       const textData = await response.text();
@@ -172,8 +183,8 @@ export async function scrapWithFetch(url: string): Promise<string> {
       return "";
     }
 
-    const contentType = response.headers['content-type'];
+    const contentType = response.headers["content-type"];
-    if (contentType && contentType.includes('application/pdf')) {
+    if (contentType && contentType.includes("application/pdf")) {
       return fetchAndProcessPdf(url);
     } else {
       const text = await response.text();
@@ -191,8 +202,13 @@ export async function scrapWithFetch(url: string): Promise<string> {
  * @param defaultScraper The default scraper to use if the URL does not have a specific scraper order defined
  * @returns The order of scrapers to be used for scraping a URL
  */
-function getScrapingFallbackOrder(defaultScraper?: string, isWaitPresent: boolean = false, isScreenshotPresent: boolean = false) {
+function getScrapingFallbackOrder(
-  const availableScrapers = baseScrapers.filter(scraper => {
+  defaultScraper?: string,
+  isWaitPresent: boolean = false,
+  isScreenshotPresent: boolean = false,
+  isHeadersPresent: boolean = false
+) {
+  const availableScrapers = baseScrapers.filter((scraper) => {
     switch (scraper) {
       case "scrapingBee":
       case "scrapingBeeLoad":
@@ -206,22 +222,45 @@ function getScrapingFallbackOrder(defaultScraper?: string, isWaitPresent: boolea
     }
   });
 
-  let defaultOrder = ["scrapingBee", "fire-engine", "playwright", "scrapingBeeLoad", "fetch"];
+  let defaultOrder = [
+    "scrapingBee",
+    "fire-engine",
+    "playwright",
+    "scrapingBeeLoad",
+    "fetch",
+  ];
 
-  if (isWaitPresent || isScreenshotPresent) {
+  if (isWaitPresent || isScreenshotPresent || isHeadersPresent) {
-    defaultOrder = ["fire-engine", "playwright", ...defaultOrder.filter(scraper => scraper !== "fire-engine" && scraper !== "playwright")];
+    defaultOrder = [
+      "fire-engine",
+      "playwright",
+      ...defaultOrder.filter(
+        (scraper) => scraper !== "fire-engine" && scraper !== "playwright"
+      ),
+    ];
   }
 
-  const filteredDefaultOrder = defaultOrder.filter((scraper: typeof baseScrapers[number]) => availableScrapers.includes(scraper));
+  const filteredDefaultOrder = defaultOrder.filter(
-  const uniqueScrapers = new Set(defaultScraper ? [defaultScraper, ...filteredDefaultOrder, ...availableScrapers] : [...filteredDefaultOrder, ...availableScrapers]);
+    (scraper: (typeof baseScrapers)[number]) =>
+      availableScrapers.includes(scraper)
+  );
+  const uniqueScrapers = new Set(
+    defaultScraper
+      ? [defaultScraper, ...filteredDefaultOrder, ...availableScrapers]
+      : [...filteredDefaultOrder, ...availableScrapers]
+  );
   const scrapersInOrder = Array.from(uniqueScrapers);
-  console.log(`Scrapers in order: ${scrapersInOrder}`);
+  return scrapersInOrder as (typeof baseScrapers)[number][];
-  return scrapersInOrder as typeof baseScrapers[number][];
 }
 
-async function handleCustomScraping(text: string, url: string): Promise<FireEngineResponse | null> {
+async function handleCustomScraping(
+  text: string,
+  url: string
+): Promise<FireEngineResponse | null> {
   if (text.includes('<meta name="readme-deploy"')) {
-    console.log(`Special use case detected for ${url}, using Fire Engine with wait time 1000ms`);
+    console.log(
+      `Special use case detected for ${url}, using Fire Engine with wait time 1000ms`
+    );
     return await scrapWithFireEngine(url, 1000);
   }
   return null;
@@ -229,7 +268,13 @@ async function handleCustomScraping(text: string, url: string): Promise<FireEngi
 
 export async function scrapSingleUrl(
   urlToScrap: string,
-  pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false, waitFor: 0, screenshot: false },
+  pageOptions: PageOptions = {
+    onlyMainContent: true,
+    includeHtml: false,
+    waitFor: 0,
+    screenshot: false,
+    headers: undefined
+  },
   existingHtml: string = ""
 ): Promise<Document> {
   urlToScrap = urlToScrap.trim();
@@ -248,7 +293,7 @@ export async function scrapSingleUrl(
 
   const attemptScraping = async (
     url: string,
-    method: typeof baseScrapers[number]
+    method: (typeof baseScrapers)[number]
   ) => {
     let text = "";
     let screenshot = "";
@@ -256,7 +301,12 @@ export async function scrapSingleUrl(
       case "fire-engine":
         if (process.env.FIRE_ENGINE_BETA_URL) {
           console.log(`Scraping ${url} with Fire Engine`);
-          const response = await scrapWithFireEngine(url, pageOptions.waitFor, pageOptions.screenshot);
+          const response = await scrapWithFireEngine(
+            url,
+            pageOptions.waitFor,
+            pageOptions.screenshot,
+            pageOptions.headers
+          );
           text = response.html;
           screenshot = response.screenshot;
         }
@@ -272,7 +322,7 @@ export async function scrapSingleUrl(
         break;
       case "playwright":
         if (process.env.PLAYWRIGHT_MICROSERVICE_URL) {
-          text = await scrapWithPlaywright(url, pageOptions.waitFor);
+          text = await scrapWithPlaywright(url, pageOptions.waitFor, pageOptions.headers);
         }
         break;
       case "scrapingBeeLoad":
@@ -288,8 +338,8 @@ export async function scrapSingleUrl(
     // Check for custom scraping conditions
     const customScrapedContent = await handleCustomScraping(text, url);
     if (customScrapedContent) {
-      text = customScrapedContent[0];
+      text = customScrapedContent.html;
-      screenshot = customScrapedContent[1];
+      screenshot = customScrapedContent.screenshot;
     }
 
     //* TODO: add an optional to return markdown or structured/extracted content
@@ -306,7 +356,12 @@ export async function scrapSingleUrl(
       console.error(`Invalid URL key, trying: ${urlToScrap}`);
     }
     const defaultScraper = urlSpecificParams[urlKey]?.defaultScraper ?? "";
-    const scrapersInOrder = getScrapingFallbackOrder(defaultScraper, pageOptions && pageOptions.waitFor && pageOptions.waitFor > 0, pageOptions && pageOptions.screenshot && pageOptions.screenshot === true) 
+    const scrapersInOrder = getScrapingFallbackOrder(
+      defaultScraper,
+      pageOptions && pageOptions.waitFor && pageOptions.waitFor > 0,
+      pageOptions && pageOptions.screenshot && pageOptions.screenshot === true,
+      pageOptions && pageOptions.headers && pageOptions.headers !== undefined
+    );
 
     for (const scraper of scrapersInOrder) {
       // If exists text coming from crawler, use it
@@ -332,20 +387,24 @@ export async function scrapSingleUrl(
     const metadata = extractMetadata(soup, urlToScrap);
 
     let document: Document;
-    if(screenshot && screenshot.length > 0) {
+    if (screenshot && screenshot.length > 0) {
       document = {
         content: text,
         markdown: text,
         html: pageOptions.includeHtml ? html : undefined,
-        metadata: { ...metadata, screenshot: screenshot, sourceURL: urlToScrap, },
+        metadata: {
-      }
+          ...metadata,
-    }else{
+          screenshot: screenshot,
+          sourceURL: urlToScrap,
+        },
+      };
+    } else {
       document = {
         content: text,
         markdown: text,
         html: pageOptions.includeHtml ? html : undefined,
-        metadata: { ...metadata, sourceURL: urlToScrap, },
+        metadata: { ...metadata, sourceURL: urlToScrap },
-      }
+      };
     }
 
     return document;

apps/api/src/scraper/WebScraper/utils/__tests__/socialBlockList.test.ts

@@ -0,0 +1,66 @@
+import { isUrlBlocked } from '../blocklist';
+
+describe('isUrlBlocked', () => {
+  it('should return true for blocked social media URLs', () => {
+    const blockedUrls = [
+      'https://www.facebook.com',
+      'https://twitter.com/someuser',
+      'https://instagram.com/someuser',
+      'https://www.linkedin.com/in/someuser',
+      'https://pinterest.com/someuser',
+      'https://snapchat.com/someuser',
+      'https://tiktok.com/@someuser',
+      'https://reddit.com/r/somesubreddit',
+      'https://flickr.com/photos/someuser',
+      'https://whatsapp.com/someuser',
+      'https://wechat.com/someuser',
+      'https://telegram.org/someuser',
+    ];
+
+    blockedUrls.forEach(url => {
+      if (!isUrlBlocked(url)) {
+        console.log(`URL not blocked: ${url}`);
+      }
+      expect(isUrlBlocked(url)).toBe(true);
+    });
+  });
+
+  it('should return false for URLs containing allowed keywords', () => {
+    const allowedUrls = [
+      'https://www.facebook.com/privacy',
+      'https://twitter.com/terms',
+      'https://instagram.com/legal',
+      'https://www.linkedin.com/help',
+      'https://pinterest.com/about',
+      'https://snapchat.com/support',
+      'https://tiktok.com/contact',
+      'https://reddit.com/user-agreement',
+      'https://tumblr.com/policy',
+      'https://flickr.com/blog',
+      'https://whatsapp.com/press',
+      'https://wechat.com/careers',
+      'https://telegram.org/conditions',
+      'https://wix.com/careers',
+    ];
+
+    allowedUrls.forEach(url => {
+      expect(isUrlBlocked(url)).toBe(false);
+    });
+  });
+
+  it('should return false for non-blocked URLs', () => {
+    const nonBlockedUrls = [
+      'https://www.example.com',
+      'https://www.somewebsite.org',
+      'https://subdomain.example.com',
+      'firecrawl.dev',
+      'amazon.com',
+      'wix.com',
+      'https://wix.com'
+    ];
+
+    nonBlockedUrls.forEach(url => {
+      expect(isUrlBlocked(url)).toBe(false);
+    });
+  });
+});

apps/api/src/scraper/WebScraper/utils/blocklist.ts

@@ -1,5 +1,6 @@
 const socialMediaBlocklist = [
   'facebook.com',
+  'x.com',
   'twitter.com',
   'instagram.com',
   'linkedin.com',
@@ -32,9 +33,22 @@ const allowedKeywords = [
 ];
 
 export function isUrlBlocked(url: string): boolean {
+  // Check if the URL contains any allowed keywords
   if (allowedKeywords.some(keyword => url.includes(keyword))) {
     return false;
   }
 
-  return socialMediaBlocklist.some(domain => url.includes(domain));
+  try {
+    // Check if the URL matches any domain in the blocklist
+    return socialMediaBlocklist.some(domain => {
+      // Create a regular expression to match the exact domain
+      const domainPattern = new RegExp(`(^|\\.)${domain.replace('.', '\\.')}$`);
+      // Test the hostname of the URL against the pattern
+      return domainPattern.test(new URL(url).hostname);
+    });
+  } catch (e) {
+    // If an error occurs (e.g., invalid URL), return false
+    return false;
+  }
 }
+

apps/api/src/scraper/WebScraper/utils/custom/website_params.ts

@@ -141,5 +141,22 @@ export const urlSpecificParams = {
         accept:
           "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
       },
-  }
+  },
+  "firecrawl.dev":{
+    defaultScraper: "fire-engine",
+    params: {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        "sec-fetch-site": "same-origin",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-dest": "empty",
+        referer: "https://www.google.com/",
+        "accept-language": "en-US,en;q=0.9",
+        "accept-encoding": "gzip, deflate, br",
+        accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+      },
+    },
+  },
 };

apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts

@@ -80,7 +80,7 @@ export async function processPdfToText(filePath: string): Promise<string> {
             await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds
           }
         } catch (error) {
-          console.error("Error fetching result:", error || '');
+          console.error("Error fetching result w/ LlamaIndex");
           attempt++;
           await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds before retrying
           // You may want to handle specific errors differently
@@ -92,7 +92,7 @@ export async function processPdfToText(filePath: string): Promise<string> {
       }
       content = resultResponse.data[resultType];
     } catch (error) {
-      console.error("Error processing document:", filePath, error);
+      console.error("Error processing pdf document w/ LlamaIndex(2)");
       content = await processPdf(filePath);
     }
   } else {

apps/api/src/services/queue-service.ts

@@ -1,6 +1,7 @@
 import Queue from "bull";
+import { Queue as BullQueue } from "bull";
 
-let webScraperQueue;
+let webScraperQueue: BullQueue;
 
 export function getWebScraperQueue() {
   if (!webScraperQueue) {

apps/api/src/services/rate-limiter.ts

@@ -54,7 +54,7 @@ export const crawlStatusRateLimiter = createRateLimiter("crawl-status", RATE_LIM
 export const testSuiteRateLimiter = createRateLimiter("test-suite", RATE_LIMITS.testSuite);
 
 export function getRateLimiter(mode: RateLimiterMode, token: string, plan?: string) {
-  if (token.includes("5089cefa58") || token.includes("6254cf9")) {
+  if (token.includes("a01ccae") || token.includes("6254cf9")) {
     return testSuiteRateLimiter;
   }
   

apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts

@@ -30,7 +30,7 @@ describe('FirecrawlApp E2E Tests', () => {
     const response = await app.scrapeUrl('https://firecrawl.dev');
     expect(response).not.toBeNull();
     expect(response.data.content).toContain("🔥 Firecrawl");
-  }, 10000); // 10 seconds timeout
+  }, 30000); // 30 seconds timeout
 
   test('should return successful response for valid scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response.data).toHaveProperty('markdown');
     expect(response.data).toHaveProperty('metadata');
     expect(response.data).not.toHaveProperty('html');
-  }, 10000); // 10 seconds timeout
+  }, 30000); // 30 seconds timeout
 
   test('should return successful response with valid API key and include HTML', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@@ -49,7 +49,7 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response.data.content).toContain("🔥 Firecrawl");
     expect(response.data.markdown).toContain("🔥 Firecrawl");
     expect(response.data.html).toContain("<h1");
-  }, 10000); // 10 seconds timeout
+  }, 30000); // 30 seconds timeout
 
   test('should return successful response for valid scrape with PDF file', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
@@ -99,12 +99,12 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
-    await new Promise(resolve => setTimeout(resolve, 10000)); // wait for 10 seconds
+    await new Promise(resolve => setTimeout(resolve, 30000)); // wait for 30 seconds
     const statusResponse = await app.checkCrawlStatus(response.jobId);
     expect(statusResponse).not.toBeNull();
     expect(statusResponse.status).toBe('completed');
     expect(statusResponse.data.length).toBeGreaterThan(0);
-  }, 30000); // 30 seconds timeout
+  }, 35000); // 35 seconds timeout
 
   test('should return successful response for search', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });

apps/playwright-service/main.py

@@ -10,10 +10,10 @@ from fastapi.responses import JSONResponse
 from playwright.async_api import Browser, async_playwright
 from pydantic import BaseModel
 
-PROXY_SERVER = environ.get('PROXY_SERVER', None)
+PROXY_SERVER = environ.get("PROXY_SERVER", None)
-PROXY_USERNAME = environ.get('PROXY_USERNAME', None)
+PROXY_USERNAME = environ.get("PROXY_USERNAME", None)
-PROXY_PASSWORD = environ.get('PROXY_PASSWORD', None)
+PROXY_PASSWORD = environ.get("PROXY_PASSWORD", None)
-BLOCK_MEDIA = environ.get('BLOCK_MEDIA', 'False').upper() == 'TRUE'
+BLOCK_MEDIA = environ.get("BLOCK_MEDIA", "False").upper() == "TRUE"
 
 app = FastAPI()
 
@@ -22,6 +22,7 @@ class UrlModel(BaseModel):
     url: str
     wait_after_load: int = 0
     timeout: int = 15000
+    headers: dict = None
 
 browser: Browser = None
 
@@ -50,17 +51,28 @@ async def root(body: UrlModel):
     """
     context = None
     if PROXY_SERVER and PROXY_USERNAME and PROXY_PASSWORD:
-        context = await browser.new_context(proxy={"server": PROXY_SERVER,
+        context = await browser.new_context(
+            proxy={
+                "server": PROXY_SERVER,
                 "username": PROXY_USERNAME,
-                                                   "password": PROXY_PASSWORD})
+                "password": PROXY_PASSWORD,
+            }
+        )
     else:
         context = await browser.new_context()
 
     if BLOCK_MEDIA:
-        await context.route("**/*.{png,jpg,jpeg,gif,svg,mp3,mp4,avi,flac,ogg,wav,webm}",
+        await context.route(
-                            handler=lambda route, request: route.abort())
+            "**/*.{png,jpg,jpeg,gif,svg,mp3,mp4,avi,flac,ogg,wav,webm}",
+            handler=lambda route, request: route.abort(),
+        )
 
     page = await context.new_page()
+
+    # Set headers if provided
+    if body.headers:
+        await page.set_extra_http_headers(body.headers)
+
     await page.goto(
         body.url,
         wait_until="load",

examples/roastmywebsite/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

examples/roastmywebsite/.gitignore

@@ -0,0 +1,38 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+.env
+node_modules

examples/roastmywebsite/README.md

@@ -0,0 +1,5 @@
+# Roast My Website 🔥
+
+Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them.
+
+Check it out at roastmywebsite.ai 😈

examples/roastmywebsite/components.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "default",
+  "rsc": true,
+  "tsx": true,
+  "tailwind": {
+    "config": "tailwind.config.ts",
+    "css": "src/app/globals.css",
+    "baseColor": "zinc",
+    "cssVariables": false,
+    "prefix": ""
+  },
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils"
+  }
+}

examples/roastmywebsite/next.config.mjs

@@ -0,0 +1,11 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  env: {
+    G1: process.env.G1,
+    G2: process.env.G2,
+    G3: process.env.G3,
+    G4: process.env.G4,
+  },
+};
+
+export default nextConfig;

examples/roastmywebsite/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "roastmywebsite",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@dqbd/tiktoken": "^1.0.15",
+    "@headlessui/react": "^2.0.4",
+    "@headlessui/tailwindcss": "^0.2.0",
+    "@mendable/firecrawl-js": "^0.0.21",
+    "@radix-ui/react-dialog": "^1.0.5",
+    "@radix-ui/react-dropdown-menu": "^2.0.6",
+    "@radix-ui/react-select": "^2.0.0",
+    "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-switch": "^1.0.3",
+    "@remixicon/react": "^4.2.0",
+    "@tremor/react": "^3.17.2",
+    "@vercel/analytics": "^1.3.1",
+    "axios": "^1.7.2",
+    "class-variance-authority": "^0.7.0",
+    "clsx": "^2.1.1",
+    "cubic-spline": "^3.0.3",
+    "html2canvas": "^1.4.1",
+    "image-size": "^1.1.1",
+    "lucide": "^0.379.0",
+    "lucide-react": "^0.379.0",
+    "next": "14.2.3",
+    "next-themes": "^0.3.0",
+    "openai": "^4.47.3",
+    "react": "^18",
+    "react-dom": "^18",
+    "sonner": "^1.4.41",
+    "tailwind-merge": "^2.3.0",
+    "tailwindcss-animate": "^1.0.7",
+    "tiktoken": "^1.0.15"
+  },
+  "devDependencies": {
+    "@tailwindcss/forms": "^0.5.7",
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "@types/react-dom": "^18",
+    "eslint": "^8",
+    "eslint-config-next": "14.2.3",
+    "postcss": "^8",
+    "tailwindcss": "^3.4.3",
+    "typescript": "^5"
+  }
+}

examples/roastmywebsite/postcss.config.mjs

@@ -0,0 +1,8 @@
+/** @type {import('postcss-load-config').Config} */
+const config = {
+  plugins: {
+    tailwindcss: {},
+  },
+};
+
+export default config;

examples/roastmywebsite/public/next.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

examples/roastmywebsite/public/site.webmanifest

@@ -0,0 +1 @@
+{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

examples/roastmywebsite/public/vercel.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

examples/roastmywebsite/src/app/globals.css

@@ -0,0 +1,10 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+
+
+.fill-tremor-content-emphasis {
+  fill: rgb(113 113 122) !important;
+}
+

examples/roastmywebsite/src/app/hooks/useGithubStars.ts

@@ -0,0 +1,5 @@
+export async function useGithubStars() {
+  const res = await fetch("https://api.github.com/repos/mendableai/firecrawl");
+  const data = await res.json();
+  return data.stargazers_count;
+}

examples/roastmywebsite/src/app/layout.tsx

@@ -0,0 +1,68 @@
+import type { Metadata } from "next";
+import { Gloria_Hallelujah } from "next/font/google";
+import "./globals.css";
+import { Toaster } from "sonner";
+import { Analytics } from "@vercel/analytics/react";
+import { useEffect, useState } from "react";
+import Head from "next/head";
+
+ 
+const inter = Gloria_Hallelujah({ weight: "400", subsets: ["latin"] });
+// const inter = Inter({ subsets: ["latin"] });
+
+const meta = {
+  title: "Roast My Website",
+  description:
+    "Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 😈",
+  cardImage: "/og.png",
+  robots: "follow, index",
+  favicon: "/favicon.ico",
+  url: "https://www.roastmywebsite.ai/",
+};
+
+export async function generateMetadata(): Promise<Metadata> {
+  return {
+    title: meta.title,
+    description: meta.description,
+    referrer: "origin-when-cross-origin",
+    keywords: ["Roast My Website", "Roast", "Website", "GitHub", "Firecrawl"],
+    authors: [
+      { name: "Roast My Website", url: "https://www.roastmywebsite.ai/" },
+    ],
+    creator: "Roast My Website",
+    publisher: "Roast My Website",
+    robots: meta.robots,
+    icons: { icon: meta.favicon },
+    metadataBase: new URL(meta.url),
+    openGraph: {
+      url: meta.url,
+      title: meta.title,
+      description: meta.description,
+      images: [meta.cardImage],
+      type: "website",
+      siteName: meta.title,
+    },
+    twitter: {
+      card: "summary_large_image",
+      site: "@Vercel",
+      creator: "@Vercel",
+      title: meta.title,
+      description: meta.description,
+      images: [meta.cardImage],
+    },
+  };
+}
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+      <Analytics />
+      <Toaster />
+    </html>
+  );
+}

examples/roastmywebsite/src/app/page.tsx

@@ -0,0 +1,16 @@
+// pages/index.tsx
+import MainComponent from "@/components/main";
+import { useGithubStars } from "./hooks/useGithubStars";
+import GithubButton from "@/components/github-button";
+
+export default async function Home() {
+  const githubStars = await useGithubStars();
+  return (
+    <div className="relative">
+      <div className="hidden md:flex z-10 absolute top-4 right-4 p-4">
+        <GithubButton githubStars={githubStars} />
+      </div>
+      <MainComponent />
+    </div>
+  );
+}

examples/roastmywebsite/src/components/github-button.tsx

@@ -0,0 +1,20 @@
+"use client";
+import { Github } from "lucide-react";
+import { Button } from "./ui/button";
+
+export default function GithubButton({ githubStars }: { githubStars: number }) {
+  return (
+    <Button
+      onClick={() => {
+        window.open("https://github.com/mendableai/firecrawl", "_blank");
+      }}
+      variant="outline"
+      size="icon"
+      className="px-3 w-22 gap-2"
+    >
+      <Github className="h-4 w-4" />{" "}
+      {/* {githubStars ? `Star us on GitHub` : "GitHub"} */}
+      See code on Github
+    </Button>
+  );
+}

examples/roastmywebsite/src/components/main.tsx

@@ -0,0 +1,113 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+import { Theme, allThemes } from "@/lib/theme";
+import html2canvas from "html2canvas";
+import { Button } from "@/components/ui/button";
+import { Input } from "./ui/input";
+import { Github } from "lucide-react";
+
+export default function MainComponent() {
+  const [roastUrl, setRoastUrl] = useState("");
+  const [loading, setLoading] = useState(false);
+  const [roastData, setRoastData] = useState("");
+
+  const [spiceLevel, setSpiceLevel] = useState(2);
+
+  return (
+    <div
+      className="h-screen"
+      style={{
+        background: `linear-gradient(to bottom right, rgba(255, 255, 255, 0.75) 58%, #fff, red )`,
+      }}
+    >
+      <main className="relative flex h-[95vh] flex-col items-center justify-center bg-transparent bg-opacity-80">
+        <div className="w-3/4 flex flex-col items-center gap-4">
+          <h1 className="font-inter-tight text-4xl lg:text-5xl xl:text-6xl font-bold bg-clip-text text-transparent bg-gradient-to-r from-zinc-500 via-zinc-900 to-zinc-900 pb-4 text-center">
+            <em className="relative px-1 italic animate-text transition-all text-transparent bg-clip-text bg-gradient-to-tr from-red-600 to-red-400 inline-flex justify-center items-center text-6xl lg:text-7xl xl:text-8xl">
+              Roast
+            </em>
+            <br />{" "}
+            <span className="text-4xl lg:text-5xl xl:text-6xl">My Website</span>
+          </h1>
+          <div className="flex flex-col sm:flex-row  items-center gap-4 justify-center w-3/5">
+            <Input
+              type="text"
+              className="w-full p-2 border border-gray-300 rounded r"
+              placeholder="https://coconut.com/"
+              value={roastUrl}
+              onChange={(e) => setRoastUrl(e.target.value)}
+            />
+            <Button
+              className="px-6 py-2 bg-red-500/25 text-red-500 0 rounded-lg hover:bg-red-300 w-1/8 whitespace-nowrap"
+              onClick={async () => {
+                if (roastUrl) {
+                  setLoading(true);
+                  try {
+                    const response = await fetch(
+                      `/api/roastWebsite?url=${encodeURIComponent(
+                        roastUrl
+                      )}&spiceLevel=${spiceLevel}`
+                    );
+                    if (!response.ok) {
+                      throw new Error(`Error: ${response.statusText}`);
+                    }
+                    const data = await response.json();
+                    setRoastData(data.roastResult);
+                  } catch (error) {
+                    console.error("Error:", error);
+                  } finally {
+                    setLoading(false);
+                  }
+                }
+              }}
+            >
+              {loading ? "Loading..." : "Get Roasted 🌶️"}
+            </Button>
+          </div>
+          <div className="flex items-center justify-center mt-4">
+            <label
+              htmlFor="spice-level"
+              className="mr-4 font-medium text-gray-700"
+            >
+              Choose your roast level:
+            </label>
+            <select
+              id="spice-level"
+              className="cursor-pointer rounded-lg border border-gray-300 bg-white py-2 px-4 text-center shadow-sm focus:border-orange-500 focus:outline-none focus:ring-1 focus:ring-orange-500"
+              value={spiceLevel}
+              onChange={(e) => setSpiceLevel(Number(e.target.value))}
+            >
+              <option value={1}>Mild 🌶️</option>
+              <option value={2}>Medium 🌶️🌶️</option>
+              <option value={3}>Spicy 🌶️🌶️🌶️</option>
+            </select>
+          </div>
+          {loading ? (
+            <div className="mt-4 w-3/5 p-4 border border-gray-300 rounded shadow bg-gradient-to-r from-red-500 to-red-400 animate-pulse">
+              <p className="text-white text-center">Preparing your roast...</p>
+            </div>
+          ) : (
+            roastData && (
+              <div className="!font-sans mt-4 w-3/5 p-4 border border-gray-300 rounded shadow">
+                <p>{roastData}</p>
+              </div>
+            )
+          )}
+        </div>
+        <div
+          className={`fixed bottom-0 left-0 right-0 p-4 text-white text-center font-light flex justify-center items-center gap-4`}
+        >
+          <a
+            href="https://firecrawl.dev"
+            target="_blank"
+            className="text-black hover:text-orange-400"
+            style={{ textShadow: "2px 2px 4px rgba(0, 0, 0, 0.15)" }}
+          >
+            A demo web scraping and vision extraction from Firecrawl 🔥
+          </a>
+        </div>
+      </main>
+    </div>
+  );
+}

examples/roastmywebsite/src/components/ui/button.tsx

@@ -0,0 +1,56 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const buttonVariants = cva(
+  "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-white transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-zinc-950 focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 dark:ring-offset-zinc-950 dark:focus-visible:ring-zinc-300",
+  {
+    variants: {
+      variant: {
+        default: "bg-zinc-900 text-zinc-50 hover:bg-zinc-900/90 dark:bg-zinc-50 dark:text-zinc-900 dark:hover:bg-zinc-50/90",
+        destructive:
+          "bg-red-500 text-zinc-50 hover:bg-red-500/90 dark:bg-red-900 dark:text-zinc-50 dark:hover:bg-red-900/90",
+        outline:
+          "border border-zinc-200 bg-white hover:bg-zinc-100 hover:text-zinc-900 dark:border-zinc-800 dark:bg-zinc-950 dark:hover:bg-zinc-800 dark:hover:text-zinc-50",
+        secondary:
+          "bg-zinc-100 text-zinc-900 hover:bg-zinc-100/80 dark:bg-zinc-800 dark:text-zinc-50 dark:hover:bg-zinc-800/80",
+        ghost: "hover:bg-zinc-100 hover:text-zinc-900 dark:hover:bg-zinc-800 dark:hover:text-zinc-50",
+        link: "text-zinc-900 underline-offset-4 hover:underline dark:text-zinc-50",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  }
+)
+
+export interface ButtonProps
+  extends React.ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof buttonVariants> {
+  asChild?: boolean
+}
+
+const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
+  ({ className, variant, size, asChild = false, ...props }, ref) => {
+    const Comp = asChild ? Slot : "button"
+    return (
+      <Comp
+        className={cn(buttonVariants({ variant, size, className }))}
+        ref={ref}
+        {...props}
+      />
+    )
+  }
+)
+Button.displayName = "Button"
+
+export { Button, buttonVariants }

examples/roastmywebsite/src/components/ui/dialog.tsx

@@ -0,0 +1,122 @@
+"use client"
+
+import * as React from "react"
+import * as DialogPrimitive from "@radix-ui/react-dialog"
+import { X } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+const Dialog = DialogPrimitive.Root
+
+const DialogTrigger = DialogPrimitive.Trigger
+
+const DialogPortal = DialogPrimitive.Portal
+
+const DialogClose = DialogPrimitive.Close
+
+const DialogOverlay = React.forwardRef<
+  React.ElementRef<typeof DialogPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DialogPrimitive.Overlay
+    ref={ref}
+    className={cn(
+      "fixed inset-0 z-50 bg-black/80  data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
+      className
+    )}
+    {...props}
+  />
+))
+DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
+
+const DialogContent = React.forwardRef<
+  React.ElementRef<typeof DialogPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DialogPortal>
+    <DialogOverlay />
+    <DialogPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed left-[50%] top-[50%] z-50 grid w-full max-w-lg translate-x-[-50%] translate-y-[-50%] gap-4 border border-zinc-200 bg-white p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg dark:border-zinc-800 dark:bg-zinc-950",
+        className
+      )}
+      {...props}
+    >
+      {children}
+      <DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-white transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-zinc-950 focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-zinc-100 data-[state=open]:text-zinc-500 dark:ring-offset-zinc-950 dark:focus:ring-zinc-300 dark:data-[state=open]:bg-zinc-800 dark:data-[state=open]:text-zinc-400">
+        <X className="h-4 w-4" />
+        <span className="sr-only">Close</span>
+      </DialogPrimitive.Close>
+    </DialogPrimitive.Content>
+  </DialogPortal>
+))
+DialogContent.displayName = DialogPrimitive.Content.displayName
+
+const DialogHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn(
+      "flex flex-col space-y-1.5 text-center sm:text-left",
+      className
+    )}
+    {...props}
+  />
+)
+DialogHeader.displayName = "DialogHeader"
+
+const DialogFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn(
+      "flex flex-col-reverse sm:flex-row sm:justify-end sm:space-x-2",
+      className
+    )}
+    {...props}
+  />
+)
+DialogFooter.displayName = "DialogFooter"
+
+const DialogTitle = React.forwardRef<
+  React.ElementRef<typeof DialogPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DialogPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className
+    )}
+    {...props}
+  />
+))
+DialogTitle.displayName = DialogPrimitive.Title.displayName
+
+const DialogDescription = React.forwardRef<
+  React.ElementRef<typeof DialogPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DialogPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-zinc-500 dark:text-zinc-400", className)}
+    {...props}
+  />
+))
+DialogDescription.displayName = DialogPrimitive.Description.displayName
+
+export {
+  Dialog,
+  DialogPortal,
+  DialogOverlay,
+  DialogClose,
+  DialogTrigger,
+  DialogContent,
+  DialogHeader,
+  DialogFooter,
+  DialogTitle,
+  DialogDescription,
+}

examples/roastmywebsite/src/components/ui/dropdown-menu.tsx

@@ -0,0 +1,200 @@
+"use client"
+
+import * as React from "react"
+import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu"
+import { Check, ChevronRight, Circle } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+const DropdownMenu = DropdownMenuPrimitive.Root
+
+const DropdownMenuTrigger = DropdownMenuPrimitive.Trigger
+
+const DropdownMenuGroup = DropdownMenuPrimitive.Group
+
+const DropdownMenuPortal = DropdownMenuPrimitive.Portal
+
+const DropdownMenuSub = DropdownMenuPrimitive.Sub
+
+const DropdownMenuRadioGroup = DropdownMenuPrimitive.RadioGroup
+
+const DropdownMenuSubTrigger = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.SubTrigger>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.SubTrigger> & {
+    inset?: boolean
+  }
+>(({ className, inset, children, ...props }, ref) => (
+  <DropdownMenuPrimitive.SubTrigger
+    ref={ref}
+    className={cn(
+      "flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none focus:bg-zinc-100 data-[state=open]:bg-zinc-100 dark:focus:bg-zinc-800 dark:data-[state=open]:bg-zinc-800",
+      inset && "pl-8",
+      className
+    )}
+    {...props}
+  >
+    {children}
+    <ChevronRight className="ml-auto h-4 w-4" />
+  </DropdownMenuPrimitive.SubTrigger>
+))
+DropdownMenuSubTrigger.displayName =
+  DropdownMenuPrimitive.SubTrigger.displayName
+
+const DropdownMenuSubContent = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.SubContent>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.SubContent>
+>(({ className, ...props }, ref) => (
+  <DropdownMenuPrimitive.SubContent
+    ref={ref}
+    className={cn(
+      "z-50 min-w-[8rem] overflow-hidden rounded-md border border-zinc-200 bg-white p-1 text-zinc-950 shadow-lg data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-zinc-800 dark:bg-zinc-950 dark:text-zinc-50",
+      className
+    )}
+    {...props}
+  />
+))
+DropdownMenuSubContent.displayName =
+  DropdownMenuPrimitive.SubContent.displayName
+
+const DropdownMenuContent = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Content>
+>(({ className, sideOffset = 4, ...props }, ref) => (
+  <DropdownMenuPrimitive.Portal>
+    <DropdownMenuPrimitive.Content
+      ref={ref}
+      sideOffset={sideOffset}
+      className={cn(
+        "z-50 min-w-[8rem] overflow-hidden rounded-md border border-zinc-200 bg-white p-1 text-zinc-950 shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-zinc-800 dark:bg-zinc-950 dark:text-zinc-50",
+        className
+      )}
+      {...props}
+    />
+  </DropdownMenuPrimitive.Portal>
+))
+DropdownMenuContent.displayName = DropdownMenuPrimitive.Content.displayName
+
+const DropdownMenuItem = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Item> & {
+    inset?: boolean
+  }
+>(({ className, inset, ...props }, ref) => (
+  <DropdownMenuPrimitive.Item
+    ref={ref}
+    className={cn(
+      "relative flex cursor-default select-none items-center rounded-sm px-2 py-1.5 text-sm outline-none transition-colors focus:bg-zinc-100 focus:text-zinc-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:focus:bg-zinc-800 dark:focus:text-zinc-50",
+      inset && "pl-8",
+      className
+    )}
+    {...props}
+  />
+))
+DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName
+
+const DropdownMenuCheckboxItem = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>
+>(({ className, children, checked, ...props }, ref) => (
+  <DropdownMenuPrimitive.CheckboxItem
+    ref={ref}
+    className={cn(
+      "relative flex cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none transition-colors focus:bg-zinc-100 focus:text-zinc-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:focus:bg-zinc-800 dark:focus:text-zinc-50",
+      className
+    )}
+    checked={checked}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <DropdownMenuPrimitive.ItemIndicator>
+        <Check className="h-4 w-4" />
+      </DropdownMenuPrimitive.ItemIndicator>
+    </span>
+    {children}
+  </DropdownMenuPrimitive.CheckboxItem>
+))
+DropdownMenuCheckboxItem.displayName =
+  DropdownMenuPrimitive.CheckboxItem.displayName
+
+const DropdownMenuRadioItem = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.RadioItem>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.RadioItem>
+>(({ className, children, ...props }, ref) => (
+  <DropdownMenuPrimitive.RadioItem
+    ref={ref}
+    className={cn(
+      "relative flex cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none transition-colors focus:bg-zinc-100 focus:text-zinc-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:focus:bg-zinc-800 dark:focus:text-zinc-50",
+      className
+    )}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <DropdownMenuPrimitive.ItemIndicator>
+        <Circle className="h-2 w-2 fill-current" />
+      </DropdownMenuPrimitive.ItemIndicator>
+    </span>
+    {children}
+  </DropdownMenuPrimitive.RadioItem>
+))
+DropdownMenuRadioItem.displayName = DropdownMenuPrimitive.RadioItem.displayName
+
+const DropdownMenuLabel = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Label> & {
+    inset?: boolean
+  }
+>(({ className, inset, ...props }, ref) => (
+  <DropdownMenuPrimitive.Label
+    ref={ref}
+    className={cn(
+      "px-2 py-1.5 text-sm font-semibold",
+      inset && "pl-8",
+      className
+    )}
+    {...props}
+  />
+))
+DropdownMenuLabel.displayName = DropdownMenuPrimitive.Label.displayName
+
+const DropdownMenuSeparator = React.forwardRef<
+  React.ElementRef<typeof DropdownMenuPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <DropdownMenuPrimitive.Separator
+    ref={ref}
+    className={cn("-mx-1 my-1 h-px bg-zinc-100 dark:bg-zinc-800", className)}
+    {...props}
+  />
+))
+DropdownMenuSeparator.displayName = DropdownMenuPrimitive.Separator.displayName
+
+const DropdownMenuShortcut = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLSpanElement>) => {
+  return (
+    <span
+      className={cn("ml-auto text-xs tracking-widest opacity-60", className)}
+      {...props}
+    />
+  )
+}
+DropdownMenuShortcut.displayName = "DropdownMenuShortcut"
+
+export {
+  DropdownMenu,
+  DropdownMenuTrigger,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuCheckboxItem,
+  DropdownMenuRadioItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+  DropdownMenuShortcut,
+  DropdownMenuGroup,
+  DropdownMenuPortal,
+  DropdownMenuSub,
+  DropdownMenuSubContent,
+  DropdownMenuSubTrigger,
+  DropdownMenuRadioGroup,
+}

examples/roastmywebsite/src/components/ui/input.tsx

@@ -0,0 +1,25 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+export interface InputProps
+  extends React.InputHTMLAttributes<HTMLInputElement> {}
+
+const Input = React.forwardRef<HTMLInputElement, InputProps>(
+  ({ className, type, ...props }, ref) => {
+    return (
+      <input
+        type={type}
+        className={cn(
+          "flex h-10 w-full rounded-md border border-zinc-200 bg-white px-3 py-2 text-sm ring-offset-white file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-zinc-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-zinc-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-zinc-800 dark:bg-zinc-950 dark:ring-offset-zinc-950 dark:placeholder:text-zinc-400 dark:focus-visible:ring-zinc-300",
+          className
+        )}
+        ref={ref}
+        {...props}
+      />
+    )
+  }
+)
+Input.displayName = "Input"
+
+export { Input }

examples/roastmywebsite/src/components/ui/select.tsx

@@ -0,0 +1,160 @@
+"use client"
+
+import * as React from "react"
+import * as SelectPrimitive from "@radix-ui/react-select"
+import { Check, ChevronDown, ChevronUp } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+const Select = SelectPrimitive.Root
+
+const SelectGroup = SelectPrimitive.Group
+
+const SelectValue = SelectPrimitive.Value
+
+const SelectTrigger = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Trigger>,
+  Omit<React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger>, 'noIcon'> & { noIcon?: boolean }
+>(({ className, children, noIcon, ...props }, ref) => (
+  <SelectPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "flex h-10 w-full items-center justify-between rounded-md border border-zinc-200 bg-white px-3 py-2 text-sm ring-offset-white placeholder:text-zinc-500 focus:outline-none focus:ring-2 focus:ring-zinc-950 focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1 dark:border-zinc-800 dark:bg-zinc-950 dark:ring-offset-zinc-950 dark:placeholder:text-zinc-400 dark:focus:ring-zinc-300",
+      className
+    )}
+    {...props}
+  >
+    {children}
+    <SelectPrimitive.Icon asChild>
+      {noIcon ? <></> : <ChevronDown className="h-4 w-4 opacity-50" />}
+    </SelectPrimitive.Icon>
+  </SelectPrimitive.Trigger>
+))
+SelectTrigger.displayName = SelectPrimitive.Trigger.displayName
+
+const SelectScrollUpButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollUpButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollUpButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollUpButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronUp className="h-4 w-4" />
+  </SelectPrimitive.ScrollUpButton>
+))
+SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName
+
+const SelectScrollDownButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollDownButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollDownButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollDownButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronDown className="h-4 w-4" />
+  </SelectPrimitive.ScrollDownButton>
+))
+SelectScrollDownButton.displayName =
+  SelectPrimitive.ScrollDownButton.displayName
+
+const SelectContent = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
+>(({ className, children, position = "popper", ...props }, ref) => (
+  <SelectPrimitive.Portal>
+    <SelectPrimitive.Content
+      ref={ref}
+      className={cn(
+        "relative z-50 max-h-96 min-w-[8rem] overflow-hidden rounded-md border border-zinc-200 bg-white text-zinc-950 shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-zinc-800 dark:bg-zinc-950 dark:text-zinc-50",
+        position === "popper" &&
+          "data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+        className
+      )}
+      position={position}
+      {...props}
+    >
+      <SelectScrollUpButton />
+      <SelectPrimitive.Viewport
+        className={cn(
+          "p-1",
+          position === "popper" &&
+            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
+        )}
+      >
+        {children}
+      </SelectPrimitive.Viewport>
+      <SelectScrollDownButton />
+    </SelectPrimitive.Content>
+  </SelectPrimitive.Portal>
+))
+SelectContent.displayName = SelectPrimitive.Content.displayName
+
+const SelectLabel = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Label
+    ref={ref}
+    className={cn("py-1.5 pl-8 pr-2 text-sm font-semibold", className)}
+    {...props}
+  />
+))
+SelectLabel.displayName = SelectPrimitive.Label.displayName
+
+const SelectItem = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Item
+    ref={ref}
+    className={cn(
+      "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none focus:bg-zinc-100 focus:text-zinc-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:focus:bg-zinc-800 dark:focus:text-zinc-50",
+      className
+    )}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <SelectPrimitive.ItemIndicator>
+        <Check className="h-4 w-4" />
+      </SelectPrimitive.ItemIndicator>
+    </span>
+
+    <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+  </SelectPrimitive.Item>
+))
+SelectItem.displayName = SelectPrimitive.Item.displayName
+
+const SelectSeparator = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Separator
+    ref={ref}
+    className={cn("-mx-1 my-1 h-px bg-zinc-100 dark:bg-zinc-800", className)}
+    {...props}
+  />
+))
+SelectSeparator.displayName = SelectPrimitive.Separator.displayName
+
+export {
+  Select,
+  SelectGroup,
+  SelectValue,
+  SelectTrigger,
+  SelectContent,
+  SelectLabel,
+  SelectItem,
+  SelectSeparator,
+  SelectScrollUpButton,
+  SelectScrollDownButton,
+}

examples/roastmywebsite/src/components/ui/sonner.tsx

@@ -0,0 +1,31 @@
+"use client"
+
+import { useTheme } from "next-themes"
+import { Toaster as Sonner } from "sonner"
+
+type ToasterProps = React.ComponentProps<typeof Sonner>
+
+const Toaster = ({ ...props }: ToasterProps) => {
+  const { theme = "system" } = useTheme()
+
+  return (
+    <Sonner
+      theme={theme as ToasterProps["theme"]}
+      className="toaster group"
+      toastOptions={{
+        classNames: {
+          toast:
+            "group toast group-[.toaster]:bg-white group-[.toaster]:text-zinc-950 group-[.toaster]:border-zinc-200 group-[.toaster]:shadow-lg dark:group-[.toaster]:bg-zinc-950 dark:group-[.toaster]:text-zinc-50 dark:group-[.toaster]:border-zinc-800",
+          description: "group-[.toast]:text-zinc-500 dark:group-[.toast]:text-zinc-400",
+          actionButton:
+            "group-[.toast]:bg-zinc-900 group-[.toast]:text-zinc-50 dark:group-[.toast]:bg-zinc-50 dark:group-[.toast]:text-zinc-900",
+          cancelButton:
+            "group-[.toast]:bg-zinc-100 group-[.toast]:text-zinc-500 dark:group-[.toast]:bg-zinc-800 dark:group-[.toast]:text-zinc-400",
+        },
+      }}
+      {...props}
+    />
+  )
+}
+
+export { Toaster }

examples/roastmywebsite/src/components/ui/switch.tsx

@@ -0,0 +1,29 @@
+"use client"
+
+import * as React from "react"
+import * as SwitchPrimitives from "@radix-ui/react-switch"
+
+import { cn } from "@/lib/utils"
+
+const Switch = React.forwardRef<
+  React.ElementRef<typeof SwitchPrimitives.Root>,
+  React.ComponentPropsWithoutRef<typeof SwitchPrimitives.Root>
+>(({ className, ...props }, ref) => (
+  <SwitchPrimitives.Root
+    className={cn(
+      "peer inline-flex h-6 w-11 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-zinc-950 focus-visible:ring-offset-2 focus-visible:ring-offset-white disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-zinc-900 data-[state=unchecked]:bg-zinc-200 dark:focus-visible:ring-zinc-300 dark:focus-visible:ring-offset-zinc-950 dark:data-[state=checked]:bg-zinc-50 dark:data-[state=unchecked]:bg-zinc-800",
+      className
+    )}
+    {...props}
+    ref={ref}
+  >
+    <SwitchPrimitives.Thumb
+      className={cn(
+        "pointer-events-none block h-5 w-5 rounded-full bg-white shadow-lg ring-0 transition-transform data-[state=checked]:translate-x-5 data-[state=unchecked]:translate-x-0 dark:bg-zinc-950"
+      )}
+    />
+  </SwitchPrimitives.Root>
+))
+Switch.displayName = SwitchPrimitives.Root.displayName
+
+export { Switch }

examples/roastmywebsite/src/components/ui/textarea.tsx

@@ -0,0 +1,24 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+export interface TextareaProps
+  extends React.TextareaHTMLAttributes<HTMLTextAreaElement> {}
+
+const Textarea = React.forwardRef<HTMLTextAreaElement, TextareaProps>(
+  ({ className, ...props }, ref) => {
+    return (
+      <textarea
+        className={cn(
+          "flex min-h-[80px] w-full rounded-md border border-zinc-200 bg-white px-3 py-2 text-sm ring-offset-white placeholder:text-zinc-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-zinc-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-zinc-800 dark:bg-zinc-950 dark:ring-offset-zinc-950 dark:placeholder:text-zinc-400 dark:focus-visible:ring-zinc-300",
+          className
+        )}
+        ref={ref}
+        {...props}
+      />
+    )
+  }
+)
+Textarea.displayName = "Textarea"
+
+export { Textarea }

examples/roastmywebsite/src/lib/LLM/llm.ts

@@ -0,0 +1,75 @@
+import OpenAI from "openai";
+import { encoding_for_model } from "@dqbd/tiktoken";
+
+/**
+ * Function to generate a roast for a website based on its screenshot and markdown content.
+ * @param roastPrompt - Initial prompt text for the roast.
+ * @param screenshotUrl - URL of the screenshot of the website.
+ * @param content - Raw markdown content of the website.
+ */
+export async function roastPrompt(roastPrompt: string, screenshotUrl: string, content: string) {
+    try {
+        // Initialize OpenAI with the API key from environment variables
+        const openai = new OpenAI({
+            apiKey: process.env.OPEN_AI_KEY
+        });
+
+    
+        let contentTruncated = await truncateContentToFit(content, 30000);
+
+        const response = await openai.chat.completions.create({
+            model: "gpt-4o",
+            messages: [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: roastPrompt },
+                        { type: "image_url", image_url: { url: screenshotUrl, detail: "low" } },
+                        { type: "text", text: contentTruncated }
+                    ],
+                },
+            ],
+        });
+        // Return the first choice's message instead of logging it
+        return response.choices[0].message.content;
+    } catch (error) {
+        console.error("Error generating roast:", error);
+        // Assert error as an instance of Error to access message property
+        return `Error generating roast: ${(error as Error).message}`;
+    }
+}
+
+
+
+export function numTokensFromString(message: string, model: string): number {
+    const encoder = encoding_for_model(model as any);
+
+    // Encode the message into tokens
+    const tokens = encoder.encode(message);
+
+    // Free the encoder resources after use
+    encoder.free();
+
+    // Return the number of tokens
+    return tokens.length;
+}
+
+
+
+async function truncateContentToFit(content: string, maxTokens: number): Promise<string> {
+    const modifier = 4;
+
+    let contentTotruncate = content;
+    const numTokens = numTokensFromString(contentTotruncate, "gpt-4");
+
+    if (numTokens > maxTokens) {
+        // trim the document to the maximum number of tokens, tokens != characters
+        contentTotruncate = content.slice(0, (maxTokens * modifier));
+
+    }
+    return contentTotruncate
+}
+
+
+
+

examples/roastmywebsite/src/lib/utils.ts

@@ -0,0 +1,6 @@
+import { type ClassValue, clsx } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs))
+}

examples/roastmywebsite/src/pages/api/roastWebsite.ts

@@ -0,0 +1,76 @@
+// pages/api/getRepoStars.ts
+import { NextApiRequest, NextApiResponse } from 'next';
+import FirecrawlApp from '@mendable/firecrawl-js';
+import { roastPrompt } from '../../lib/LLM/llm';
+
+
+const mildPrompt = "Give me a light hearted roast for this website:"
+const mediumPrompt = "Give me a roast for this website:"
+const spicyPrompt ="Roast this website. The person who made this website wants to be roasted really hard. I mean REALLY hard, don't hold back, and don't worry about hurting their feelings. Be mean but concisely mean. No more than 2 paragraphs."
+
+
+async function getScreenShotAndHtml(url: string) {
+    const app = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY });
+    const scrapeResult = await app.scrapeUrl(url, {
+        pageOptions: {
+            includeHtml: true,
+            screenshot: true
+        }
+    });
+    return {
+        screenshotUrl: scrapeResult.data.metadata.screenshot,
+        htmlContent: scrapeResult.data.html
+    };
+}
+
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  const rawUrl = typeof req.query.url === 'string' ? req.query.url : null;
+  const spiceLevel = typeof req.query.spiceLevel === 'string' ? parseInt(req.query.spiceLevel, 10) : null;
+
+  if (!rawUrl || spiceLevel === null || isNaN(spiceLevel)) {
+    res.status(400).json({ error: 'Invalid query parameters' });
+    return;
+  }
+
+  try {
+    const { screenshotUrl, htmlContent } = await getScreenShotAndHtml(rawUrl);
+    
+    // Define a roast prompt message
+    let roastMessage: string;
+
+    // Determine the roast message based on the spice level
+    switch (spiceLevel) {
+      case 1:
+        roastMessage = mildPrompt;
+        break;
+      case 2:
+        roastMessage = mediumPrompt;
+        break;
+      case 3:
+        roastMessage = spicyPrompt;
+        break;
+      default:
+        // If an invalid spice level is provided, default to mild roast
+        roastMessage = mildPrompt;
+        res.status(400).json({ error: 'Invalid spice level' });
+        return;
+    }
+
+
+
+    // Convert HTML content to a markdown-like format for the roast generation
+    // This is a simplified conversion, assuming HTML content is already sanitized and suitable for direct usage
+
+    // Call the roastPrompt function to generate a roast
+    const roastResult = await roastPrompt(roastMessage, screenshotUrl, htmlContent);
+
+    // Log the roast result for debugging
+    // console.log("Roast Result:", roastResult);
+
+    res.status(200).json({ screenshotUrl, htmlContent, roastResult});
+  } catch (error: any) {
+    res.status(500).json({ error: error.message });
+  }
+}
+

examples/roastmywebsite/tailwind.config.ts

@@ -0,0 +1,137 @@
+import type { Config } from 'tailwindcss';
+import colors from 'tailwindcss/colors';
+
+const config: Config = {
+  darkMode: 'class',
+  content: [
+    './src/**/*.{js,ts,jsx,tsx}',
+
+    // Path to Tremor module
+    './node_modules/@tremor/**/*.{js,ts,jsx,tsx}',
+  ],
+  theme: {
+    transparent: 'transparent',
+    current: 'currentColor',
+    extend: {
+      colors: {
+        // light mode
+        tremor: {
+          brand: {
+            faint: colors.blue[50],
+            muted: colors.blue[200],
+            subtle: colors.blue[400],
+            DEFAULT: colors.blue[500],
+            emphasis: colors.blue[700],
+            inverted: colors.white,
+          },
+          background: {
+            muted: colors.gray[50],
+            subtle: colors.gray[100],
+            DEFAULT: colors.white,
+            emphasis: colors.gray[700],
+          },
+          border: {
+            DEFAULT: colors.gray[200],
+          },
+          ring: {
+            DEFAULT: colors.gray[200],
+          },
+          content: {
+            subtle: colors.gray[400],
+            DEFAULT: colors.gray[500],
+            emphasis: colors.gray[700],
+            strong: colors.gray[900],
+            inverted: colors.white,
+          },
+        },
+        // dark mode
+        'dark-tremor': {
+          brand: {
+            faint: '#0B1229',
+            muted: colors.blue[950],
+            subtle: colors.blue[800],
+            DEFAULT: colors.blue[500],
+            emphasis: colors.blue[400],
+            inverted: colors.blue[950],
+          },
+          background: {
+            muted: '#131A2B',
+            subtle: colors.gray[800],
+            DEFAULT: colors.gray[900],
+            emphasis: colors.gray[300],
+          },
+          border: {
+            DEFAULT: colors.gray[800],
+          },
+          ring: {
+            DEFAULT: colors.gray[800],
+          },
+          content: {
+            subtle: colors.gray[600],
+            DEFAULT: colors.gray[500],
+            emphasis: colors.gray[200],
+            strong: colors.gray[50],
+            inverted: colors.gray[950],
+          },
+        },
+      },
+      boxShadow: {
+        // light
+        'tremor-input': '0 1px 2px 0 rgb(0 0 0 / 0.05)',
+        'tremor-card':
+          '0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)',
+        'tremor-dropdown':
+          '0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)',
+        // dark
+        'dark-tremor-input': '0 1px 2px 0 rgb(0 0 0 / 0.05)',
+        'dark-tremor-card':
+          '0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1)',
+        'dark-tremor-dropdown':
+          '0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1)',
+      },
+      borderRadius: {
+        'tremor-small': '0.375rem',
+        'tremor-default': '0.5rem',
+        'tremor-full': '9999px',
+      },
+      fontSize: {
+        'tremor-label': ['0.75rem', { lineHeight: '1rem' }],
+        'tremor-default': ['0.875rem', { lineHeight: '1.25rem' }],
+        'tremor-title': ['1.125rem', { lineHeight: '1.75rem' }],
+        'tremor-metric': ['1.875rem', { lineHeight: '2.25rem' }],
+      },
+    },
+  },
+  safelist: [
+    {
+      pattern:
+        /^(bg-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+      variants: ['hover', 'ui-selected'],
+    },
+    {
+      pattern:
+        /^(text-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+      variants: ['hover', 'ui-selected'],
+    },
+    {
+      pattern:
+        /^(border-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+      variants: ['hover', 'ui-selected'],
+    },
+    {
+      pattern:
+        /^(ring-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+    },
+    {
+      pattern:
+        /^(stroke-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+    },
+    {
+      pattern:
+        /^(fill-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/,
+    },
+  ],
+  plugins: [require('@headlessui/tailwindcss'), require('@tailwindcss/forms')],
+};
+
+export default config;

examples/roastmywebsite/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}