github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-06-18 13:21:50 -04:00
parent 818751a256
commit d0c05accf6
5 changed files with 28 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/controllers/search.ts
View File

@ -91,6 +91,7 @@ export async function searchHelper(
});
const docs = await a.getDocuments(false);
if (docs.length === 0) {
return { success: true, error: "No search results found", returnCode: 200 };
}

11
apps/js-sdk/firecrawl/package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.26",
"version": "0.0.28",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mendable/firecrawl-js",
"version": "0.0.26",
"version": "0.0.28",
"license": "MIT",
"dependencies": {
"axios": "^1.6.8",
@ -20,6 +20,7 @@
"@types/axios": "^0.14.0",
"@types/dotenv": "^8.2.0",
"@types/jest": "^29.5.12",
"@types/mocha": "^10.0.6",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
"jest": "^29.7.0",
@ -1071,6 +1072,12 @@
"pretty-format": "^29.0.0"
}
},
"node_modules/@types/mocha": {
"version": "10.0.6",
"resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.6.tgz",
"integrity": "sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg==",
"dev": true
},
"node_modules/@types/node": {
"version": "20.12.12",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz",

1
apps/js-sdk/firecrawl/package.json
View File

@ -33,6 +33,7 @@
"@types/axios": "^0.14.0",
"@types/dotenv": "^8.2.0",
"@types/jest": "^29.5.12",
"@types/mocha": "^10.0.6",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
"jest": "^29.7.0",

31
apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -2,6 +2,7 @@ import FirecrawlApp from '../../index';
import { v4 as uuidv4 } from 'uuid';
import dotenv from 'dotenv';
dotenv.config();
const TEST_API_KEY = process.env.TEST_API_KEY;
@ -29,14 +30,14 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
const response = await app.scrapeUrl('https://roastmywebsite.ai');
expect(response).not.toBeNull();
expect(response.data.content).toContain("_Roast_");
expect(response.data?.content).toContain("_Roast_");
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://roastmywebsite.ai');
expect(response).not.toBeNull();
expect(response.data.content).toContain("_Roast_");
expect(response.data?.content).toContain("_Roast_");
expect(response.data).toHaveProperty('markdown');
expect(response.data).toHaveProperty('metadata');
expect(response.data).not.toHaveProperty('html');
@ -46,23 +47,23 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://roastmywebsite.ai', { pageOptions: { includeHtml: true } });
expect(response).not.toBeNull();
expect(response.data.content).toContain("_Roast_");
expect(response.data.markdown).toContain("_Roast_");
expect(response.data.html).toContain("<h1");
expect(response.data?.content).toContain("_Roast_");
expect(response.data?.markdown).toContain("_Roast_");
expect(response.data?.html).toContain("<h1");
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
expect(response).not.toBeNull();
expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
}, 30000); // 30 seconds timeout
test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
expect(response).not.toBeNull();
expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
}, 30000); // 30 seconds timeout
test.concurrent('should throw error for invalid API key on crawl', async () => {
@ -112,15 +113,15 @@ describe('FirecrawlApp E2E Tests', () => {
expect(statusResponse).not.toBeNull();
expect(statusResponse.status).toBe('completed');
expect(statusResponse.data.length).toBeGreaterThan(0);
expect(statusResponse?.data?.length).toBeGreaterThan(0);
}, 35000); // 35 seconds timeout
test.concurrent('should return successful response for search', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.search("test query");
expect(response).not.toBeNull();
expect(response.data[0].content).toBeDefined();
expect(response.data.length).toBeGreaterThan(2);
expect(response?.data?.[0]?.content).toBeDefined();
expect(response?.data?.length).toBeGreaterThan(2);
}, 30000); // 30 seconds timeout
test.concurrent('should throw error for invalid API key on search', async () => {
@ -146,10 +147,10 @@ describe('FirecrawlApp E2E Tests', () => {
}
});
expect(response).not.toBeNull();
expect(response.data.llm_extraction).toBeDefined();
const llmExtraction = response.data.llm_extraction;
expect(llmExtraction.company_mission).toBeDefined();
expect(typeof llmExtraction.supports_sso).toBe('boolean');
expect(typeof llmExtraction.is_open_source).toBe('boolean');
expect(response.data?.llm_extraction).toBeDefined();
const llmExtraction = response.data?.llm_extraction;
expect(llmExtraction?.company_mission).toBeDefined();
expect(typeof llmExtraction?.supports_sso).toBe('boolean');
expect(typeof llmExtraction?.is_open_source).toBe('boolean');
}, 30000); // 30 seconds timeout
});

2
apps/js-sdk/firecrawl/src/__tests__/index.test.ts
View File

@ -43,6 +43,6 @@ describe('the firecrawl JS SDK', () => {
expect.objectContaining({ headers: expect.objectContaining({'Authorization': `Bearer ${apiKey}`}) }),
)
expect(scrapedData.success).toBe(true);
expect(scrapedData.data.metadata.title).toEqual('Mendable');
expect(scrapedData?.data?.metadata.title).toEqual('Mendable');
});
})