github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Nick: fixed

Browse Source
This commit is contained in:
Nicolas 2024-05-17 10:40:44 -07:00
parent eb88447e8b
commit 5be208f595
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -584,7 +584,7 @@ describe("E2E Tests for API Routes", () => {
.post('/v0/crawl') .post('/v0/crawl')
.set('Authorization', `Bearer ${process.env.TEST_API_KEY}`) .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
.set('Content-Type', 'application/json') .set('Content-Type', 'application/json')
.send({ url: 'https://arxiv.org/abs/astro-ph/9301001', crawlerOptions: { limit: 10, excludes: [ 'list/*', 'login', 'abs/*', 'static/*', 'about/*', 'archive/*' ] }}); .send({ url: 'https://arxiv.org/pdf/astro-ph/9301001', crawlerOptions: { limit: 10, excludes: [ 'list/*', 'login', 'abs/*', 'static/*', 'about/*', 'archive/*' ] }});
expect(crawlResponse.statusCode).toBe(200); expect(crawlResponse.statusCode).toBe(200);
let isCompleted = false; let isCompleted = false;
@ -606,7 +606,7 @@ describe("E2E Tests for API Routes", () => {
} }
expect(completedResponse.body.status).toBe('completed'); expect(completedResponse.body.status).toBe('completed');
expect(completedResponse.body).toHaveProperty('data'); expect(completedResponse.body).toHaveProperty('data');
expect(completedResponse.body.data.length).toBeGreaterThan(1); expect(completedResponse.body.data.length).toEqual(1);
expect(completedResponse.body.data).toEqual( expect(completedResponse.body.data).toEqual(
expect.arrayContaining([ expect.arrayContaining([
expect.objectContaining({ expect.objectContaining({

2
apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts
View File

@ -80,7 +80,7 @@ export async function processPdfToText(filePath: string): Promise<string> {
await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds
} }
} catch (error) { } catch (error) {
console.error("Error fetching result:", error.data.detail || ''); console.error("Error fetching result:", error || '');
attempt++; attempt++;
await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds before retrying await new Promise((resolve) => setTimeout(resolve, 500)); // Wait for 0.5 seconds before retrying
// You may want to handle specific errors differently // You may want to handle specific errors differently