github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update replacePaths.test.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-11 13:02:39 -07:00
parent 2239e03269
commit 1e3e06a1d5
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
apps/api/src/scraper/WebScraper/utils/__tests__/replacePaths.test.ts
View File

@ -6,12 +6,14 @@ describe('replacePaths', () => {
it('should replace relative paths with absolute paths', () => { it('should replace relative paths with absolute paths', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'This is a [link](/path/to/resource).' content: 'This is a [link](/path/to/resource).',
markdown: 'This is a [link](/path/to/resource).'
}]; }];
const expectedDocuments: Document[] = [{ const expectedDocuments: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'This is a [link](https://example.com/path/to/resource).' content: 'This is a [link](https://example.com/path/to/resource).',
markdown: 'This is a [link](https://example.com/path/to/resource).'
}]; }];
const result = replacePathsWithAbsolutePaths(documents); const result = replacePathsWithAbsolutePaths(documents);
@ -21,7 +23,8 @@ describe('replacePaths', () => {
it('should not alter absolute URLs', () => { it('should not alter absolute URLs', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'This is an [external link](https://external.com/path).' content: 'This is an [external link](https://external.com/path).',
markdown: 'This is an [external link](https://external.com/path).'
}]; }];
const result = replacePathsWithAbsolutePaths(documents); const result = replacePathsWithAbsolutePaths(documents);
@ -31,7 +34,8 @@ describe('replacePaths', () => {
it('should not alter data URLs for images', () => { it('should not alter data URLs for images', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'This is an image: ![alt text](data:image/png;base64,ABC123==).' content: 'This is an image: ![alt text](data:image/png;base64,ABC123==).',
markdown: 'This is an image: ![alt text](data:image/png;base64,ABC123==).'
}]; }];
const result = replacePathsWithAbsolutePaths(documents); const result = replacePathsWithAbsolutePaths(documents);
@ -41,12 +45,14 @@ describe('replacePaths', () => {
it('should handle multiple links and images correctly', () => { it('should handle multiple links and images correctly', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Here are two links: [link1](/path1) and [link2](/path2).' content: 'Here are two links: [link1](/path1) and [link2](/path2).',
markdown: 'Here are two links: [link1](/path1) and [link2](/path2).'
}]; }];
const expectedDocuments: Document[] = [{ const expectedDocuments: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2).' content: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2).',
markdown: 'Here are two links: [link1](https://example.com/path1) and [link2](https://example.com/path2).'
}]; }];
const result = replacePathsWithAbsolutePaths(documents); const result = replacePathsWithAbsolutePaths(documents);
@ -56,12 +62,14 @@ describe('replacePaths', () => {
it('should correctly handle a mix of absolute and relative paths', () => { it('should correctly handle a mix of absolute and relative paths', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).' content: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).',
markdown: 'Mixed paths: [relative](/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
}]; }];
const expectedDocuments: Document[] = [{ const expectedDocuments: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).' content: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).',
markdown: 'Mixed paths: [relative](https://example.com/path), [absolute](https://example.com/path), and [data image](data:image/png;base64,ABC123==).'
}]; }];
const result = replacePathsWithAbsolutePaths(documents); const result = replacePathsWithAbsolutePaths(documents);
@ -74,12 +82,14 @@ describe('replacePaths', () => {
it('should replace relative image paths with absolute paths', () => { it('should replace relative image paths with absolute paths', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Here is an image: ![alt text](/path/to/image.jpg).' content: 'Here is an image: ![alt text](/path/to/image.jpg).',
markdown: 'Here is an image: ![alt text](/path/to/image.jpg).'
}]; }];
const expectedDocuments: Document[] = [{ const expectedDocuments: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Here is an image: ![alt text](https://example.com/path/to/image.jpg).' content: 'Here is an image: ![alt text](https://example.com/path/to/image.jpg).',
markdown: 'Here is an image: ![alt text](https://example.com/path/to/image.jpg).'
}]; }];
const result = replaceImgPathsWithAbsolutePaths(documents); const result = replaceImgPathsWithAbsolutePaths(documents);
@ -89,7 +99,8 @@ describe('replacePaths', () => {
it('should not alter data:image URLs', () => { it('should not alter data:image URLs', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'An image with a data URL: ![alt text](data:image/png;base64,ABC123==).' content: 'An image with a data URL: ![alt text](data:image/png;base64,ABC123==).',
markdown: 'An image with a data URL: ![alt text](data:image/png;base4,ABC123==).'
}]; }];
const result = replaceImgPathsWithAbsolutePaths(documents); const result = replaceImgPathsWithAbsolutePaths(documents);
@ -99,12 +110,14 @@ describe('replacePaths', () => {
it('should handle multiple images with a mix of data and relative URLs', () => { it('should handle multiple images with a mix of data and relative URLs', () => {
const documents: Document[] = [{ const documents: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Multiple images: ![img1](/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](/img3.jpg).' content: 'Multiple images: ![img1](/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](/img3.jpg).',
markdown: 'Multiple images: ![img1](/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](/img3.jpg).'
}]; }];
const expectedDocuments: Document[] = [{ const expectedDocuments: Document[] = [{
metadata: { sourceURL: 'https://example.com' }, metadata: { sourceURL: 'https://example.com' },
content: 'Multiple images: ![img1](https://example.com/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](https://example.com/img3.jpg).' content: 'Multiple images: ![img1](https://example.com/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](https://example.com/img3.jpg).',
markdown: 'Multiple images: ![img1](https://example.com/img1.jpg) ![img2](data:image/png;base64,ABC123==) ![img3](https://example.com/img3.jpg).'
}]; }];
const result = replaceImgPathsWithAbsolutePaths(documents); const result = replaceImgPathsWithAbsolutePaths(documents);