0
v-firecrawl/apps/api/src/scraper/WebScraper/utils/utils.ts

24 lines
517 B
TypeScript
Raw Normal View History

2024-04-15 17:01:47 -04:00
import axios from "axios";
export async function attemptScrapWithRequests(
urlToScrap: string
): Promise<string | null> {
try {
const response = await axios.get(urlToScrap);
if (!response.data) {
console.log("Failed normal requests as well");
return null;
}
return response.data;
} catch (error) {
console.error(`Error in attemptScrapWithRequests: ${error}`);
return null;
}
}
export function sanitizeText(text: string): string {
return text.replace("\u0000", "");
}