github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Change the way the playwright response is parsed

Browse Source 
Was failing with a Type Error, but actually looked ok.
This fixes the type error, and stop scraper fallback.
This commit is contained in:
Matt Joyce 2024-06-01 19:16:56 +10:00
parent 14896a9fdd
commit deefe65cbe
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/scraper/WebScraper/single_url.ts
View File

@ -146,12 +146,18 @@ export async function scrapWithPlaywright(url: string, waitFor: number = 0): Pro
if (contentType && contentType.includes('application/pdf')) { if (contentType && contentType.includes('application/pdf')) {
return fetchAndProcessPdf(url); return fetchAndProcessPdf(url);
} else { } else {
const data = await response.json(); const textData = await response.text();
try {
const data = JSON.parse(textData);
const html = data.content; const html = data.content;
return html ?? ""; return html ?? "";
} catch (jsonError) {
console.error(`[Playwright] Error parsing JSON response for url: ${url} -> ${jsonError}`);
return "";
}
} }
} catch (error) { } catch (error) {
console.error(`[Playwright][c] Error fetching url: ${url} -> ${error}`); console.error(`[Playwright] Error fetching url: ${url} -> ${error}`);
return ""; return "";
} }
} }