github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update index.test.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-08 13:13:38 -07:00
parent c50076c377
commit 6ced8e73a7
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/test-suite/index.test.ts
View File

@ -82,8 +82,12 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`; const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`;
let msg = null;
const msg = await openai.chat.completions.create({ const maxRetries = 3;
let attempts = 0;
while (!msg && attempts < maxRetries) {
try {
msg = await openai.chat.completions.create({
model: "gpt-4-turbo", model: "gpt-4-turbo",
max_tokens: 100, max_tokens: 100,
temperature: 0, temperature: 0,
@ -94,9 +98,18 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
}, },
], ],
}); });
} catch (error) {
console.error(`Attempt ${attempts + 1}: Failed to prompt for ${websiteData.website}, error: ${error}`);
attempts++;
if (attempts < maxRetries) {
console.log(`Retrying... Attempt ${attempts + 1}`);
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for 2 seconds before retrying
}
}
}
if (!msg) { if (!msg) {
console.error(`Failed to prompt for ${websiteData.website}`); console.error(`Failed to prompt for ${websiteData.website} after ${maxRetries} attempts`);
errorLog.push({ errorLog.push({
website: websiteData.website, website: websiteData.website,
prompt: websiteData.prompt, prompt: websiteData.prompt,