github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update index.test.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-08 13:13:38 -07:00
parent c50076c377
commit 6ced8e73a7
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
apps/test-suite/index.test.ts
View File

@ -81,22 +81,35 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
}); });
const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`; const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`;
const msg = await openai.chat.completions.create({ let msg = null;
model: "gpt-4-turbo", const maxRetries = 3;
max_tokens: 100, let attempts = 0;
temperature: 0, while (!msg && attempts < maxRetries) {
messages: [ try {
{ msg = await openai.chat.completions.create({
role: "user", model: "gpt-4-turbo",
content: prompt max_tokens: 100,
}, temperature: 0,
], messages: [
}); {
role: "user",
content: prompt
},
],
});
} catch (error) {
console.error(`Attempt ${attempts + 1}: Failed to prompt for ${websiteData.website}, error: ${error}`);
attempts++;
if (attempts < maxRetries) {
console.log(`Retrying... Attempt ${attempts + 1}`);
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for 2 seconds before retrying
}
}
}
if (!msg) { if (!msg) {
console.error(`Failed to prompt for ${websiteData.website}`); console.error(`Failed to prompt for ${websiteData.website} after ${maxRetries} attempts`);
errorLog.push({ errorLog.push({
website: websiteData.website, website: websiteData.website,
prompt: websiteData.prompt, prompt: websiteData.prompt,