github/v-firecrawl
github/v-firecrawl
0
Fork 0
Code Actions

Update index.test.ts

Browse Source
This commit is contained in:
Nicolas 2024-05-08 13:13:38 -07:00
parent c50076c377
commit 6ced8e73a7
1 changed files with 26 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
apps/test-suite/index.test.ts
View File

@ -82,21 +82,34 @@ describe("Scraping/Crawling Checkup (E2E)", () => {
const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`; const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`;
let msg = null;
const msg = await openai.chat.completions.create({ const maxRetries = 3;
model: "gpt-4-turbo", let attempts = 0;
max_tokens: 100, while (!msg && attempts < maxRetries) {
temperature: 0, try {
messages: [ msg = await openai.chat.completions.create({
{ model: "gpt-4-turbo",
role: "user", max_tokens: 100,
content: prompt temperature: 0,
}, messages: [
], {
}); role: "user",
content: prompt
},
],
});
} catch (error) {
console.error(`Attempt ${attempts + 1}: Failed to prompt for ${websiteData.website}, error: ${error}`);
attempts++;
if (attempts < maxRetries) {
console.log(`Retrying... Attempt ${attempts + 1}`);
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for 2 seconds before retrying
}
}
}
if (!msg) { if (!msg) {
console.error(`Failed to prompt for ${websiteData.website}`); console.error(`Failed to prompt for ${websiteData.website} after ${maxRetries} attempts`);
errorLog.push({ errorLog.push({
website: websiteData.website, website: websiteData.website,
prompt: websiteData.prompt, prompt: websiteData.prompt,