commit
529e77d3e7
@ -134,15 +134,15 @@ app.post("/v0/scrape", async (req, res) => {
|
|||||||
filteredDocs.length
|
filteredDocs.length
|
||||||
);
|
);
|
||||||
if (!success) {
|
if (!success) {
|
||||||
// throw new Error("Failed to bill team, no subscribtion was found");
|
// throw new Error("Failed to bill team, no subscription was found");
|
||||||
// return {
|
// return {
|
||||||
// success: false,
|
// success: false,
|
||||||
// message: "Failed to bill team, no subscribtion was found",
|
// message: "Failed to bill team, no subscription was found",
|
||||||
// docs: [],
|
// docs: [],
|
||||||
// };
|
// };
|
||||||
return res
|
return res
|
||||||
.status(402)
|
.status(402)
|
||||||
.json({ error: "Failed to bill, no subscribtion was found" });
|
.json({ error: "Failed to bill, no subscription was found" });
|
||||||
}
|
}
|
||||||
return res.json({
|
return res.json({
|
||||||
success: true,
|
success: true,
|
||||||
|
@ -79,10 +79,10 @@ export async function runWebScraper({
|
|||||||
filteredDocs.length
|
filteredDocs.length
|
||||||
);
|
);
|
||||||
if (!success) {
|
if (!success) {
|
||||||
// throw new Error("Failed to bill team, no subscribtion was found");
|
// throw new Error("Failed to bill team, no subscription was found");
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
message: "Failed to bill team, no subscribtion was found",
|
message: "Failed to bill team, no subscription was found",
|
||||||
docs: [],
|
docs: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -3,7 +3,7 @@ import * as redis from "redis";
|
|||||||
|
|
||||||
const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5;
|
const MAX_REQUESTS_PER_MINUTE_PREVIEW = 5;
|
||||||
const MAX_CRAWLS_PER_MINUTE_STARTER = 2;
|
const MAX_CRAWLS_PER_MINUTE_STARTER = 2;
|
||||||
const MAX_CRAWLS_PER_MINUTE_STANDAR = 4;
|
const MAX_CRAWLS_PER_MINUTE_STANDARD = 4;
|
||||||
const MAX_CRAWLS_PER_MINUTE_SCALE = 20;
|
const MAX_CRAWLS_PER_MINUTE_SCALE = 20;
|
||||||
|
|
||||||
const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 40;
|
const MAX_REQUESTS_PER_MINUTE_ACCOUNT = 40;
|
||||||
@ -35,7 +35,7 @@ export function crawlRateLimit(plan: string){
|
|||||||
return new RateLimiterRedis({
|
return new RateLimiterRedis({
|
||||||
storeClient: redisClient,
|
storeClient: redisClient,
|
||||||
keyPrefix: "middleware",
|
keyPrefix: "middleware",
|
||||||
points: MAX_CRAWLS_PER_MINUTE_STANDAR,
|
points: MAX_CRAWLS_PER_MINUTE_STANDARD,
|
||||||
duration: 60, // Duration in seconds
|
duration: 60, // Duration in seconds
|
||||||
});
|
});
|
||||||
}else if(plan === "scale"){
|
}else if(plan === "scale"){
|
||||||
|
2
apps/playwright-service/.gitignore
vendored
2
apps/playwright-service/.gitignore
vendored
@ -145,7 +145,7 @@ dmypy.json
|
|||||||
cython_debug/
|
cython_debug/
|
||||||
|
|
||||||
# PyCharm
|
# PyCharm
|
||||||
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
Loading…
Reference in New Issue
Block a user