0
This commit is contained in:
Nicolas 2024-06-05 13:20:26 -07:00
parent 164676c70a
commit 5683bb2cc8
7 changed files with 332 additions and 9 deletions

View File

@ -53,3 +53,6 @@ BLOCK_MEDIA=
# Set this to the URL of your webhook when using the self-hosted version of FireCrawl # Set this to the URL of your webhook when using the self-hosted version of FireCrawl
SELF_HOSTED_WEBHOOK_URL= SELF_HOSTED_WEBHOOK_URL=
# Resend API Key for transactional emails
RESEND_API_KEY=

View File

@ -90,6 +90,7 @@
"puppeteer": "^22.6.3", "puppeteer": "^22.6.3",
"rate-limiter-flexible": "^2.4.2", "rate-limiter-flexible": "^2.4.2",
"redis": "^4.6.7", "redis": "^4.6.7",
"resend": "^3.2.0",
"robots-parser": "^3.0.1", "robots-parser": "^3.0.1",
"scrapingbee": "^1.7.4", "scrapingbee": "^1.7.4",
"stripe": "^12.2.0", "stripe": "^12.2.0",

View File

@ -146,6 +146,9 @@ dependencies:
redis: redis:
specifier: ^4.6.7 specifier: ^4.6.7
version: 4.6.14 version: 4.6.14
resend:
specifier: ^3.2.0
version: 3.2.0
robots-parser: robots-parser:
specifier: ^3.0.1 specifier: ^3.0.1
version: 3.0.1 version: 3.0.1
@ -1471,6 +1474,10 @@ packages:
- debug - debug
dev: false dev: false
/@one-ini/wasm@0.1.1:
resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}
dev: false
/@opentelemetry/api-logs@0.51.1: /@opentelemetry/api-logs@0.51.1:
resolution: {integrity: sha512-E3skn949Pk1z2XtXu/lxf6QAZpawuTM/IUEXcAzpiUkTd73Hmvw26FiN3cJuTmkpM5hZzHwkomVdtrh/n/zzwA==} resolution: {integrity: sha512-E3skn949Pk1z2XtXu/lxf6QAZpawuTM/IUEXcAzpiUkTd73Hmvw26FiN3cJuTmkpM5hZzHwkomVdtrh/n/zzwA==}
engines: {node: '>=14'} engines: {node: '>=14'}
@ -2522,6 +2529,16 @@ packages:
- supports-color - supports-color
dev: false dev: false
/@react-email/render@0.0.12:
resolution: {integrity: sha512-S8WRv/PqECEi6x0QJBj0asnAb5GFtJaHlnByxLETLkgJjc76cxMYDH4r9wdbuJ4sjkcbpwP3LPnVzwS+aIjT7g==}
engines: {node: '>=18.0.0'}
dependencies:
html-to-text: 9.0.5
js-beautify: 1.15.1
react: 18.2.0
react-dom: 18.2.0(react@18.2.0)
dev: false
/@redis/bloom@1.2.0(@redis/client@1.5.16): /@redis/bloom@1.2.0(@redis/client@1.5.16):
resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
peerDependencies: peerDependencies:
@ -2571,6 +2588,13 @@ packages:
'@redis/client': 1.5.16 '@redis/client': 1.5.16
dev: false dev: false
/@selderee/plugin-htmlparser2@0.11.0:
resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==}
dependencies:
domhandler: 5.0.3
selderee: 0.11.0
dev: false
/@sentry-internal/tracing@7.116.0: /@sentry-internal/tracing@7.116.0:
resolution: {integrity: sha512-y5ppEmoOlfr77c/HqsEXR72092qmGYS4QE5gSz5UZFn9CiinEwGfEorcg2xIrrCuU7Ry/ZU2VLz9q3xd04drRA==} resolution: {integrity: sha512-y5ppEmoOlfr77c/HqsEXR72092qmGYS4QE5gSz5UZFn9CiinEwGfEorcg2xIrrCuU7Ry/ZU2VLz9q3xd04drRA==}
engines: {node: '>=8'} engines: {node: '>=8'}
@ -3027,6 +3051,11 @@ packages:
resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}
engines: {node: '>=10.0.0'} engines: {node: '>=10.0.0'}
/abbrev@2.0.0:
resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
dev: false
/abort-controller@3.0.0: /abort-controller@3.0.0:
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
engines: {node: '>=6.5'} engines: {node: '>=6.5'}
@ -3669,6 +3698,13 @@ packages:
/concat-map@0.0.1: /concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
/config-chain@1.1.13:
resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==}
dependencies:
ini: 1.3.8
proto-list: 1.2.4
dev: false
/content-disposition@0.5.4: /content-disposition@0.5.4:
resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
engines: {node: '>= 0.6'} engines: {node: '>= 0.6'}
@ -3990,6 +4026,17 @@ packages:
/eastasianwidth@0.2.0: /eastasianwidth@0.2.0:
resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
/editorconfig@1.0.4:
resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==}
engines: {node: '>=14'}
hasBin: true
dependencies:
'@one-ini/wasm': 0.1.1
commander: 10.0.1
minimatch: 9.0.1
semver: 7.6.2
dev: false
/ee-first@1.1.1: /ee-first@1.1.1:
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
@ -4593,6 +4640,17 @@ packages:
resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
dev: true dev: true
/html-to-text@9.0.5:
resolution: {integrity: sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==}
engines: {node: '>=14'}
dependencies:
'@selderee/plugin-htmlparser2': 0.11.0
deepmerge: 4.3.1
dom-serializer: 2.0.0
htmlparser2: 8.0.2
selderee: 0.11.0
dev: false
/htmlparser2@8.0.2: /htmlparser2@8.0.2:
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
dependencies: dependencies:
@ -4701,6 +4759,10 @@ packages:
/inherits@2.0.4: /inherits@2.0.4:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
/ini@1.3.8:
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
dev: false
/ioredis@5.4.1: /ioredis@5.4.1:
resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==} resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==}
engines: {node: '>=12.22.0'} engines: {node: '>=12.22.0'}
@ -5313,6 +5375,23 @@ packages:
resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==} resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==}
dev: false dev: false
/js-beautify@1.15.1:
resolution: {integrity: sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==}
engines: {node: '>=14'}
hasBin: true
dependencies:
config-chain: 1.1.13
editorconfig: 1.0.4
glob: 10.4.1
js-cookie: 3.0.5
nopt: 7.2.1
dev: false
/js-cookie@3.0.5:
resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==}
engines: {node: '>=14'}
dev: false
/js-tiktoken@1.0.12: /js-tiktoken@1.0.12:
resolution: {integrity: sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==} resolution: {integrity: sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==}
dependencies: dependencies:
@ -5716,6 +5795,10 @@ packages:
engines: {node: '>= 0.4.8'} engines: {node: '>= 0.4.8'}
dev: false dev: false
/leac@0.6.0:
resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==}
dev: false
/leven@3.1.0: /leven@3.1.0:
resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==}
engines: {node: '>=6'} engines: {node: '>=6'}
@ -5818,6 +5901,13 @@ packages:
resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==}
dev: false dev: false
/loose-envify@1.4.0:
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
dependencies:
js-tokens: 4.0.0
dev: false
/lop@0.4.1: /lop@0.4.1:
resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==} resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==}
dependencies: dependencies:
@ -5961,6 +6051,13 @@ packages:
dependencies: dependencies:
brace-expansion: 2.0.1 brace-expansion: 2.0.1
/minimatch@9.0.1:
resolution: {integrity: sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==}
engines: {node: '>=16 || 14 >=14.17'}
dependencies:
brace-expansion: 2.0.1
dev: false
/minimatch@9.0.4: /minimatch@9.0.4:
resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==} resolution: {integrity: sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==}
engines: {node: '>=16 || 14 >=14.17'} engines: {node: '>=16 || 14 >=14.17'}
@ -6253,6 +6350,14 @@ packages:
undefsafe: 2.0.5 undefsafe: 2.0.5
dev: true dev: true
/nopt@7.2.1:
resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
hasBin: true
dependencies:
abbrev: 2.0.0
dev: false
/normalize-path@3.0.0: /normalize-path@3.0.0:
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
@ -6465,6 +6570,13 @@ packages:
entities: 4.5.0 entities: 4.5.0
dev: false dev: false
/parseley@0.12.1:
resolution: {integrity: sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==}
dependencies:
leac: 0.6.0
peberminta: 0.9.0
dev: false
/parseurl@1.3.3: /parseurl@1.3.3:
resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}
engines: {node: '>= 0.8'} engines: {node: '>= 0.8'}
@ -6505,6 +6617,10 @@ packages:
- supports-color - supports-color
dev: false dev: false
/peberminta@0.9.0:
resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==}
dev: false
/pend@1.2.0: /pend@1.2.0:
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
dev: false dev: false
@ -6700,6 +6816,10 @@ packages:
sisteransi: 1.0.5 sisteransi: 1.0.5
dev: true dev: true
/proto-list@1.2.4:
resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}
dev: false
/protobufjs@7.3.0: /protobufjs@7.3.0:
resolution: {integrity: sha512-YWD03n3shzV9ImZRX3ccbjqLxj7NokGN0V/ESiBV5xWqrommYHYiihuIyavq03pWSGqlyvYUFmfoMKd+1rPA/g==} resolution: {integrity: sha512-YWD03n3shzV9ImZRX3ccbjqLxj7NokGN0V/ESiBV5xWqrommYHYiihuIyavq03pWSGqlyvYUFmfoMKd+1rPA/g==}
engines: {node: '>=12.0.0'} engines: {node: '>=12.0.0'}
@ -6831,10 +6951,27 @@ packages:
iconv-lite: 0.4.24 iconv-lite: 0.4.24
unpipe: 1.0.0 unpipe: 1.0.0
/react-dom@18.2.0(react@18.2.0):
resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==}
peerDependencies:
react: ^18.2.0
dependencies:
loose-envify: 1.4.0
react: 18.2.0
scheduler: 0.23.2
dev: false
/react-is@18.3.1: /react-is@18.3.1:
resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==}
dev: true dev: true
/react@18.2.0:
resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==}
engines: {node: '>=0.10.0'}
dependencies:
loose-envify: 1.4.0
dev: false
/read-cmd-shim@4.0.0: /read-cmd-shim@4.0.0:
resolution: {integrity: sha512-yILWifhaSEEytfXI76kB9xEEiG1AiozaCJZ83A87ytjRiN+jVibXjedjCRNjoZviinhG+4UkalO3mWTd8u5O0Q==} resolution: {integrity: sha512-yILWifhaSEEytfXI76kB9xEEiG1AiozaCJZ83A87ytjRiN+jVibXjedjCRNjoZviinhG+4UkalO3mWTd8u5O0Q==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@ -6933,6 +7070,13 @@ packages:
- supports-color - supports-color
dev: false dev: false
/resend@3.2.0:
resolution: {integrity: sha512-lDHhexiFYPoLXy7zRlJ8D5eKxoXy6Tr9/elN3+Vv7PkUoYuSSD1fpiIfa/JYXEWyiyN2UczkCTLpkT8dDPJ4Pg==}
engines: {node: '>=18'}
dependencies:
'@react-email/render': 0.0.12
dev: false
/resolve-cwd@3.0.0: /resolve-cwd@3.0.0:
resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==}
engines: {node: '>=8'} engines: {node: '>=8'}
@ -7020,6 +7164,12 @@ packages:
sanitize-html: 2.13.0 sanitize-html: 2.13.0
dev: false dev: false
/scheduler@0.23.2:
resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==}
dependencies:
loose-envify: 1.4.0
dev: false
/scrapingbee@1.7.4: /scrapingbee@1.7.4:
resolution: {integrity: sha512-cTo+mfLi+T3mSeCHIefVZpjWEX2O70SkmCoWj9ypsnIFqBI2GmljdHYXt8yoT6D/YKjI0rHE7YH9iVRdhyoMmQ==} resolution: {integrity: sha512-cTo+mfLi+T3mSeCHIefVZpjWEX2O70SkmCoWj9ypsnIFqBI2GmljdHYXt8yoT6D/YKjI0rHE7YH9iVRdhyoMmQ==}
dependencies: dependencies:
@ -7030,6 +7180,12 @@ packages:
- debug - debug
dev: false dev: false
/selderee@0.11.0:
resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==}
dependencies:
parseley: 0.12.1
dev: false
/semver@5.7.2: /semver@5.7.2:
resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==}
hasBin: true hasBin: true

View File

@ -1,10 +1,11 @@
import { parseApi } from "../../src/lib/parseApi"; import { parseApi } from "../../src/lib/parseApi";
import { getRateLimiter, } from "../../src/services/rate-limiter"; import { getRateLimiter, } from "../../src/services/rate-limiter";
import { AuthResponse, RateLimiterMode } from "../../src/types"; import { AuthResponse, NotificationType, RateLimiterMode } from "../../src/types";
import { supabase_service } from "../../src/services/supabase"; import { supabase_service } from "../../src/services/supabase";
import { withAuth } from "../../src/lib/withAuth"; import { withAuth } from "../../src/lib/withAuth";
import { RateLimiterRedis } from "rate-limiter-flexible"; import { RateLimiterRedis } from "rate-limiter-flexible";
import { setTraceAttributes } from '@hyperdx/node-opentelemetry'; import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
import { sendNotification } from "../services/notification/email_notification";
export async function authenticateUser(req, res, mode?: RateLimiterMode): Promise<AuthResponse> { export async function authenticateUser(req, res, mode?: RateLimiterMode): Promise<AuthResponse> {
return withAuth(supaAuthenticateUser)(req, res, mode); return withAuth(supaAuthenticateUser)(req, res, mode);
@ -52,8 +53,11 @@ export async function supaAuthenticateUser(
let subscriptionData: { team_id: string, plan: string } | null = null; let subscriptionData: { team_id: string, plan: string } | null = null;
let normalizedApi: string; let normalizedApi: string;
let team_id: string;
if (token == "this_is_just_a_preview_token") { if (token == "this_is_just_a_preview_token") {
rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);
team_id = "preview";
} else { } else {
normalizedApi = parseApi(token); normalizedApi = parseApi(token);
@ -90,7 +94,9 @@ export async function supaAuthenticateUser(
status: 401, status: 401,
}; };
} }
const team_id = data[0].team_id; const internal_team_id = data[0].team_id;
team_id = internal_team_id;
const plan = getPlanByPriceId(data[0].price_id); const plan = getPlanByPriceId(data[0].price_id);
// HyperDX Logging // HyperDX Logging
setTrace(team_id, normalizedApi); setTrace(team_id, normalizedApi);
@ -124,12 +130,20 @@ export async function supaAuthenticateUser(
} }
} }
const team_endpoint_token = team_id;
try { try {
await rateLimiter.consume(iptoken); await rateLimiter.consume(team_endpoint_token);
} catch (rateLimiterRes) { } catch (rateLimiterRes) {
console.error(rateLimiterRes); console.error(rateLimiterRes);
const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1; const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1;
const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext); const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);
// We can only send a rate limit email every 7 days, send notification already has the date in between checking
const startDate = new Date();
const endDate = new Date();
endDate.setDate(endDate.getDate() + 7);
await sendNotification(team_id, NotificationType.RATE_LIMIT_REACHED, startDate.toISOString(), endDate.toISOString());
return { return {
success: false, success: false,
error: `Rate limit exceeded. Consumed points: ${rateLimiterRes.consumedPoints}, Remaining points: ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`, error: `Rate limit exceeded. Consumed points: ${rateLimiterRes.consumedPoints}, Remaining points: ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,

View File

@ -1,4 +1,6 @@
import { NotificationType } from "../../types";
import { withAuth } from "../../lib/withAuth"; import { withAuth } from "../../lib/withAuth";
import { sendNotification } from "../notification/email_notification";
import { supabase_service } from "../supabase"; import { supabase_service } from "../supabase";
const FREE_CREDITS = 500; const FREE_CREDITS = 500;
@ -187,12 +189,10 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
.select("credits_used") .select("credits_used")
.is("subscription_id", null) .is("subscription_id", null)
.eq("team_id", team_id); .eq("team_id", team_id);
// .gte("created_at", subscription.current_period_start)
// .lte("created_at", subscription.current_period_end);
if (creditUsageError) { if (creditUsageError) {
throw new Error( throw new Error(
`Failed to retrieve credit usage for subscription_id: ${subscription.id}` `Failed to retrieve credit usage for team_id: ${team_id}`
); );
} }
@ -202,8 +202,26 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
); );
console.log("totalCreditsUsed", totalCreditsUsed); console.log("totalCreditsUsed", totalCreditsUsed);
const end = new Date();
end.setDate(end.getDate() + 30);
// check if usage is within 80% of the limit
const creditLimit = FREE_CREDITS;
const creditUsagePercentage = (totalCreditsUsed + credits) / creditLimit;
console.log("creditUsagePercentage", creditUsagePercentage);
if (creditUsagePercentage >= 0.8) {
await sendNotification(team_id, NotificationType.APPROACHING_LIMIT, new Date().toISOString(), end.toISOString());
}
// 5. Compare the total credits used with the credits allowed by the plan. // 5. Compare the total credits used with the credits allowed by the plan.
if (totalCreditsUsed + credits > FREE_CREDITS) { if (totalCreditsUsed + credits > FREE_CREDITS) {
// Send email notification for insufficient credits
await sendNotification(team_id, NotificationType.LIMIT_REACHED, new Date().toISOString(), end.toISOString());
return { return {
success: false, success: false,
message: "Insufficient credits, please upgrade!", message: "Insufficient credits, please upgrade!",
@ -227,12 +245,11 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
if (creditUsages && creditUsages.length > 0) { if (creditUsages && creditUsages.length > 0) {
totalCreditsUsed = creditUsages[0].total_credits_used; totalCreditsUsed = creditUsages[0].total_credits_used;
// console.log("Total Credits Used:", totalCreditsUsed);
} }
} catch (error) { } catch (error) {
console.error("Error calculating credit usage:", error); console.error("Error calculating credit usage:", error);
} }
// Adjust total credits used by subtracting coupon value // Adjust total credits used by subtracting coupon value
const adjustedCreditsUsed = Math.max(0, totalCreditsUsed - couponCredits); const adjustedCreditsUsed = Math.max(0, totalCreditsUsed - couponCredits);
@ -247,9 +264,17 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
throw new Error(`Failed to retrieve price for price_id: ${subscription.price_id}`); throw new Error(`Failed to retrieve price for price_id: ${subscription.price_id}`);
} }
const creditLimit = price.credits;
const creditUsagePercentage = (adjustedCreditsUsed + credits) / creditLimit;
console.log("creditUsagePercentage", creditUsagePercentage);
// Compare the adjusted total credits used with the credits allowed by the plan // Compare the adjusted total credits used with the credits allowed by the plan
if (adjustedCreditsUsed + credits > price.credits) { if (adjustedCreditsUsed + credits > price.credits) {
await sendNotification(team_id, NotificationType.LIMIT_REACHED, subscription.current_period_start, subscription.current_period_end);
return { success: false, message: "Insufficient credits, please upgrade!" }; return { success: false, message: "Insufficient credits, please upgrade!" };
} else if (creditUsagePercentage >= 0.8) {
// Send email notification for approaching credit limit
await sendNotification(team_id, NotificationType.APPROACHING_LIMIT, subscription.current_period_start, subscription.current_period_end);
} }
return { success: true, message: "Sufficient credits available" }; return { success: true, message: "Sufficient credits available" };

View File

@ -0,0 +1,119 @@
import { supabase_service } from "../supabase";
import { withAuth } from "../../lib/withAuth";
import { Resend } from "resend";
import { NotificationType } from "../../types";
const resend = new Resend(process.env.RESEND_API_KEY);
const emailTemplates: Record<
NotificationType,
{ subject: string; html: string }
> = {
[NotificationType.APPROACHING_LIMIT]: {
subject: "You've used 80% of your credit limit - Firecrawl",
html: "Hey there,<br/><p>You are approaching your credit limit for this billing period. Your usage right now is around 80% of your total credit limit. Consider upgrading your plan to avoid hitting the limit. Check out our <a href='https://firecrawl.dev/pricing'>pricing page</a> for more info.</p><br/>Thanks,<br/>Firecrawl Team<br/>",
},
[NotificationType.LIMIT_REACHED]: {
subject: "Credit Limit Reached! Take action now to resume usage - Firecrawl",
html: "Hey there,<br/><p>You have reached your credit limit for this billing period. To resume usage, please upgrade your plan. Check out our <a href='https://firecrawl.dev/pricing'>pricing page</a> for more info.</p><br/>Thanks,<br/>Firecrawl Team<br/>",
},
[NotificationType.RATE_LIMIT_REACHED]: {
subject: "Rate Limit Reached - Firecrawl",
html: "Hey there,<br/><p>You've hit your Firecrawl rate limit! Take a breather and try again in a few moments. If you need more higher rate limits, consider upgrading your plan. Check out our <a href='https://firecrawl.dev/pricing'>pricing page</a> for more info.</p><p>If you have any questions, feel free to reach out to us at <a href='mailto:hello@firecrawl.com'>hello@firecrawl.com</a></p><br/>Thanks,<br/>Firecrawl Team<br/>Ps. this email is only sent once every 7 days if you reach the limit.",
},
};
export async function sendNotification(
team_id: string,
notificationType: NotificationType,
startDateString: string,
endDateString: string
) {
return withAuth(sendNotificationInternal)(
team_id,
notificationType,
startDateString,
endDateString
);
}
async function sendEmailNotification(
email: string,
notificationType: NotificationType
) {
try {
const { data, error } = await resend.emails.send({
from: "Firecrawl <firecrawl@getmendableai.com>",
to: [email],
reply_to: "hello@firecrawl.com",
subject: emailTemplates[notificationType].subject,
html: emailTemplates[notificationType].html,
});
if (error) {
console.error("Error sending email: ", error);
return { success: false };
}
} catch (error) {
console.error("Error sending email (2): ", error);
return { success: false };
}
}
export async function sendNotificationInternal(
team_id: string,
notificationType: NotificationType,
startDateString: string,
endDateString: string
): Promise<{ success: boolean }> {
const { data, error } = await supabase_service
.from("user_notifications")
.select("*")
.eq("team_id", team_id)
.eq("notification_type", notificationType)
.gte("sent_date", startDateString)
.lte("sent_date", endDateString);
if (error) {
console.error("Error fetching notifications: ", error);
return { success: false };
}
if (data.length !== 0) {
return { success: false };
} else {
// get the emails from the user with the team_id
const { data: emails, error: emailsError } = await supabase_service
.from("users")
.select("email")
.eq("team_id", team_id);
if (emailsError) {
console.error("Error fetching emails: ", emailsError);
return { success: false };
}
for (const email of emails) {
await sendEmailNotification(email.email, notificationType);
}
const { error: insertError } = await supabase_service
.from("user_notifications")
.insert([
{
team_id: team_id,
notification_type: notificationType,
sent_date: new Date().toISOString(),
},
]);
if (insertError) {
console.error("Error inserting notification record: ", insertError);
return { success: false };
}
return { success: true };
}
}

View File

@ -61,3 +61,8 @@ export interface AuthResponse {
} }
export enum NotificationType {
APPROACHING_LIMIT = "approachingLimit",
LIMIT_REACHED = "limitReached",
RATE_LIMIT_REACHED = "rateLimitReached",
}