From dec225d36815ec04523134b2012f2733a0703065 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Tue, 28 May 2024 20:08:41 +1000 Subject: [PATCH 1/3] Move version to __init__.py Setup.py does not need to be edited when building the package. --- apps/python-sdk/firecrawl/__init__.py | 2 ++ apps/python-sdk/setup.py | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index e7f8063..6899915 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -1 +1,3 @@ from .firecrawl import FirecrawlApp + +__version__ = "0.0.11" diff --git a/apps/python-sdk/setup.py b/apps/python-sdk/setup.py index beee059..4978559 100644 --- a/apps/python-sdk/setup.py +++ b/apps/python-sdk/setup.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from setuptools import find_packages, setup @@ -5,9 +6,19 @@ from setuptools import find_packages, setup this_directory = Path(__file__).parent long_description_content = (this_directory / "README.md").read_text() + +def get_version(): + """Dynamically set version""" + version_file = (this_directory / "firecrawl" / "__init__.py").read_text() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + setup( name="firecrawl-py", - version="0.0.11", + version=get_version(), url="https://github.com/mendableai/firecrawl", author="Mendable.ai", author_email="nick@mendable.ai", @@ -20,7 +31,7 @@ setup( 'pytest', 'python-dotenv', ], - python_requires='>=3.8', + python_requires=">=3.8", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -41,7 +52,7 @@ setup( "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Text Processing", "Topic :: Text Processing :: Indexing", - ], + ], keywords="SDK API firecrawl", project_urls={ "Documentation": "https://docs.firecrawl.dev", From 5c4b3e8f8a0c01c4104c6b1106cec668e119f725 Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Tue, 28 May 2024 20:12:44 +1000 Subject: [PATCH 2/3] Initial pyproject.toml This will enable building using 'python -m build', without impacting the utility of setup.py, also provide a base for other build tools and automation. --- apps/python-sdk/pyproject.toml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 apps/python-sdk/pyproject.toml diff --git a/apps/python-sdk/pyproject.toml b/apps/python-sdk/pyproject.toml new file mode 100644 index 0000000..25fa7c1 --- /dev/null +++ b/apps/python-sdk/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +dynamic = ["version"] +name = "firecrawl-py" +description = "Python SDK for Firecrawl API" +readme = {file="README.md", content-type = "text/markdown"} +requires-python = ">=3.8" +dependencies = [ + "requests", +] +authors = [{name = "Mendable.ai",email = "nick@mendable.ai"}] +maintainers = [{name = "Mendable.ai",email = "nick@mendable.ai"}] +license = {text = "GNU General Public License v3 (GPLv3)"} + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Internet", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: Indexing/Search", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Text Processing", + "Topic :: Text Processing :: Indexing", +] + +keywords = ["SDK", "API", "firecrawl"] + +[project.urls] +"Documentation" = "https://docs.firecrawl.dev" +"Source" = "https://github.com/mendableai/firecrawl" +"Tracker" = "https://github.com/mendableai/firecrawl/issues" + +[tool.setuptools.packages.find] +where = ["."] \ No newline at end of file From 9f8792f00c1469a2037a4d5caa1c59558611654d Mon Sep 17 00:00:00 2001 From: Matt Joyce Date: Thu, 30 May 2024 09:02:55 +1000 Subject: [PATCH 3/3] Script to check local vs published versions This script is for use with Github workflows --- .../scripts/check_version_has_incremented.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/scripts/check_version_has_incremented.py diff --git a/.github/scripts/check_version_has_incremented.py b/.github/scripts/check_version_has_incremented.py new file mode 100644 index 0000000..87e5cc6 --- /dev/null +++ b/.github/scripts/check_version_has_incremented.py @@ -0,0 +1,88 @@ +""" +checks local verions against published verions. + +Usage: + +python .github\scripts\check_version_has_incremented.py js .\apps\js-sdk\firecrawl @mendable/firecrawl-js +Local version: 0.0.22 +Published version: 0.0.21 +true + +python .github\scripts\check_version_has_incremented.py python .\apps\python-sdk\firecrawl firecrawl-py +Local version: 0.0.11 +Published version: 0.0.11 +false + +""" +import json +import os +import re +import sys +from pathlib import Path + +import requests +from packaging.version import Version +from packaging.version import parse as parse_version + + +def get_python_version(file_path: str) -> str: + """Extract version string from Python file.""" + version_file = Path(file_path).read_text() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) + if version_match: + return version_match.group(1).strip() + raise RuntimeError("Unable to find version string.") + +def get_pypi_version(package_name: str) -> str: + """Get latest version of Python package from PyPI.""" + response = requests.get(f"https://pypi.org/pypi/{package_name}/json") + version = response.json()['info']['version'] + return version.strip() + +def get_js_version(file_path: str) -> str: + """Extract version string from package.json.""" + with open(file_path, 'r') as file: + package_json = json.load(file) + if 'version' in package_json: + return package_json['version'].strip() + raise RuntimeError("Unable to find version string in package.json.") + +def get_npm_version(package_name: str) -> str: + """Get latest version of JavaScript package from npm.""" + response = requests.get(f"https://registry.npmjs.org/{package_name}/latest") + version = response.json()['version'] + return version.strip() + +def is_version_incremented(local_version: str, published_version: str) -> bool: + """Compare local and published versions.""" + local_version_parsed: Version = parse_version(local_version) + published_version_parsed: Version = parse_version(published_version) + return local_version_parsed > published_version_parsed + +if __name__ == "__main__": + package_type = sys.argv[1] + package_path = sys.argv[2] + package_name = sys.argv[3] + + if package_type == "python": + # Get current version from __init__.py + current_version = get_python_version(os.path.join(package_path, '__init__.py')) + # Get published version from PyPI + published_version = get_pypi_version(package_name) + elif package_type == "js": + # Get current version from package.json + current_version = get_js_version(os.path.join(package_path, 'package.json')) + # Get published version from npm + published_version = get_npm_version(package_name) + else: + raise ValueError("Invalid package type. Use 'python' or 'js'.") + + # Print versions for debugging + print(f"Local version: {current_version}") + print(f"Published version: {published_version}") + + # Compare versions and print result + if is_version_incremented(current_version, published_version): + print("true") + else: + print("false")