pyproject.toml

[project]
name = "etl"
version = "0.1.0"
description = ""
authors = [
    {name = "Our World in Data", email = "tech@ourworldindata.org"},
]
requires-python = ">=3.10, <3.13"
dependencies = [
    "click>=8.0.1",
    "PyYAML>=6.0.1",
    "papermill>=2.3.3",
    "jupyterlab>=3.1.13",
    # we had to pin this because of this bug https://github.com/owid/etl/issues/2320, versions newer than 2.2.4 might be working again
    "SPARQLWrapper>=1.8.5",
    "openpyxl>=3.0.9",
    "sh==1.14.3",
    "Unidecode>=1.3.2",
    "python-dotenv>=0.19.0",
    "regex>=2022.1.18",
    "wikipedia>=1.4.0",
    # sentence-transformers and torch don't yet support numpy 2
    "numpy>=1.22.1, <2.0.0",
    "pydantic>=1.9.0",
    "structlog>=21.5.0",
    "rich>=12.1.0",
    "rich-click>=1.7.3",
    "tenacity>=8.0.1",
    "simplejson>=3.17.6",
    "bugsnag>=4.2.1",
    "xlrd>=2.0.1",
    "PyPDF2>=2.11.1",
    "ruamel.yaml>=0.17.21",
    "gitpython>=3.1.30",
    "rapidfuzz>=2.13.7",
    "fsspec>=2022.11.0",
    "openai>=1.3.6",
    "pdfplumber>=0.9.0",
    "pyhumps>=3.8.0",
    "wbgapi>=1.0.12",
    "odfpy>=1.4.1",
    "world-bank-data>=0.1.3",
    "questionary>=2.0.0",
    "typing-extensions>=4.7.1",
    "scipy>=1.11.2",
    "selenium>=4.15.1",
    "geopandas>=0.14.1",
    "shapely>=2.0.3",
    "fasteners>=0.19",
    "cdsapi>=0.7.0",
    "rioxarray>=0.15.1",
    "html2text>=2020.1.16",
    "pygithub>=2.3.0",
    "pandas==2.2.3",
    "sqlalchemy>=2.0.30",
    "pymysql>=1.1.1",
    "tiktoken>=0.7.0",
    "earthengine-api>=0.1.411",
    "python-docx>=1.1.2",
    "h5netcdf>=1.3.0",
    "frictionless[pandas]>=5.0.3",
    "owid-catalog",
    "owid-datautils",
    "owid-repack",
    "walden",
    "deprecated>=1.2.14",
    "scikit-learn>=1.5.2",
    "geopy>=2.4.1",
    "py7zr>=0.22.0",
    "pyreadr>=0.5.2",
]

[tool.uv.sources]
owid-catalog = { path = "lib/catalog", editable = true }
owid-datautils = { path = "lib/datautils", editable = true }
owid-repack = { path = "lib/repack", editable = true }
walden = { path = "lib/walden", editable = true }
sqlacodegen = { git = "https://github.com/agronholm/sqlacodegen.git" }

[tool.uv]
dev-dependencies = [
    "pytest>=8.3.2",
    "watchdog>=4.0.1",
    "pyyaml>=6.0.2",
    "argh>=0.31.3",
    "jupyter>=1.0.0",
    "ipython>=8.14.0",
    "matplotlib>=3.9.1.post1",
    "plotly>=5.23.0",
    "cookiecutter>=2.6.0",
    "types-pyyaml>=6.0.12.20240808",
    "hydra-core>=1.3.2",
    "boto3-stubs[s3]>=1.34.154",
    "gspread>=5.12.4",
    "jsonref>=1.1.0",
    "mkdocs-material>=9.5.34",
    "mkdocs-jupyter>=0.24.8",
    "mkdocs-exclude>=1.0.2",
    "mkdocs-gen-files>=0.5.0",
    "mkdocs-git-authors-plugin>=0.9.2",
    "mkdocs-git-revision-date-localized-plugin>=1.2.6",
    "mkdocs-click>=0.8.1",
    "mkdocs-glightbox>=0.3.7",
    "memory-profiler>=0.61.0",
    "line-profiler>=4.1.3",
    "sqlacodegen",
    # unpinning those would introduce tons of type errors
    "pyright==1.1.373",
    "pandas-stubs==1.2.0.62",
    "ruff>=0.8.2",
    "ipdb>=0.13.13",
]

[project.optional-dependencies]
api = [
    "fastapi>=0.109.0",
    "uvicorn[standard]>=0.25.0",
    "slack-sdk>=3.26.2",
    "joblib>=1.3.2",
]
wizard = [
    "streamlit>=1.41.0",
    "streamlit-aggrid>=0.3.4.post3",
    "streamlit-ace>=0.1.1",
    "streamlit-extras>=0.3.6",
    "streamlit-agraph>=0.0.45",
    "plotly>=5.23.0",
    "geographiclib>=2.0",
    "pyproj>=3.6.1",
    "streamlit-feedback>=0.1.3",
    "statsmodels>=0.14.4",
    # Starting from PyTorch 2.3.0, the minimum requirement for macOS is macOS 11.0+ ARM64. Some poor people on the team still use it. Jeez...
    # Torch doesn't work with python 3.13
    # error: distribution torch==2.2.2 @ registry+https://pypi.org/simple can't be installed because it doesn't have a source distribution or wheel for the current platform
    "torch<2.3.0",
    "sentence-transformers>=2.2.2",
    "moviepy>=2.1.1",
    "pandas-gbq>=0.25.0",
]

[project.scripts]
etl = 'apps.cli:cli'
etlwiz = 'apps.wizard.cli:cli'
etlr = 'etl.command:main_cli'
etlp = 'apps.pr.cli:cli'
etl-wizard = 'apps.wizard.cli:cli'
compare = 'etl.compare:cli'
backport = 'apps.backport.backport:backport_cli'

[tool.ruff]
lint.extend-select = [
    # isort
    "I"
]
lint.ignore = ["E501"]
line-length = 120
target-version = "py310"
extend-exclude = [
    ".ipynb_checkpoints",
    "*cookiecutter",
    "*.ipynb",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.pyright]
exclude = [
    "lib/",
    "apps/wizard/etl_steps/cookiecutter/",
    "apps/wizard/etl_steps/cookiecutter/snapshot/**",
    "**/node_modules",
    "**/__pycache__",
    ".git",
    "**/.venv",
    "**/.ipynb_checkpoints",
    "playground/",
    ".cachedir/",
    ".cache/",
    "etl/steps",
    "snapshots/",
]

[tool.pytest.ini_options]
markers = [
    "integration: marks tests as integration tests (deselect with '-m \"not integration\"')"
]