From 1de24029c87470dbf7954af11348b5f79ab68df7 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 5 Nov 2018 20:09:20 +0000 Subject: [PATCH 1/7] Unifying dev and optional dependencies, moving them to their standard locations, and create script to generate the pip file, and make sure in the CI they are always sync --- ci/code_checks.sh | 22 +++-- ci/environment-dev.yaml | 19 ---- ci/requirements-optional-pip.txt | 30 ------ ci/requirements_dev.txt | 15 --- doc/source/contributing.rst | 11 +-- environment.yml | 52 ++++++++++ ...nts-optional-conda.txt => requirements.txt | 15 ++- scripts/conda_to_pip.py | 96 +++++++++++++++++++ scripts/convert_deps.py | 31 ------ 9 files changed, 181 insertions(+), 110 deletions(-) delete mode 100644 ci/environment-dev.yaml delete mode 100644 ci/requirements-optional-pip.txt delete mode 100644 ci/requirements_dev.txt create mode 100644 environment.yml rename ci/requirements-optional-conda.txt => requirements.txt (63%) create mode 100755 scripts/conda_to_pip.py delete mode 100755 scripts/convert_deps.py diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 2e42912965f97..707f9cf6e2a34 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -9,16 +9,19 @@ # In the future we may want to add the validation of docstrings and other checks here. # # Usage: -# $ ./ci/code_checks.sh # run all checks -# $ ./ci/code_checks.sh lint # run linting only -# $ ./ci/code_checks.sh patterns # check for patterns that should not exist -# $ ./ci/code_checks.sh doctests # run doctests +# $ ./ci/code_checks.sh # run all checks +# $ ./ci/code_checks.sh lint # run linting only +# $ ./ci/code_checks.sh patterns # check for patterns that should not exist +# $ ./ci/code_checks.sh doctests # run doctests +# $ ./ci/code_checks.sh dependencies # check that dependencies are consistent echo "inside $0" [[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; } -[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; } +[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "dependencies" ]] \ + || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|dependencies]"; exit 9999; } -source activate pandas +#source activate pandas +BASE_DIR="$(dirname $0)/.." RET=0 CHECK=$1 @@ -162,4 +165,11 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then fi +### DEPENDENCIES ### +if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then + MSG='Check that requirements.txt has been generated from environment.yml' ; echo $MSG + $BASE_DIR/scripts/conda_to_pip.py --compare + RET=$(($RET + $?)) ; echo $MSG "DONE" +fi + exit $RET diff --git a/ci/environment-dev.yaml b/ci/environment-dev.yaml deleted file mode 100644 index 3e69b1f725b24..0000000000000 --- a/ci/environment-dev.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - Cython>=0.28.2 - - NumPy - - flake8 - - flake8-comprehensions - - hypothesis>=3.58.0 - - isort - - moto - - pytest>=3.6 - - python-dateutil>=2.5.0 - - python=3 - - pytz - - setuptools>=24.2.0 - - sphinx - - sphinxcontrib-spelling diff --git a/ci/requirements-optional-pip.txt b/ci/requirements-optional-pip.txt deleted file mode 100644 index 347ea0d9832b0..0000000000000 --- a/ci/requirements-optional-pip.txt +++ /dev/null @@ -1,30 +0,0 @@ -# This file was autogenerated by scripts/convert_deps.py -# Do not modify directly -beautifulsoup4>=4.2.1 -blosc -bottleneck>=1.2.0 -fastparquet -gcsfs -html5lib -ipython>=5.6.0 -ipykernel -jinja2 -lxml -matplotlib>=2.0.0 -nbsphinx -numexpr>=2.6.1 -openpyxl -pyarrow>=0.4.1 -pymysql -tables -pytest-cov -pytest-xdist -s3fs -scipy>=0.18.1 -seaborn -sqlalchemy -statsmodels -xarray -xlrd -xlsxwriter -xlwt diff --git a/ci/requirements_dev.txt b/ci/requirements_dev.txt deleted file mode 100644 index 6a8b8d64d943b..0000000000000 --- a/ci/requirements_dev.txt +++ /dev/null @@ -1,15 +0,0 @@ -# This file was autogenerated by scripts/convert_deps.py -# Do not modify directly -Cython>=0.28.2 -NumPy -flake8 -flake8-comprehensions -hypothesis>=3.58.0 -isort -moto -pytest>=3.6 -python-dateutil>=2.5.0 -pytz -setuptools>=24.2.0 -sphinx -sphinxcontrib-spelling \ No newline at end of file diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 3ec505998fde0..0a7790a601b50 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -170,7 +170,7 @@ We'll now kick off a three-step process: .. code-block:: none # Create and activate the build environment - conda env create -f ci/environment-dev.yaml + conda env create conda activate pandas-dev # or with older versions of Anaconda: @@ -180,9 +180,6 @@ We'll now kick off a three-step process: python setup.py build_ext --inplace -j 4 python -m pip install -e . - # Install the rest of the optional dependencies - conda install -c defaults -c conda-forge --file=ci/requirements-optional-conda.txt - At this point you should be able to import pandas from your locally built version:: $ python # start an interpreter @@ -221,14 +218,12 @@ You'll need to have at least python3.5 installed on your system. . ~/virtualenvs/pandas-dev/bin/activate # Install the build dependencies - python -m pip install -r ci/requirements_dev.txt + python -m pip install -r requirements.txt + # Build and install pandas python setup.py build_ext --inplace -j 4 python -m pip install -e . - # Install additional dependencies - python -m pip install -r ci/requirements-optional-pip.txt - Creating a branch ----------------- diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000000000..276306716e989 --- /dev/null +++ b/environment.yml @@ -0,0 +1,52 @@ +name: pandas-dev +channels: + - defaults + - conda-forge +dependencies: + # required + - NumPy + - python=3 + - python-dateutil>=2.5.0 + - pytz + + # development + - Cython>=0.28.2 + - flake8 + - flake8-comprehensions + - hypothesis>=3.58.0 + - isort + - moto + - pytest>=3.6 + - setuptools>=24.2.0 + - sphinx + - sphinxcontrib-spelling + + # optional + - beautifulsoup4>=4.2.1 + - blosc + - bottleneck>=1.2.0 + - fastparquet + - gcsfs + - html5lib + - ipython>=5.6.0 + - ipykernel + - jinja2 + - lxml + - matplotlib>=2.0.0 + - nbsphinx + - numexpr>=2.6.1 + - openpyxl + - pyarrow>=0.4.1 + - pymysql + - pytables>=3.4.2 + - pytest-cov + - pytest-xdist + - s3fs + - scipy>=0.18.1 + - seaborn + - sqlalchemy + - statsmodels + - xarray + - xlrd + - xlsxwriter + - xlwt diff --git a/ci/requirements-optional-conda.txt b/requirements.txt similarity index 63% rename from ci/requirements-optional-conda.txt rename to requirements.txt index c9dc385b87986..6ef504c4af0bb 100644 --- a/ci/requirements-optional-conda.txt +++ b/requirements.txt @@ -1,3 +1,16 @@ +NumPy +python-dateutil>=2.5.0 +pytz +Cython>=0.28.2 +flake8 +flake8-comprehensions +hypothesis>=3.58.0 +isort +moto +pytest>=3.6 +setuptools>=24.2.0 +sphinx +sphinxcontrib-spelling beautifulsoup4>=4.2.1 blosc bottleneck>=1.2.0 @@ -25,4 +38,4 @@ statsmodels xarray xlrd xlsxwriter -xlwt +xlwt \ No newline at end of file diff --git a/scripts/conda_to_pip.py b/scripts/conda_to_pip.py new file mode 100755 index 0000000000000..3578ef7294f76 --- /dev/null +++ b/scripts/conda_to_pip.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +""" +Convert the conda environment.yml to the pip requirements.txt, +or check that they have the same packages (for the CI) + +Usage: + + Generate `requirements.txt` + $ ./conda_to_pip + + Compare and fail (exit status != 0) if `requirements.txt` has not been + generated with this script: + $ ./conda_to_pip --compare +""" +import argparse +import os +import re +import sys +import yaml + + +EXCLUDE = {'python=3'} +RENAME = {'pytables': 'tables'} + + +def conda_package_to_pip(package): + """ + Convert a conda package to its pip equivalent. + + In most cases they are the same, those are the exceptions: + - Packages that should be excluded (in `EXCLUDE`) + - Packages that should be renamed (in `RENAME`) + - A package requiring a specific version, in conda is defined with a single + equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``) + """ + if package in EXCLUDE: + return + + if package in RENAME: + return RENAME[package] + + return re.sub('(?<=[^<>])=', '==', package).strip() + + +def main(conda_fname, pip_fname, compare=False): + """ + Generate the pip dependencies file from the conda file, or compare that + they are synchronized (``compare=True``). + + Parameters + ---------- + conda_fname : str + Path to the conda file with dependencies (e.g. `environment.yml`). + pip_fname : str + Path to the pip file with dependencies (e.g. `requirements.txt`). + compare : bool, default False + Whether to generate the pip file (``False``) or to compare if the + pip file has been generated with this script and the last version + of the conda file (``True``). + + Returns + ------- + bool + True if the comparison fails, False otherwise + """ + with open(conda_fname) as conda_fd: + deps = yaml.load(conda_fd)['dependencies'] + + pip_content = '\n'.join(filter(None, map(conda_package_to_pip, deps))) + + if compare: + with open(pip_fname) as pip_fd: + return pip_content != pip_fd.read() + else: + with open(pip_fname, 'w') as pip_fd: + pip_fd.write(pip_content) + return False + + +if __name__ == '__main__': + argparser = argparse.ArgumentParser( + description='convert (or compare) conda file to pip') + argparser.add_argument('--compare', + action='store_true', + help='compare whether the two files are equivalent') + args = argparser.parse_args() + + repo_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) + res = main(os.path.join(repo_path, 'environment.yml'), + os.path.join(repo_path, 'requirements.txt'), + compare=args.compare) + if res: + sys.stderr.write('`requirements.txt` has to be generated with `{}` ' + 'after `environment.yml` is modified.\n'.format( + sys.argv[0])) + sys.exit(res) diff --git a/scripts/convert_deps.py b/scripts/convert_deps.py deleted file mode 100755 index 3ff157e0a0d7b..0000000000000 --- a/scripts/convert_deps.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Convert the conda environment.yaml to a pip requirements.txt -""" -import re -import yaml - -exclude = {'python=3'} -rename = {'pytables': 'tables'} - -with open("ci/environment-dev.yaml") as f: - dev = yaml.load(f) - -with open("ci/requirements-optional-conda.txt") as f: - optional = [x.strip() for x in f.readlines()] - -required = dev['dependencies'] -required = [rename.get(dep, dep) for dep in required if dep not in exclude] -optional = [rename.get(dep, dep) for dep in optional if dep not in exclude] -optional = [re.sub("(?<=[^<>])=", '==', dep) for dep in optional] - - -with open("ci/requirements_dev.txt", 'wt') as f: - f.write("# This file was autogenerated by scripts/convert_deps.py\n") - f.write("# Do not modify directly\n") - f.write('\n'.join(required)) - - -with open("ci/requirements-optional-pip.txt", 'wt') as f: - f.write("# This file was autogenerated by scripts/convert_deps.py\n") - f.write("# Do not modify directly\n") - f.write("\n".join(optional)) From 826482fb99be4582c6ec75ba5803e7243269b7ca Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Mon, 5 Nov 2018 20:40:32 +0000 Subject: [PATCH 2/7] using yaml.safe_load instead of yaml.load (just in case :) --- scripts/conda_to_pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/conda_to_pip.py b/scripts/conda_to_pip.py index 3578ef7294f76..6b38cfebd4b17 100755 --- a/scripts/conda_to_pip.py +++ b/scripts/conda_to_pip.py @@ -64,7 +64,7 @@ def main(conda_fname, pip_fname, compare=False): True if the comparison fails, False otherwise """ with open(conda_fname) as conda_fd: - deps = yaml.load(conda_fd)['dependencies'] + deps = yaml.safe_load(conda_fd)['dependencies'] pip_content = '\n'.join(filter(None, map(conda_package_to_pip, deps))) From 84601da169d10b3c388269752afa88d6d3a8169a Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 6 Nov 2018 10:32:32 +0000 Subject: [PATCH 3/7] Restoring sourcing of the evironment (commented for local testing) --- ci/code_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index ae4f4ac55d84d..7fae4b50ebbad 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -20,7 +20,7 @@ echo "inside $0" [[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "dependencies" ]] \ || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|dependencies]"; exit 9999; } -#source activate pandas +source activate pandas BASE_DIR="$(dirname $0)/.." RET=0 CHECK=$1 From 1e716efa6532ad74f7e2b8e697dad9c370c79c26 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Tue, 6 Nov 2018 13:40:58 +0000 Subject: [PATCH 4/7] Making explicit the conda requirements file in the installation instructions --- doc/source/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 0a7790a601b50..d9c543e9d4521 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -170,7 +170,7 @@ We'll now kick off a three-step process: .. code-block:: none # Create and activate the build environment - conda env create + conda env create -f environment.yml conda activate pandas-dev # or with older versions of Anaconda: From c8658e1876a37a38a6af1238a74e2a45243f6a45 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 01:20:32 +0000 Subject: [PATCH 5/7] Updating requirements.txt with new dependencies --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0c26976809f89..93145d948c218 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ pytz Cython>=0.28.2 flake8 flake8-comprehensions +flake8-rst hypothesis>=3.58.0 isort moto From f13ec9c1161a662a976cb038e337baa16f1165b7 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 17:43:28 +0000 Subject: [PATCH 6/7] Renaming requirements.txt to requirements_dev.txt --- requirements.txt => requirements_dev.txt | 0 scripts/conda_to_pip.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename requirements.txt => requirements_dev.txt (100%) diff --git a/requirements.txt b/requirements_dev.txt similarity index 100% rename from requirements.txt rename to requirements_dev.txt diff --git a/scripts/conda_to_pip.py b/scripts/conda_to_pip.py index 6b38cfebd4b17..958b0a20ceada 100755 --- a/scripts/conda_to_pip.py +++ b/scripts/conda_to_pip.py @@ -87,10 +87,10 @@ def main(conda_fname, pip_fname, compare=False): repo_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) res = main(os.path.join(repo_path, 'environment.yml'), - os.path.join(repo_path, 'requirements.txt'), + os.path.join(repo_path, 'requirements_dev.txt'), compare=args.compare) if res: - sys.stderr.write('`requirements.txt` has to be generated with `{}` ' - 'after `environment.yml` is modified.\n'.format( + sys.stderr.write('`requirements_dev.txt` has to be generated with ' + '`{}` after `environment.yml` is modified.\n'.format( sys.argv[0])) sys.exit(res) From 6f30a04cfc14bcef765c692adc1c34a8ddb5e2d8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sun, 11 Nov 2018 18:11:46 +0000 Subject: [PATCH 7/7] Renaming requirements_dev.txt to requirements-dev.txt and conda_to_pip.py to generate_pip_deps_from_conda.py --- ci/code_checks.sh | 4 ++-- doc/source/contributing.rst | 2 +- requirements_dev.txt => requirements-dev.txt | 0 ...nda_to_pip.py => generate_pip_deps_from_conda.py} | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) rename requirements_dev.txt => requirements-dev.txt (100%) rename scripts/{conda_to_pip.py => generate_pip_deps_from_conda.py} (89%) diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 61ea1c0930ac3..fac5c211cdad8 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -177,8 +177,8 @@ fi ### DEPENDENCIES ### if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then - MSG='Check that requirements.txt has been generated from environment.yml' ; echo $MSG - $BASE_DIR/scripts/conda_to_pip.py --compare + MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG + $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare RET=$(($RET + $?)) ; echo $MSG "DONE" fi diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 304accb096944..514a58456bcd9 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -218,7 +218,7 @@ You'll need to have at least python3.5 installed on your system. . ~/virtualenvs/pandas-dev/bin/activate # Install the build dependencies - python -m pip install -r requirements.txt + python -m pip install -r requirements-dev.txt # Build and install pandas python setup.py build_ext --inplace -j 4 diff --git a/requirements_dev.txt b/requirements-dev.txt similarity index 100% rename from requirements_dev.txt rename to requirements-dev.txt diff --git a/scripts/conda_to_pip.py b/scripts/generate_pip_deps_from_conda.py similarity index 89% rename from scripts/conda_to_pip.py rename to scripts/generate_pip_deps_from_conda.py index 958b0a20ceada..2474214a4a53b 100755 --- a/scripts/conda_to_pip.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -1,14 +1,14 @@ #!/usr/bin/env python """ -Convert the conda environment.yml to the pip requirements.txt, +Convert the conda environment.yml to the pip requirements-dev.txt, or check that they have the same packages (for the CI) Usage: - Generate `requirements.txt` + Generate `requirements-dev.txt` $ ./conda_to_pip - Compare and fail (exit status != 0) if `requirements.txt` has not been + Compare and fail (exit status != 0) if `requirements-dev.txt` has not been generated with this script: $ ./conda_to_pip --compare """ @@ -52,7 +52,7 @@ def main(conda_fname, pip_fname, compare=False): conda_fname : str Path to the conda file with dependencies (e.g. `environment.yml`). pip_fname : str - Path to the pip file with dependencies (e.g. `requirements.txt`). + Path to the pip file with dependencies (e.g. `requirements-dev.txt`). compare : bool, default False Whether to generate the pip file (``False``) or to compare if the pip file has been generated with this script and the last version @@ -87,10 +87,10 @@ def main(conda_fname, pip_fname, compare=False): repo_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) res = main(os.path.join(repo_path, 'environment.yml'), - os.path.join(repo_path, 'requirements_dev.txt'), + os.path.join(repo_path, 'requirements-dev.txt'), compare=args.compare) if res: - sys.stderr.write('`requirements_dev.txt` has to be generated with ' + sys.stderr.write('`requirements-dev.txt` has to be generated with ' '`{}` after `environment.yml` is modified.\n'.format( sys.argv[0])) sys.exit(res)