From b6830d8e53c6b37cd73d7e8a8e71fbaa0b01d85d Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 24 Jan 2023 20:53:35 +0100 Subject: [PATCH 01/20] Add conda files --- .conda/README.md | 52 +++++++++++++++++++++++++++++++ .conda/meta.yaml | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 .conda/README.md create mode 100644 .conda/meta.yaml diff --git a/.conda/README.md b/.conda/README.md new file mode 100644 index 00000000..67d38c97 --- /dev/null +++ b/.conda/README.md @@ -0,0 +1,52 @@ +# Publication vers Anaconda + +Ce readme décrit la partie publication vers conda, pour utiliser le paquet conda résultant, voir [le readme principal](https://github.com/openfisca/openfisca-france/tree/publish-to-conda#installez-un-environnement-virtuel-avec-conda). + +Pour envoyer un paquet vers conda il faut obligatoirement un fichier `meta.yaml` qui décrit le package. + +Nous avons fait le choix d'utiliser le paquet PyPi comme source du paquet Conda, pour s'assurer que l'on publie bien la même chose sur les deux plateformes. + +L'upload automatique est fait de la façon suivante par la CI uniquement si l'étape de livraison sur PyPi s'est bien déroulée : +- Installation de Miniconda. +- Récupération par le script `.github/get_pypi_info.py` des informations du package sur PyPi. +- Ecriture de ces informations par ce même script dans le fichier `.conda/meta.yaml`. +- Exécution de `conda build` pour construire et publier le paquet conformément au fichier `.conda/meta.yaml`. Cette étape nécessite la variable de CI ANACONDA_TOKEN. + +Pour valider que tout a fonctionné, une étape `test-on-windows` a été ajoutée en fin de CI. Cette étape récupère le paquet conda sur une machine Windows et exécute les tests. + +**A noter** : Le paquet OpenFisca-France est aussi publié sur `conda-forge`, pour cela voir [le feedstock](https://github.com/openfisca/openfisca-france-feedstock/tree/master/recipe) + +C'est le channel `conda-forge` qui est le channel stable à conseiller aux utilisateurs. Le channel `openfisca` reçoit les derniers paquets de façon automatique. + +## Etapes préparatoires pour arriver à cette automatisation + +- Création d'un compte sur https://anaconda.org. +- Création d'un token sur https://anaconda.org/openfisca/settings/access avec le droit _Allow write access to the API site_. Attention il expire le 2023/01/13. + +- Mis en place du token dans GitHub Action sous le nom de variable ANACONDA_TOKEN. + +### Publication manuelle du package + +Les étapes suivantes peuvent être réalisées sous Windows pour tester la publication : + +_Cela fonctionne aussi sous macOS et Linux, à condition d'adapter les chemins._ + +- Editer `.conda/meta.yaml` pour vérifier son contenu. +- Installer [MiniConda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html) si vous n'avez pas déjà [AnaConda](https://www.anaconda.com/products/individual). +- Puis dans le terminal saisissez les instructions suivantes : + - `conda install -c anaconda conda-build anaconda-client` Pour installer les outils indispensable. + - `conda build --croot c:\temp .conda` L'option `--croot ` est nécessaire sous Windows à cause des chemins trop long. + - `conda install -c anaconda anaconda-client` Pour installer l'outil en ligne de commande. + - `anaconda login` Pour vous connecter avec le compte _openfisca_, voir le Keepass OpenFisca. + - `anaconda upload c:\temp\noarch\openfisca-france--py_0.tar.bz2` pour publier le package. +- Vérifier que tout c'est bien passé sur https://anaconda.org/search?q=openfisca. + +## Test avec Docker + +Docker peut être utilisé pour valider le bon fonctionnement + +``` +docker run -i -t continuumio/anaconda3 /bin/bash +conda install -c openfisca -c conda-forge openfisca-france-dev +openfisca test --country-package openfisca_france tests +``` \ No newline at end of file diff --git a/.conda/meta.yaml b/.conda/meta.yaml new file mode 100644 index 00000000..fd7ce360 --- /dev/null +++ b/.conda/meta.yaml @@ -0,0 +1,81 @@ +############################################################################### +## Fichier de description du package pour Anaconda.org +## Attention, les chaines PYPI_VERSION, PYPI_URL et PYPI_SHA256 sont remplacées +## par la CI, il faut les conserver. +############################################################################### + +{% set name = "openfisca-france-data" %} +{% set version = "PYPI_VERSION" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + path: .. + +build: + noarch: python + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" + +requirements: + host: + - python + - pip + run: + - python >=3.6,<4.0 + - click >=8.0.0,<9.0.0 + - matplotlib >=3.1.1,<4.0.0 + - multipledispatch >=0.6.0,<1.0.0 + - openFisca-france >=113.0.0 + - openFisca-survey-manager >= 0.44.2, < 1.0.0 + - wquantiles >= 0.3.0, < 1.0.0 + +test: + imports: + - openfisca_france + requires: + - pip + commands: + - pip check + +outputs: + - name: openfisca-france + + - name: openfisca-france-scipy + build: + noarch: python + requirements: + host: + - python + run: + - scipy >=0.17 + - {{ pin_subpackage('openfisca-france', exact=True) }} + + - name: openfisca-france-dev + build: + noarch: python + requirements: + host: + - python + run: + - autopep8 ==1.5.7 + - flake8 >=3.8.0,<3.10.0 + - flake8-print + - pytest >=5.0.0, <7.0.0 + - requests >=2.8 + - yamllint >=1.11.1,<1.27 + - {{ pin_subpackage('openfisca-france-scipy', exact=True) }} + +about: + home: https://fr.openfisca.org/ + license_family: AGPL + license: AGPL-3.0-only + license_file: LICENSE.AGPL.txt + summary: "OpenFisca-France-Data module to work with French survey data" + description: | + OpenFisca is a versatile microsimulation free software. + This repository contains the module to work with French survey data. + doc_url: https://fr.openfisca.org/ + dev_url: https://github.com/openfisca/openfisca-france/ From c3f4faaf2f59308005f77fa36c0016f73728bb15 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 09:14:33 +0100 Subject: [PATCH 02/20] Add files to ignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7026e82f..81693e04 100644 --- a/.gitignore +++ b/.gitignore @@ -85,5 +85,5 @@ openfisca_erfs_fpr.json *.html .venv*/ # PyEnv -.pytest_cache .python-version +.pytest_cache/ From b4aa0fdbf9f85bed5ecdc2b8408f27791c62115f Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 09:22:45 +0100 Subject: [PATCH 03/20] Conda change deps --- .conda/meta.yaml | 40 ++++++++++++++++------------------------ setup.py | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index fd7ce360..9302a3b1 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -24,49 +24,41 @@ requirements: - python - pip run: - - python >=3.6,<4.0 + - python >=3.7,<4.0 - click >=8.0.0,<9.0.0 - matplotlib >=3.1.1,<4.0.0 - multipledispatch >=0.6.0,<1.0.0 - openFisca-france >=113.0.0 - - openFisca-survey-manager >= 0.44.2, < 1.0.0 - - wquantiles >= 0.3.0, < 1.0.0 + - openFisca-survey-manager >= 0.44.2,< 1.0.0 + - wquantiles >= 0.3.0,< 1.0.0 test: imports: - - openfisca_france + - openfisca-france-data requires: - pip commands: - pip check outputs: - - name: openfisca-france + - name: openfisca-france-data - - name: openfisca-france-scipy + - name: openfisca-france-test build: noarch: python requirements: host: - python run: - - scipy >=0.17 - - {{ pin_subpackage('openfisca-france', exact=True) }} - - - name: openfisca-france-dev - build: - noarch: python - requirements: - host: - - python - run: - - autopep8 ==1.5.7 - - flake8 >=3.8.0,<3.10.0 - - flake8-print - - pytest >=5.0.0, <7.0.0 - - requests >=2.8 - - yamllint >=1.11.1,<1.27 - - {{ pin_subpackage('openfisca-france-scipy', exact=True) }} + - autopep8 >=1.4.0,< 1.5.0 + - flake8 >=3.7.0,<3.8.0 + - ipython >=7.5.0,< 8.0.0 + - mypy >=0.670,< 1.0.0 + - pytest >=4.3.0,< 5.0.0 + - pytest-cov >=2.6.0,< 3.0.0 + - scipy >=1.2.1,< 2.0.0 + - toolz >=0.9.0,< 1.0.0 + - {{ pin_subpackage('openfisca-france-data', exact=True) }} about: home: https://fr.openfisca.org/ @@ -78,4 +70,4 @@ about: OpenFisca is a versatile microsimulation free software. This repository contains the module to work with French survey data. doc_url: https://fr.openfisca.org/ - dev_url: https://github.com/openfisca/openfisca-france/ + dev_url: https://github.com/openfisca/openfisca-france-data/ diff --git a/setup.py b/setup.py index 80010821..a89584e0 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "mypy >= 0.670, < 1.0.0", "pytest >= 4.3.0, < 5.0.0", "pytest-cov >= 2.6.0, < 3.0.0", - 'scipy >= 1.2.1, < 2.0.0', + "scipy >= 1.2.1, < 2.0.0", "toolz >= 0.9.0, < 1.0.0", ], }, From a61fd8bb084ed050cc516d00680aed64753daf97 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:07:16 +0100 Subject: [PATCH 04/20] WIP: conda CI --- .conda/meta.yaml | 6 +- .gitlab-ci.yml | 701 +-------------------------------------------- runner/build_ci.py | 176 +++++++----- 3 files changed, 117 insertions(+), 766 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 9302a3b1..226d5770 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -1,11 +1,10 @@ ############################################################################### ## Fichier de description du package pour Anaconda.org -## Attention, les chaines PYPI_VERSION, PYPI_URL et PYPI_SHA256 sont remplacées -## par la CI, il faut les conserver. +## Le numéro de version est mis à jour automatiquement par BumpVer ############################################################################### {% set name = "openfisca-france-data" %} -{% set version = "PYPI_VERSION" %} +{% set version = "0.21" %} package: name: {{ name|lower }} @@ -58,6 +57,7 @@ outputs: - pytest-cov >=2.6.0,< 3.0.0 - scipy >=1.2.1,< 2.0.0 - toolz >=0.9.0,< 1.0.0 + - bumpver - {{ pin_subpackage('openfisca-france-data', exact=True) }} about: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6290ead5..5e010e1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ cache: stages: - docker - test + - anaconda - build_collection - build_input_data - aggregates @@ -56,6 +57,16 @@ test: stage: test tags: - openfisca +deploy_conda: + before_script: + - '' + image: continuumio/miniconda3 + script: + - conda install -y conda-build anaconda-client toml + - python3 runner/get_pypi_info.py -p openfisca-france-data + - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca + .conda + stage: anaconda build_collection: image: $CI_REGISTRY_IMAGE:latest script: @@ -74,666 +85,6 @@ build_collection: tags: - openfisca when: manual -in_dt-1996: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-1996" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 1996 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_1996.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1996.ini - stage: build_input_data - tags: - - openfisca -agg-1996: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-1996 - script: - - echo "aggregates-1996" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1996.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 1996 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-1997: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-1997" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 1997 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_1997.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1997.ini - stage: build_input_data - tags: - - openfisca -agg-1997: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-1997 - script: - - echo "aggregates-1997" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1997.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 1997 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-1998: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-1998" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 1998 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_1998.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1998.ini - stage: build_input_data - tags: - - openfisca -agg-1998: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-1998 - script: - - echo "aggregates-1998" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1998.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 1998 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-1999: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-1999" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 1999 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_1999.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1999.ini - stage: build_input_data - tags: - - openfisca -agg-1999: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-1999 - script: - - echo "aggregates-1999" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-1999.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 1999 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2000: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2000" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2000 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2000.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2000.ini - stage: build_input_data - tags: - - openfisca -agg-2000: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2000 - script: - - echo "aggregates-2000" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2000.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2000 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2001: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2001" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2001 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2001.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2001.ini - stage: build_input_data - tags: - - openfisca -agg-2001: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2001 - script: - - echo "aggregates-2001" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2001.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2001 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2002: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2002" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2002 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2002.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2002.ini - stage: build_input_data - tags: - - openfisca -agg-2002: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2002 - script: - - echo "aggregates-2002" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2002.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2002 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2003: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2003" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2003 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2003.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2003.ini - stage: build_input_data - tags: - - openfisca -agg-2003: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2003 - script: - - echo "aggregates-2003" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2003.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2003 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2004: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2004" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2004 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2004.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2004.ini - stage: build_input_data - tags: - - openfisca -agg-2004: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2004 - script: - - echo "aggregates-2004" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2004.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2004 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2005: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2005" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2005 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2005.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2005.ini - stage: build_input_data - tags: - - openfisca -agg-2005: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2005 - script: - - echo "aggregates-2005" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2005.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2005 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2006: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2006" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2006 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2006.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2006.ini - stage: build_input_data - tags: - - openfisca -agg-2006: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2006 - script: - - echo "aggregates-2006" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2006.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2006 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2007: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2007" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2007 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2007.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2007.ini - stage: build_input_data - tags: - - openfisca -agg-2007: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2007 - script: - - echo "aggregates-2007" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2007.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2007 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2008: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2008" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2008 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2008.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2008.ini - stage: build_input_data - tags: - - openfisca -agg-2008: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2008 - script: - - echo "aggregates-2008" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2008.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2008 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2009: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2009" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2009 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2009.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2009.ini - stage: build_input_data - tags: - - openfisca -agg-2009: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2009 - script: - - echo "aggregates-2009" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2009.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2009 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2010: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2010" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2010 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2010.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2010.ini - stage: build_input_data - tags: - - openfisca -agg-2010: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2010 - script: - - echo "aggregates-2010" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2010.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2010 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2011: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2011" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2011 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2011.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2011.ini - stage: build_input_data - tags: - - openfisca -agg-2011: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2011 - script: - - echo "aggregates-2011" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2011.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2011 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2012: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2012" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2012 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2012.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2012.ini - stage: build_input_data - tags: - - openfisca -agg-2012: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2012 - script: - - echo "aggregates-2012" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2012.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2012 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2013: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2013" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2013 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2013.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2013.ini - stage: build_input_data - tags: - - openfisca -agg-2013: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2013 - script: - - echo "aggregates-2013" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2013.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2013 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2014: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2014" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2014 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2014.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2014.ini - stage: build_input_data - tags: - - openfisca -agg-2014: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2014 - script: - - echo "aggregates-2014" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2014.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2014 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2015: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2015" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2015 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2015.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2015.ini - stage: build_input_data - tags: - - openfisca -agg-2015: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2015 - script: - - echo "aggregates-2015" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2015.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2015 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2016: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2016" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2016 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2016.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2016.ini - stage: build_input_data - tags: - - openfisca -agg-2016: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2016 - script: - - echo "aggregates-2016" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2016.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2016 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca -in_dt-2017: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2017" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2017 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2017.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2017.ini - stage: build_input_data - tags: - - openfisca -agg-2017: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2017 - script: - - echo "aggregates-2017" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2017.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2017 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca in_dt-2018: image: $CI_REGISTRY_IMAGE:latest script: @@ -764,33 +115,3 @@ agg-2018: stage: aggregates tags: - openfisca -in_dt-2019: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2019" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2019 -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_2019.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini - stage: build_input_data - tags: - - openfisca -agg-2019: - artifacts: - paths: - - ./*.html - - ./*.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - in_dt-2019 - script: - - echo "aggregates-2019" - - cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2019 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER - stage: aggregates - tags: - - openfisca diff --git a/runner/build_ci.py b/runner/build_ci.py index c5d1b661..4031360b 100644 --- a/runner/build_ci.py +++ b/runner/build_ci.py @@ -9,10 +9,11 @@ import yaml # Config file use to get the available years -CONFIG='./runner/openfisca_survey_manager_raw_data.ini' +CONFIG = "./runner/openfisca_survey_manager_raw_data.ini" + def header(): - return ''' + return """ ################################################ # GENERATED FILE, DO NOT EDIT # Please visit runner/README.md @@ -34,6 +35,7 @@ def header(): stages: - docker - test + - anaconda - build_collection - build_input_data - aggregates @@ -63,72 +65,74 @@ def header(): # Build Docker is needed only if code as changed. when: manual -''' - +""" def build_collections(): build_collection = { - 'build_collection': - {'stage': 'build_collection', - 'image': '$CI_REGISTRY_IMAGE:latest', - 'tags': ['openfisca'], - 'script': [ - 'echo "Begin with fresh config"', - 'mkdir -p $ROOT_FOLDER/data_collections/$OUT_FOLDER/', - 'rm $ROOT_FOLDER/data_collections/$OUT_FOLDER/*.json || true', # || true to ignore error - 'cp ./runner/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini', - 'echo "Custom output folder"', - 'sed -i "s/data_collections/data_collections\/$OUT_FOLDER\//" ~/.config/openfisca-survey-manager/config.ini', - 'cat ~/.config/openfisca-survey-manager/config.ini', - '#build-collection -c enquete_logement -d -m -s 2013', - 'build-collection -c erfs_fpr -d -m -v', - 'echo "Backup updated config"', - 'cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini' - - ], - 'when': 'manual', + "build_collection": { + "stage": "build_collection", + "image": "$CI_REGISTRY_IMAGE:latest", + "tags": ["openfisca"], + "script": [ + 'echo "Begin with fresh config"', + "mkdir -p $ROOT_FOLDER/data_collections/$OUT_FOLDER/", + "rm $ROOT_FOLDER/data_collections/$OUT_FOLDER/*.json || true", # || true to ignore error + "cp ./runner/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini", + 'echo "Custom output folder"', + 'sed -i "s/data_collections/data_collections\/$OUT_FOLDER\//" ~/.config/openfisca-survey-manager/config.ini', + "cat ~/.config/openfisca-survey-manager/config.ini", + "#build-collection -c enquete_logement -d -m -s 2013", + "build-collection -c erfs_fpr -d -m -v", + 'echo "Backup updated config"', + "cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini", + ], + "when": "manual", } } return build_collection + def build_input_data(year): return { - 'in_dt-'+ year: - {'stage': 'build_input_data', - 'image': '$CI_REGISTRY_IMAGE:latest', - # Remove needs because it prevent execution if build_collection is in manual - #'needs': ['build_collection'], - 'tags': ['openfisca'], - 'script': [ - 'echo "build_input_data-' + year + '"', - 'mkdir -p $ROOT_FOLDER/$OUT_FOLDER', - # Put the config from build collections step - 'cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini', - f'build-erfs-fpr -y {year} -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_{year}.h5', - 'cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-' + year + '.ini', + "in_dt-" + + year: { + "stage": "build_input_data", + "image": "$CI_REGISTRY_IMAGE:latest", + # Remove needs because it prevent execution if build_collection is in manual + #'needs': ['build_collection'], + "tags": ["openfisca"], + "script": [ + 'echo "build_input_data-' + year + '"', + "mkdir -p $ROOT_FOLDER/$OUT_FOLDER", + # Put the config from build collections step + "cp $ROOT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini", + f"build-erfs-fpr -y {year} -f $ROOT_FOLDER/$OUT_FOLDER/erfs_flat_{year}.h5", + "cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-" + + year + + ".ini", ], } } + def aggregates(year): return { - 'agg-'+ year: - {'stage': 'aggregates', - 'image': '$CI_REGISTRY_IMAGE:latest', - 'needs': ['in_dt-' + year ], - 'tags': ['openfisca'], - 'script': [ - 'echo "aggregates-' + year + '"', - f'cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-{year}.ini ~/.config/openfisca-survey-manager/config.ini', - f'python tests/erfs_fpr/integration/test_aggregates.py --year {year}', - 'mkdir -p $ROOT_FOLDER/$OUT_FOLDER', - 'cp ./*.html $ROOT_FOLDER/$OUT_FOLDER', - 'cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER', + "agg-" + + year: { + "stage": "aggregates", + "image": "$CI_REGISTRY_IMAGE:latest", + "needs": ["in_dt-" + year], + "tags": ["openfisca"], + "script": [ + 'echo "aggregates-' + year + '"', + f"cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-{year}.ini ~/.config/openfisca-survey-manager/config.ini", + f"python tests/erfs_fpr/integration/test_aggregates.py --year {year}", + "mkdir -p $ROOT_FOLDER/$OUT_FOLDER", + "cp ./*.html $ROOT_FOLDER/$OUT_FOLDER", + "cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER", ], - 'artifacts':{ - 'paths': ['./*.html', './*.csv'] - } + "artifacts": {"paths": ["./*.html", "./*.csv"]}, } } @@ -136,30 +140,32 @@ def aggregates(year): # Warning : not used yet : test are independant for now. def make_test_by_year(year): return { - 'test-'+year:{ - 'stage': 'test', - 'image': '$CI_REGISTRY_IMAGE:latest', - 'needs': ['agg-' + year], - 'tags': ['openfisca'], - 'script':[ - f'cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-{year}.ini ~/.config/openfisca-survey-manager/config.ini', - 'make test', - ], - } + "test-" + + year: { + "stage": "test", + "image": "$CI_REGISTRY_IMAGE:latest", + "needs": ["agg-" + year], + "tags": ["openfisca"], + "script": [ + f"cp $ROOT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-{year}.ini ~/.config/openfisca-survey-manager/config.ini", + "make test", + ], } + } def make_test(): return { - 'test':{ - 'stage': 'test', - 'image': '$CI_REGISTRY_IMAGE:latest', - 'tags': ['openfisca'], - 'script':[ - 'make test', - ], - } + "test": { + "stage": "test", + "image": "$CI_REGISTRY_IMAGE:latest", + "tags": ["openfisca"], + "script": [ + "make test", + ], } + } + def get_erfs_years(): """ @@ -169,7 +175,7 @@ def get_erfs_years(): try: config = configparser.ConfigParser() config.read(CONFIG) - for key in config['erfs_fpr']: + for key in config["erfs_fpr"]: if key.isnumeric(): years.append(key) return years @@ -177,24 +183,48 @@ def get_erfs_years(): print(f"Key 'erfs_fpr' not found in {configfile}, switchin to default {years}") raise KeyError + +def build_conda_package(): + """ + Build conda package + """ + return { + "deploy_conda": { + "stage": "anaconda", + "before_script": [""], + "image": "continuumio/miniconda3", + "script": [ + "conda install -y conda-build anaconda-client toml", + "python3 runner/get_pypi_info.py -p openfisca-france-data", + #"conda config --set anaconda_upload yes", + "conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda", + ], + #"only": ["master"], + } + } + + def build_gitlab_ci(erfs_years): gitlab_ci = header() gitlab_ci += yaml.dump(make_test()) + gitlab_ci += yaml.dump(build_conda_package()) gitlab_ci += yaml.dump(build_collections()) for year in erfs_years: - print('\t ERFS : Building for year', year) + print("\t ERFS : Building for year", year) gitlab_ci += yaml.dump(build_input_data(year)) gitlab_ci += yaml.dump(aggregates(year)) return gitlab_ci + def main(): print("Reading survey manager config...") erfs_years = get_erfs_years() # For testing only some years - # erfs_years = ["2018"] + erfs_years = ["2018"] gitlab_ci = build_gitlab_ci(erfs_years) - with open(r'.gitlab-ci.yml', mode='w') as file: + with open(r".gitlab-ci.yml", mode="w") as file: file.write(gitlab_ci) print("Done with success!") + main() From 17f776ee0be7645e303f68992098fe82d22f398d Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:07:58 +0100 Subject: [PATCH 05/20] Add BumpVer --- Makefile | 3 +++ setup.cfg | 16 ++++++++++++++++ setup.py | 1 + 3 files changed, 20 insertions(+) diff --git a/Makefile b/Makefile index 2166cec6..a4c748fc 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,6 @@ archive: clean ctags: ctags --recurse=yes . + +bump: + bumpver update --minor diff --git a/setup.cfg b/setup.cfg index b196a407..71d6404c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,3 +18,19 @@ in-place = true addopts = --showlocals --doctest-modules --disable-pytest-warnings testpaths = tests python_files = **/*.py + +[bumpver] +current_version = "0.21" +version_pattern = "MAJOR.MINOR" +commit_message = "bump version {old_version} -> {new_version}" +commit = True +tag = True +push = True + +[bumpver:file_patterns] +setup.cfg = + current_version = "{version}" +setup.py = + "{version}" +.conda/meta.yaml = + "{version}" diff --git a/setup.py b/setup.py index a89584e0..82b6f552 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ "pytest-cov >= 2.6.0, < 3.0.0", "scipy >= 1.2.1, < 2.0.0", "toolz >= 0.9.0, < 1.0.0", + "bumpver >= 2022.1120", ], }, packages = find_packages(exclude = ("docs", "tests")), From b3920e5c5eea686238f35d70932dfb3d4b214113 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:08:12 +0100 Subject: [PATCH 06/20] bump version 0.21 -> 0.22 --- .conda/meta.yaml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 226d5770..e37a8dc3 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -4,7 +4,7 @@ ############################################################################### {% set name = "openfisca-france-data" %} -{% set version = "0.21" %} +{% set version = "0.22" %} package: name: {{ name|lower }} diff --git a/setup.cfg b/setup.cfg index 71d6404c..de0d4ea8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ testpaths = tests python_files = **/*.py [bumpver] -current_version = "0.21" +current_version = "0.22" version_pattern = "MAJOR.MINOR" commit_message = "bump version {old_version} -> {new_version}" commit = True From 24bcc62fdebe00deb5809d2373549a2539bff5cf Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 12:17:01 +0100 Subject: [PATCH 07/20] Add Conda to CI WIP: remove getpypi WIP: remove getpypi WIP: remove space WIP: remove space WIP: License WIP: test WIP: remove test --- .conda/meta.yaml | 25 +++++++++++++------------ .gitlab-ci.yml | 17 +++++++++++++++-- runner/build_ci.py | 26 ++++++++++++++++++++++---- setup.cfg | 2 +- 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index e37a8dc3..53625a19 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -28,16 +28,17 @@ requirements: - matplotlib >=3.1.1,<4.0.0 - multipledispatch >=0.6.0,<1.0.0 - openFisca-france >=113.0.0 - - openFisca-survey-manager >= 0.44.2,< 1.0.0 - - wquantiles >= 0.3.0,< 1.0.0 + - openFisca-survey-manager >=0.44.2,<1.0.0 + - wquantiles >=0.3.0,<1.0.0 test: imports: - - openfisca-france-data + - openfisca_france_data requires: - pip commands: - - pip check + #- pip check # TODO: openfisca-core 35.7.6 requires pytest, which is not installed. !!! + - pip list outputs: - name: openfisca-france-data @@ -49,14 +50,14 @@ outputs: host: - python run: - - autopep8 >=1.4.0,< 1.5.0 + - autopep8 >=1.4.0,<1.5.0 - flake8 >=3.7.0,<3.8.0 - - ipython >=7.5.0,< 8.0.0 - - mypy >=0.670,< 1.0.0 - - pytest >=4.3.0,< 5.0.0 - - pytest-cov >=2.6.0,< 3.0.0 - - scipy >=1.2.1,< 2.0.0 - - toolz >=0.9.0,< 1.0.0 + - ipython >=7.5.0,<8.0.0 + - mypy >=0.670,<1.0.0 + - pytest >=4.3.0,<5.0.0 + - pytest-cov >=2.6.0,<3.0.0 + - scipy >=1.2.1,<2.0.0 + - toolz >=0.9.0,<1.0.0 - bumpver - {{ pin_subpackage('openfisca-france-data', exact=True) }} @@ -64,7 +65,7 @@ about: home: https://fr.openfisca.org/ license_family: AGPL license: AGPL-3.0-only - license_file: LICENSE.AGPL.txt + license_file: LICENSE summary: "OpenFisca-France-Data module to work with French survey data" description: | OpenFisca is a versatile microsimulation free software. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e010e1d..0e546333 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,10 +60,23 @@ test: deploy_conda: before_script: - '' + except: + - master image: continuumio/miniconda3 script: - - conda install -y conda-build anaconda-client toml - - python3 runner/get_pypi_info.py -p openfisca-france-data + - conda install -y conda-build anaconda-client + - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca + .conda + stage: anaconda +deploy_conda: + before_script: + - '' + image: continuumio/miniconda3 + only: + - master + script: + - conda install -y conda-build anaconda-client + - conda config --set anaconda_upload yes - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda stage: anaconda diff --git a/runner/build_ci.py b/runner/build_ci.py index 4031360b..25a97b00 100644 --- a/runner/build_ci.py +++ b/runner/build_ci.py @@ -194,12 +194,29 @@ def build_conda_package(): "before_script": [""], "image": "continuumio/miniconda3", "script": [ - "conda install -y conda-build anaconda-client toml", - "python3 runner/get_pypi_info.py -p openfisca-france-data", - #"conda config --set anaconda_upload yes", + "conda install -y conda-build anaconda-client", "conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda", ], - #"only": ["master"], + "except": ["master"], + } + } + + +def build_and_deploy_conda_package(): + """ + Build and deploy conda package + """ + return { + "deploy_conda": { + "stage": "anaconda", + "before_script": [""], + "image": "continuumio/miniconda3", + "script": [ + "conda install -y conda-build anaconda-client", + "conda config --set anaconda_upload yes", + "conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda", + ], + "only": ["master"], } } @@ -208,6 +225,7 @@ def build_gitlab_ci(erfs_years): gitlab_ci = header() gitlab_ci += yaml.dump(make_test()) gitlab_ci += yaml.dump(build_conda_package()) + gitlab_ci += yaml.dump(build_and_deploy_conda_package()) gitlab_ci += yaml.dump(build_collections()) for year in erfs_years: print("\t ERFS : Building for year", year) diff --git a/setup.cfg b/setup.cfg index de0d4ea8..be2886f4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ version_pattern = "MAJOR.MINOR" commit_message = "bump version {old_version} -> {new_version}" commit = True tag = True -push = True +# push = True [bumpver:file_patterns] setup.cfg = From 2594b447edeba2bd89d422b8f4ea9a1df005817d Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 27 Jan 2023 16:52:45 +0100 Subject: [PATCH 08/20] Publish on master --- .gitlab-ci.yml | 4 ++-- runner/build_ci.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e546333..1863e9f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ test: stage: test tags: - openfisca -deploy_conda: +build_conda_package: before_script: - '' except: @@ -68,7 +68,7 @@ deploy_conda: - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda stage: anaconda -deploy_conda: +build_and_deploy_conda_package: before_script: - '' image: continuumio/miniconda3 diff --git a/runner/build_ci.py b/runner/build_ci.py index 25a97b00..3726fe44 100644 --- a/runner/build_ci.py +++ b/runner/build_ci.py @@ -189,7 +189,7 @@ def build_conda_package(): Build conda package """ return { - "deploy_conda": { + "build_conda_package": { "stage": "anaconda", "before_script": [""], "image": "continuumio/miniconda3", @@ -207,7 +207,7 @@ def build_and_deploy_conda_package(): Build and deploy conda package """ return { - "deploy_conda": { + "build_and_deploy_conda_package": { "stage": "anaconda", "before_script": [""], "image": "continuumio/miniconda3", From 603f249a4faa07a9a697fd5a1d2c2d6a4d26fa34 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 31 Jan 2023 09:43:52 +0100 Subject: [PATCH 09/20] WIP: GitlabCI --- .github/has-functional-changes.sh | 12 ++ .github/is-version-number-acceptable.sh | 33 ++++ .github/publish-git-tag.sh | 4 + .github/workflows/workflow.yml | 244 ++++++++++++++++++++++++ setup.cfg | 2 +- 5 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 .github/has-functional-changes.sh create mode 100644 .github/is-version-number-acceptable.sh create mode 100644 .github/publish-git-tag.sh create mode 100644 .github/workflows/workflow.yml diff --git a/.github/has-functional-changes.sh b/.github/has-functional-changes.sh new file mode 100644 index 00000000..48f97805 --- /dev/null +++ b/.github/has-functional-changes.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*" + +last_tagged_commit=`git describe --tags --abbrev=0 --first-parent` # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit + +if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'` # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. +then + echo "No functional changes detected." + exit 1 +else echo "The functional files above were changed." +fi diff --git a/.github/is-version-number-acceptable.sh b/.github/is-version-number-acceptable.sh new file mode 100644 index 00000000..0f704a93 --- /dev/null +++ b/.github/is-version-number-acceptable.sh @@ -0,0 +1,33 @@ +#! /usr/bin/env bash + +if [[ ${GITHUB_REF#refs/heads/} == master ]] +then + echo "No need for a version check on master." + exit 0 +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh +then + echo "No need for a version update." + exit 0 +fi + +current_version=`python setup.py --version` + +if git rev-parse --verify --quiet $current_version +then + echo "Version $current_version already exists in commit:" + git --no-pager log -1 $current_version + echo + echo "Update the version number in setup.py before merging this branch into master." + echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated." + exit 1 +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh | grep --quiet CHANGELOG.md +then + echo "CHANGELOG.md has not been modified, while functional changes were made." + echo "Explain what you changed before merging this branch into master." + echo "Look at the CONTRIBUTING.md file to learn how to write the changelog." + exit 2 +fi diff --git a/.github/publish-git-tag.sh b/.github/publish-git-tag.sh new file mode 100644 index 00000000..4450357c --- /dev/null +++ b/.github/publish-git-tag.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env bash + +git tag `python setup.py --version` +git push --tags # update the repository version diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 00000000..12817c58 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,244 @@ +name: OpenFisca France + +on: + push: + pull_request: + types: [opened, reopened] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: ["ubuntu-20.04"] + python-version: ["3.8.9", "3.9.9"] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache build + id: restore-build + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} + restore-keys: | # in case of a cache miss (systematically unless the same commit is built repeatedly), the keys below will be used to restore dependencies from previous builds, and the cache will be stored at the end of the job, making up-to-date dependencies available for all jobs of the workflow; see more at https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action + build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ matrix.os }} + build-${{ env.pythonLocation }}-${{ matrix.os }} + - name: Build package + run: make build + - name: Cache release + id: restore-release + uses: actions/cache@v2 + with: + path: dist + key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} + + lint-files: + runs-on: ubuntu-20.04 + needs: [ build ] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all the tags + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + - name: Cache build + id: restore-build + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 + - run: make check-syntax-errors + - run: make check-style + + test-python: + runs-on: ${{ matrix.os }} + needs: [ build ] + strategy: + fail-fast: true + matrix: + os: [ "ubuntu-20.04" ] # On peut ajouter "macos-latest" si besoin + python-version: ["3.8.9", "3.9.9"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache build + id: restore-build + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} + - run: make test + + check-version-and-changelog: + runs-on: ubuntu-20.04 + needs: [ lint-files, test-python, test-yaml, test-api ] # Last job to run + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all the tags + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" + + # GitHub Actions does not have a halt job option, to stop from deploying if no functional changes were found. + # We build a separate job to substitute the halt option. + # The `deploy` job is dependent on the output of the `check-for-functional-changes` job. + check-for-functional-changes: + runs-on: ubuntu-20.04 + if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch + needs: [ check-version-and-changelog ] + outputs: + status: ${{ steps.stop-early.outputs.status }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all the tags + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + - id: stop-early + run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi + + deploy: + runs-on: ubuntu-20.04 + needs: [ check-for-functional-changes ] + if: needs.check-for-functional-changes.outputs.status == 'success' + env: + PYPI_USERNAME: openfisca-bot + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all the tags + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + - name: Cache build + id: restore-build + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 + - name: Cache release + id: restore-release + uses: actions/cache@v2 + with: + path: dist + key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 + - name: Upload a Python package to PyPi + run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD + - name: Publish a git tag + run: "${GITHUB_WORKSPACE}/.github/publish-git-tag.sh" + + build-conda: + runs-on: "ubuntu-20.04" + needs: [ check-version-and-changelog ] + # Do not build on master, the artifact will be used + if: github.ref != 'refs/heads/master' + steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: "3.7.9" + # Add conda-forge for OpenFisca-Core + channels: conda-forge + activate-environment: true + - uses: actions/checkout@v3 + - name: Get version + run: echo "PACKAGE_VERSION=$(python3 ./setup.py --version)" >> $GITHUB_ENV + - name: Conda Config + run: | + conda install conda-build anaconda-client + conda info + - name: Build Conda package + run: conda build --croot /tmp/conda .conda + - name: Upload Conda build + uses: actions/upload-artifact@v3 + with: + name: conda-build-${{ env.PACKAGE_VERSION }}-${{ github.sha }} + path: /tmp/conda + retention-days: 30 + + test-on-windows: + runs-on: "windows-latest" + needs: [ build-conda ] + steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: "3.9.9" + # Add conda-forge for OpenFisca-Core + channels: conda-forge + activate-environment: true + - uses: actions/checkout@v3 + - name: Get version + run: | + # chcp 65001 #set code page to utf-8 + echo ("PACKAGE_VERSION=" + (python3 ./setup.py --version) ) >> $env:GITHUB_ENV + echo "Version setup.py: ${{ env.PACKAGE_VERSION }}" + - name: Download conda build + uses: actions/download-artifact@v3 + with: + name: conda-build-${{ env.PACKAGE_VERSION }}-${{ github.sha }} + path: conda-build-tmp + - name: Install with conda + run: | + conda install -c ./conda-build-tmp/noarch/openfisca-france-data-${{ env.PACKAGE_VERSION }}-py_0.tar.bz2 openfisca-france-data + + publish-to-conda: + runs-on: "ubuntu-20.04" + needs: [ deploy ] + steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: "3.7.9" + # Add conda-forge for OpenFisca-Core + channels: conda-forge + activate-environment: true + - name: Get source code + uses: actions/checkout@v3 + - name: Get version + run: echo "PACKAGE_VERSION=$(python3 ./setup.py --version)" >> $GITHUB_ENV + # Get the last commit hash on the PR (-2 : before the merge commit) + - uses: actions/github-script@v6 + id: last_pr_commit + with: + script: | + const commits = ${{ toJSON(github.event.commits) }} + return commits.at(-2).id; + result-encoding: string + # Default Download artifact don't see artifact of other workflow + # So we use dawidd6/action-download-artifact@v2 to do it. + - name: Download artifact + id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: + workflow_conclusion: success + commit: ${{steps.last_pr_commit.outputs.result}} + name: conda-build-${{ env.PACKAGE_VERSION }}-${{steps.last_pr_commit.outputs.result}} + path: conda-build-tmp + if_no_artifact_found: fail + - name: Conda upload + # This shell is made necessary by https://github.com/conda-incubator/setup-miniconda/issues/128 + shell: bash -l {0} + run: | + conda install anaconda-client + conda info + anaconda -t ${{ secrets.ANACONDA_TOKEN }} upload -u openfisca ./conda-build-tmp/noarch/openfisca-france-data-*-py_0.tar.bz2 --force diff --git a/setup.cfg b/setup.cfg index be2886f4..75782079 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ current_version = "0.22" version_pattern = "MAJOR.MINOR" commit_message = "bump version {old_version} -> {new_version}" commit = True -tag = True +# tag = True # push = True [bumpver:file_patterns] From a0618347fd64614fba299e9f5e78ba4ca824787d Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 31 Jan 2023 09:47:47 +0100 Subject: [PATCH 10/20] WIP:CI --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 12817c58..73037d9e 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -82,7 +82,7 @@ jobs: check-version-and-changelog: runs-on: ubuntu-20.04 - needs: [ lint-files, test-python, test-yaml, test-api ] # Last job to run + needs: [ lint-files, test-python ] # Last job to run steps: - uses: actions/checkout@v2 with: From 655ecb6e6565cf994e31173dac7d34f3b767e750 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 31 Jan 2023 10:36:58 +0100 Subject: [PATCH 11/20] build --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 73037d9e..edb6dd6d 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -30,7 +30,7 @@ jobs: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ matrix.os }} build-${{ env.pythonLocation }}-${{ matrix.os }} - name: Build package - run: make build + run: python setup.py bdist_wheel - name: Cache release id: restore-release uses: actions/cache@v2 From 8be20f49db647de37fa3ac59ce3ea7f21e028632 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 31 Jan 2023 11:13:06 +0100 Subject: [PATCH 12/20] wip --- .github/workflows/workflow.yml | 4 ++-- Makefile | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index edb6dd6d..ba46a5f8 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -1,4 +1,4 @@ -name: OpenFisca France +name: OpenFisca France Data on: push: @@ -30,7 +30,7 @@ jobs: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ matrix.os }} build-${{ env.pythonLocation }}-${{ matrix.os }} - name: Build package - run: python setup.py bdist_wheel + run: make build - name: Cache release id: restore-release uses: actions/cache@v2 diff --git a/Makefile b/Makefile index a4c748fc..def01023 100644 --- a/Makefile +++ b/Makefile @@ -45,3 +45,7 @@ ctags: bump: bumpver update --minor + +build: + python3 -m pip install --upgrade build twine + python3 -m build From 7c0d4e5dac9e2e706baedc6f9c849f4bc92e1199 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 31 Jan 2023 11:32:59 +0100 Subject: [PATCH 13/20] wip WIP wip upgrade python disable flake8 Remove coveralls Set permission "openfisca" for openfisca-survey-manager Add GITHUB_TOKEN Test micromamba WIP Remove Mamba --- .github/has-functional-changes.sh | 0 .github/is-version-number-acceptable.sh | 0 .github/publish-git-tag.sh | 0 .github/workflows/workflow.yml | 49 +++++++++++++++++++------ Makefile | 1 + 5 files changed, 38 insertions(+), 12 deletions(-) mode change 100644 => 100755 .github/has-functional-changes.sh mode change 100644 => 100755 .github/is-version-number-acceptable.sh mode change 100644 => 100755 .github/publish-git-tag.sh diff --git a/.github/has-functional-changes.sh b/.github/has-functional-changes.sh old mode 100644 new mode 100755 diff --git a/.github/is-version-number-acceptable.sh b/.github/is-version-number-acceptable.sh old mode 100644 new mode 100755 diff --git a/.github/publish-git-tag.sh b/.github/publish-git-tag.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index ba46a5f8..91ce14ad 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -48,15 +48,41 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7.9 + python-version: 3.9.9 - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 - - run: make check-syntax-errors - - run: make check-style + - run: | + make check-syntax-errors + # pip install flake8 + # make check-style # Pas lancé car 1 600 erreurs !!! + + coverage: + runs-on: ubuntu-20.04 + needs: [ build ] + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TERM: xterm-256color # To colorize output of make tasks. + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9.9 + - name: Cache build + id: restore-build + uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 + - run: | + pip install --upgrade coveralls + coveralls --service=github test-python: runs-on: ${{ matrix.os }} @@ -90,7 +116,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7.9 + python-version: 3.8.9 - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" @@ -110,7 +136,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7.9 + python-version: 3.9.9 - id: stop-early run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi @@ -128,7 +154,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7.9 + python-version: 3.9.9 - name: Cache build id: restore-build uses: actions/cache@v2 @@ -155,9 +181,9 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: "3.7.9" - # Add conda-forge for OpenFisca-Core - channels: conda-forge + python-version: "3.9.9" + # Add "conda-forge" for OpenFisca-Core and "openfisca" for openfisca-survey-manager + channels: conda-forge, openfisca activate-environment: true - uses: actions/checkout@v3 - name: Get version @@ -183,8 +209,7 @@ jobs: with: auto-update-conda: true python-version: "3.9.9" - # Add conda-forge for OpenFisca-Core - channels: conda-forge + channels: conda-forge, openfisca activate-environment: true - uses: actions/checkout@v3 - name: Get version @@ -208,7 +233,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: "3.7.9" + python-version: "3.9.9" # Add conda-forge for OpenFisca-Core channels: conda-forge activate-environment: true diff --git a/Makefile b/Makefile index def01023..485fda33 100644 --- a/Makefile +++ b/Makefile @@ -49,3 +49,4 @@ bump: build: python3 -m pip install --upgrade build twine python3 -m build + find dist -name "*.whl" -exec pip install --force-reinstall {}[test] \; From 690467c461048a96fe6f5fc07452539a7d2396cc Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:00:48 +0100 Subject: [PATCH 14/20] Update doc --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79764166..86881530 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,8 @@ In case of breaking changes, you **must** give details about what features were ### Version number +Use `make bump` to bump the version number. + We follow the [semantic versioning](http://semver.org/) spec: any change impacts the version number, and the version number conveys API compatibility information **only**. Examples: From 668ea7dfa23a4b278fd1ece4c55e86f0014c6348 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:00:56 +0100 Subject: [PATCH 15/20] Changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fdee4d4..64c42788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### 0.22.0 [#206](https://github.com/openfisca/openfisca-france-data/pull/206) + +* Technical changes + - Replace Circle CI by GitHub Action + - Add publication to Conda https://anaconda.org/openfisca/openfisca-france-data + ### 0.21.1 [#207](https://github.com/openfisca/openfisca-france-data/pull/207) * Technical changes From 8dbb759976fad4d68f8d97d335603d8a563232b4 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:01:48 +0100 Subject: [PATCH 16/20] Remove Circle CI --- .circleci/config.yml | 141 ---------------------- .circleci/has-functional-changes.sh | 16 --- .circleci/is-version-number-acceptable.sh | 33 ----- .circleci/publish-git-tag.sh | 4 - .circleci/publish-python-package.sh | 4 - 5 files changed, 198 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/has-functional-changes.sh delete mode 100755 .circleci/is-version-number-acceptable.sh delete mode 100755 .circleci/publish-git-tag.sh delete mode 100755 .circleci/publish-python-package.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 5db5eebc..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,141 +0,0 @@ -version: 2 -jobs: - run_tests: - docker: - - image: python:3.7 - - steps: - - checkout - - - restore_cache: - key: v1-py3-apt-{{ checksum "setup.py" }} - - - restore_cache: - key: v1-py3-venv-{{ .Branch }}-{{ checksum "setup.py" }} - - - run: - name: Install HDF5 - command: | - apt-get update - apt-get -y install libhdf5-serial-dev - - - run: - name: Create a virtualenv - command: | - mkdir -p /tmp/venv/openfisca-france-data - python -m venv /tmp/venv/openfisca-france-data - - - run: - name: Activate virtualenv - command: echo "source /tmp/venv/openfisca-france-data/bin/activate" >> $BASH_ENV - - - run: - name: Install dependencies - command: make install - - - save_cache: - key: v1-py3-venv-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - /tmp/venv/openfisca-france-data - - - save_cache: - key: v1-py3-apt-{{ checksum "setup.py" }} - paths: - - /var/cache/apt/archives - - - run: - name: Run tests - command: env PYTEST_ADDOPTS="--exitfirst --cov=openfisca_france_data --no-cov-on-fail" make test - - - persist_to_workspace: - root: . - paths: - - .coverage - - check_version: - docker: - - image: python:3.7 - - steps: - - checkout - - - run: - name: Check version number has been properly updated - command: | - git fetch - .circleci/is-version-number-acceptable.sh - - submit_coverage: - docker: - - image: python:3.7 - - steps: - - checkout - - - attach_workspace: - at: . - - - restore_cache: - key: v1-py3-venv-{{ .Branch }}-{{ checksum "setup.py" }} - - - run: - name: Activate virtualenv - command: echo "source /tmp/venv/openfisca-france-data/bin/activate" >> $BASH_ENV - - - run: - name: Submit coverage to Coveralls - command: | - pip install --upgrade coveralls - coveralls - - - save_cache: - key: v1-py3-venv-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - /tmp/venv/openfisca-france-data - - deploy: - docker: - - image: python:3.7 - - environment: - PYPI_USERNAME: openfisca-bot - # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! - - steps: - - checkout - - - restore_cache: - key: v1-py3-venv-{{ .Branch }}-{{ checksum "setup.py" }} - - - run: - name: Activate virtualenv - command: echo "source /tmp/venv/openfisca-france-data/bin/activate" >> $BASH_ENV - - - run: - name: Check for functional changes - command: if ! .circleci/has-functional-changes.sh ; then circleci step halt ; fi - - - run: - name: Publish a git tag - command: .circleci/publish-git-tag.sh - - - run: - name: Upload a Python package to Pypi - command: .circleci/publish-python-package.sh - -workflows: - version: 2 - build: - jobs: - - check_version - - run_tests - - submit_coverage: - requires: - - run_tests - - deploy: - requires: - - check_version - - run_tests - filters: - branches: - only: master diff --git a/.circleci/has-functional-changes.sh b/.circleci/has-functional-changes.sh deleted file mode 100755 index fd7740af..00000000 --- a/.circleci/has-functional-changes.sh +++ /dev/null @@ -1,16 +0,0 @@ -#! /usr/bin/env bash - -IGNORE_FILES="README.md CONTRIBUTING.md Makefile .gitignore LICENSE* .circleci/* .github/* tests/*" -EXCLUDE_DIFF=$(echo " $IGNORE_FILES" | sed 's/ / :(exclude)/g') - -# --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit -LAST_TAGGED_COMMIT=$(git describe --tags --abbrev=0 --first-parent) - -# Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. -# shellcheck disable=SC2086 -if git diff-index --name-only --exit-code "$LAST_TAGGED_COMMIT" -- . $EXCLUDE_DIFF -then - echo "No functional changes detected." - exit 1 -else echo "The functional files above were changed." -fi diff --git a/.circleci/is-version-number-acceptable.sh b/.circleci/is-version-number-acceptable.sh deleted file mode 100755 index b368f259..00000000 --- a/.circleci/is-version-number-acceptable.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /usr/bin/env bash - -if [[ $CIRCLE_BRANCH == master ]] -then - echo "No need for a version check on master." - exit 0 -fi - -if ! "$PWD"/.circleci/has-functional-changes.sh -then - echo "No need for a version update." - exit 0 -fi - -current_version=$(python setup.py --version) - -if git rev-parse --verify --quiet "$current_version" -then - echo "Version $current_version already exists in commit:" - git --no-pager log -1 "$current_version" - echo - echo "Update the version number in setup.py before merging this branch into master." - echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated." - exit 1 -fi - -if ! "$PWD"/.circleci/has-functional-changes.sh | grep --quiet CHANGELOG.md -then - echo "CHANGELOG.md has not been modified, while functional changes were made." - echo "Explain what you changed before merging this branch into master." - echo "Look at the CONTRIBUTING.md file to learn how to write the changelog." - exit 2 -fi diff --git a/.circleci/publish-git-tag.sh b/.circleci/publish-git-tag.sh deleted file mode 100755 index 48d99b68..00000000 --- a/.circleci/publish-git-tag.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env bash - -git tag "$(python setup.py --version)" -git push --tags # update the repository version diff --git a/.circleci/publish-python-package.sh b/.circleci/publish-python-package.sh deleted file mode 100755 index f55321ec..00000000 --- a/.circleci/publish-python-package.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env bash - -python setup.py bdist_wheel # build this package in the dist directory -twine upload dist/* --username "$PYPI_USERNAME" --password "$PYPI_PASSWORD" # publish From d3c7cb52000844ad9f5a819de3e8f6977ef054aa Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:36:43 +0100 Subject: [PATCH 17/20] Set version to 0.22.0 --- .conda/meta.yaml | 2 +- setup.cfg | 4 ++-- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 53625a19..f3c066ad 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -4,7 +4,7 @@ ############################################################################### {% set name = "openfisca-france-data" %} -{% set version = "0.22" %} +{% set version = "0.22.0" %} package: name: {{ name|lower }} diff --git a/setup.cfg b/setup.cfg index 75782079..bb63e6a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,8 +20,8 @@ testpaths = tests python_files = **/*.py [bumpver] -current_version = "0.22" -version_pattern = "MAJOR.MINOR" +current_version = "0.22.0" +version_pattern = "MAJOR.MINOR.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = True # tag = True diff --git a/setup.py b/setup.py index 82b6f552..7419f468 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.21.1", + version = "0.22.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From d369efe1b2f14ebdcacb356eed505598566cea02 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 3 Feb 2023 18:52:56 +0100 Subject: [PATCH 18/20] Update version --- .github/workflows/workflow.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 91ce14ad..d765aac7 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -15,9 +15,9 @@ jobs: python-version: ["3.8.9", "3.9.9"] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache build @@ -42,11 +42,11 @@ jobs: runs-on: ubuntu-20.04 needs: [ build ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Fetch all the tags - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9.9 - name: Cache build @@ -67,11 +67,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TERM: xterm-256color # To colorize output of make tasks. steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9.9 - name: Cache build @@ -93,9 +93,9 @@ jobs: os: [ "ubuntu-20.04" ] # On peut ajouter "macos-latest" si besoin python-version: ["3.8.9", "3.9.9"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache build @@ -110,11 +110,11 @@ jobs: runs-on: ubuntu-20.04 needs: [ lint-files, test-python ] # Last job to run steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Fetch all the tags - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8.9 - name: Check version number has been properly updated @@ -130,11 +130,11 @@ jobs: outputs: status: ${{ steps.stop-early.outputs.status }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Fetch all the tags - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9.9 - id: stop-early @@ -148,11 +148,11 @@ jobs: PYPI_USERNAME: openfisca-bot PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Fetch all the tags - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9.9 - name: Cache build From 0d4e074e906f8e0243c1ee070948ddf50a03c2a4 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:06:43 +0100 Subject: [PATCH 19/20] Remove circleCI badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 955bdbc8..a8a4b789 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![Newsletter](https://img.shields.io/badge/newsletter-subscribe!-informational.svg?style=flat)](mailto:contact%40openfisca.org?subject=Subscribe%20to%20your%20newsletter%20%7C%20S'inscrire%20%C3%A0%20votre%20newsletter&body=%5BEnglish%20version%20below%5D%0A%0ABonjour%2C%0A%0AVotre%C2%A0pr%C3%A9sence%C2%A0ici%C2%A0nous%C2%A0ravit%C2%A0!%20%F0%9F%98%83%0A%0AEnvoyez-nous%20cet%20email%20pour%20que%20l'on%20puisse%20vous%20inscrire%20%C3%A0%20la%20newsletter.%20%0A%0AAh%C2%A0!%20Et%20si%20vous%20pouviez%20remplir%20ce%20petit%20questionnaire%2C%20%C3%A7a%20serait%20encore%20mieux%C2%A0!%0Ahttps%3A%2F%2Fgoo.gl%2Fforms%2F45M0VR1TYKD1RGzX2%0A%0AAmiti%C3%A9%2C%0AL%E2%80%99%C3%A9quipe%20OpenFisca%0A%0A%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%20ENGLISH%20VERSION%20%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%0A%0AHi%2C%20%0A%0AWe're%20glad%20to%20see%20you%20here!%20%F0%9F%98%83%0A%0APlease%20send%20us%20this%20email%2C%20so%20we%20can%20subscribe%20you%20to%20the%20newsletter.%0A%0AAlso%2C%20if%20you%20can%20fill%20out%20this%20short%20survey%2C%20even%20better!%0Ahttps%3A%2F%2Fgoo.gl%2Fforms%2FsOg8K1abhhm441LG2%0A%0ACheers%2C%0AThe%20OpenFisca%20Team) [![Twitter](https://img.shields.io/badge/twitter-follow%20us!-9cf.svg?style=flat)](https://twitter.com/intent/follow?screen_name=openfisca) [![Slack](https://img.shields.io/badge/slack-join%20us!-blueviolet.svg?style=flat)](mailto:contact%40openfisca.org?subject=Join%20you%20on%20Slack%20%7C%20Nous%20rejoindre%20sur%20Slack&body=%5BEnglish%20version%20below%5D%0A%0ABonjour%2C%0A%0AVotre%C2%A0pr%C3%A9sence%C2%A0ici%C2%A0nous%C2%A0ravit%C2%A0!%20%F0%9F%98%83%0A%0ARacontez-nous%20un%20peu%20de%20vous%2C%20et%20du%20pourquoi%20de%20votre%20int%C3%A9r%C3%AAt%20de%20rejoindre%20la%20communaut%C3%A9%20OpenFisca%20sur%20Slack.%0A%0AAh%C2%A0!%20Et%20si%20vous%20pouviez%20remplir%20ce%20petit%20questionnaire%2C%20%C3%A7a%20serait%20encore%20mieux%C2%A0!%0Ahttps%3A%2F%2Fgoo.gl%2Fforms%2F45M0VR1TYKD1RGzX2%0A%0AN%E2%80%99oubliez%20pas%20de%20nous%20envoyer%20cet%20email%C2%A0!%20Sinon%2C%20on%20ne%20pourra%20pas%20vous%20contacter%20ni%20vous%20inviter%20sur%20Slack.%0A%0AAmiti%C3%A9%2C%0AL%E2%80%99%C3%A9quipe%20OpenFisca%0A%0A%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%20ENGLISH%20VERSION%20%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%3D%0A%0AHi%2C%20%0A%0AWe're%20glad%20to%20see%20you%20here!%20%F0%9F%98%83%0A%0APlease%20tell%20us%20a%20bit%20about%20you%20and%20why%20you%20want%20to%20join%20the%20OpenFisca%20community%20on%20Slack.%0A%0AAlso%2C%20if%20you%20can%20fill%20out%20this%20short%20survey%2C%20even%20better!%0Ahttps%3A%2F%2Fgoo.gl%2Fforms%2FsOg8K1abhhm441LG2.%0A%0ADon't%20forget%20to%20send%20us%20this%20email!%20Otherwise%20we%20won't%20be%20able%20to%20contact%20you%20back%2C%20nor%20invite%20you%20on%20Slack.%0A%0ACheers%2C%0AThe%20OpenFisca%20Team) -[![CircleCI](https://img.shields.io/circleci/project/github/openfisca/openfisca-france-data/master.svg?style=flat)](https://circleci.com/gh/openfisca/openfisca-france-data) [![Coveralls](https://img.shields.io/coveralls/github/openfisca/openfisca-france-data/master.svg?style=flat)](https://coveralls.io/github/openfisca/openfisca-france-data?branch=master) [![Python](https://img.shields.io/pypi/pyversions/openfisca-france-data.svg)](https://pypi.python.org/pypi/openfisca-france-data) [![PyPi](https://img.shields.io/pypi/v/openfisca-france-data.svg?style=flat)](https://pypi.python.org/pypi/openfisca-france-data) From cae76cef13e9b2e768b864f6df8da61e2a6d615b Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 7 Feb 2023 08:53:31 +0100 Subject: [PATCH 20/20] Update docs --- .conda/README.md | 86 +++++++++++++++++++----------------------------- .conda/meta.yaml | 4 +-- README.md | 26 ++++++++++++++- 3 files changed, 61 insertions(+), 55 deletions(-) diff --git a/.conda/README.md b/.conda/README.md index 67d38c97..db39784f 100644 --- a/.conda/README.md +++ b/.conda/README.md @@ -1,52 +1,34 @@ -# Publication vers Anaconda - -Ce readme décrit la partie publication vers conda, pour utiliser le paquet conda résultant, voir [le readme principal](https://github.com/openfisca/openfisca-france/tree/publish-to-conda#installez-un-environnement-virtuel-avec-conda). - -Pour envoyer un paquet vers conda il faut obligatoirement un fichier `meta.yaml` qui décrit le package. - -Nous avons fait le choix d'utiliser le paquet PyPi comme source du paquet Conda, pour s'assurer que l'on publie bien la même chose sur les deux plateformes. - -L'upload automatique est fait de la façon suivante par la CI uniquement si l'étape de livraison sur PyPi s'est bien déroulée : -- Installation de Miniconda. -- Récupération par le script `.github/get_pypi_info.py` des informations du package sur PyPi. -- Ecriture de ces informations par ce même script dans le fichier `.conda/meta.yaml`. -- Exécution de `conda build` pour construire et publier le paquet conformément au fichier `.conda/meta.yaml`. Cette étape nécessite la variable de CI ANACONDA_TOKEN. - -Pour valider que tout a fonctionné, une étape `test-on-windows` a été ajoutée en fin de CI. Cette étape récupère le paquet conda sur une machine Windows et exécute les tests. - -**A noter** : Le paquet OpenFisca-France est aussi publié sur `conda-forge`, pour cela voir [le feedstock](https://github.com/openfisca/openfisca-france-feedstock/tree/master/recipe) - -C'est le channel `conda-forge` qui est le channel stable à conseiller aux utilisateurs. Le channel `openfisca` reçoit les derniers paquets de façon automatique. - -## Etapes préparatoires pour arriver à cette automatisation - -- Création d'un compte sur https://anaconda.org. -- Création d'un token sur https://anaconda.org/openfisca/settings/access avec le droit _Allow write access to the API site_. Attention il expire le 2023/01/13. - -- Mis en place du token dans GitHub Action sous le nom de variable ANACONDA_TOKEN. - -### Publication manuelle du package - -Les étapes suivantes peuvent être réalisées sous Windows pour tester la publication : - -_Cela fonctionne aussi sous macOS et Linux, à condition d'adapter les chemins._ - -- Editer `.conda/meta.yaml` pour vérifier son contenu. -- Installer [MiniConda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html) si vous n'avez pas déjà [AnaConda](https://www.anaconda.com/products/individual). -- Puis dans le terminal saisissez les instructions suivantes : - - `conda install -c anaconda conda-build anaconda-client` Pour installer les outils indispensable. - - `conda build --croot c:\temp .conda` L'option `--croot ` est nécessaire sous Windows à cause des chemins trop long. - - `conda install -c anaconda anaconda-client` Pour installer l'outil en ligne de commande. - - `anaconda login` Pour vous connecter avec le compte _openfisca_, voir le Keepass OpenFisca. - - `anaconda upload c:\temp\noarch\openfisca-france--py_0.tar.bz2` pour publier le package. -- Vérifier que tout c'est bien passé sur https://anaconda.org/search?q=openfisca. - -## Test avec Docker - -Docker peut être utilisé pour valider le bon fonctionnement - -``` -docker run -i -t continuumio/anaconda3 /bin/bash -conda install -c openfisca -c conda-forge openfisca-france-dev -openfisca test --country-package openfisca_france tests -``` \ No newline at end of file +# Publish to conda + +There are two ways to publish to conda: + +- A fully automatic in CI that publish to an "openfisca" channel. See below for more information. +- A more complex for Conda-Forge. + +We use both for openfisca-core but only _openfisca channel_ for _OpenFisca-France-Data_. + +## Automatic upload + +The CI automaticaly build the Conda package and publish it when merging on master, see the `.github/workflow.yml`, step `publish-to-conda`. + +## Manual actions made to make it works the first time + +- Create an account on https://anaconda.org. +- Create a token on https://anaconda.org/openfisca/settings/access with _Allow write access to the API site_. Warning, it expire on 2025/01/14. +- Put the token in a CI env variable ANACONDA_TOKEN. + +## Manual actions before CI exist + +To create the package you can do the following in the project root folder: + +- Edit `.conda/meta.yaml` and update it if needed: + - Version number + - Dependencies + +- Install `conda-build` to build the package and [anaconda-client](https://github.com/Anaconda-Platform/anaconda-client) to push the package to anaconda.org: + - `conda install -c anaconda conda-build anaconda-client` + +- Build & Upload package: + - `conda build .conda` + - `anaconda login` + - `anaconda upload openfisca-france-data--py_0.tar.bz2` diff --git a/.conda/meta.yaml b/.conda/meta.yaml index f3c066ad..38851711 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -27,7 +27,7 @@ requirements: - click >=8.0.0,<9.0.0 - matplotlib >=3.1.1,<4.0.0 - multipledispatch >=0.6.0,<1.0.0 - - openFisca-france >=113.0.0 + - openFisca-france >=113.0.0,<120.0.0 # Max 120 because of a bug in OF : https://github.com/openfisca/openfisca-france/issues/1996 - openFisca-survey-manager >=0.44.2,<1.0.0 - wquantiles >=0.3.0,<1.0.0 @@ -43,7 +43,7 @@ test: outputs: - name: openfisca-france-data - - name: openfisca-france-test + - name: openfisca-france-data-test build: noarch: python requirements: diff --git a/README.md b/README.md index a8a4b789..26031b9c 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,37 @@ Ceci est la liste les données traitées : ## Environment -OpenFisca-France-Data runs runs on Python 3.7. +OpenFisca-France-Data runs on Python from version 3.7. More recent versions should work, but are not tested. > Backward compatibility with Python 2.7 has been dropped since January 1st, 2019. ## Installation +### Installing with `conda` + +Since version `0.22.0`, you could use `conda` to install OpenFisca-France-Data. + +Conda is the easiest way to use OpenFisca under Windows as by installing Anaconda you will get: +- Python +- The package manager [Anaconda.org](https://docs.anaconda.com/anacondaorg/user-guide/) +- A virtual environment manager : [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) +- A GUI [Anaconda Navigator](https://docs.anaconda.com/anaconda/navigator/index.html) if you choose to install the full [Anaconda](https://www.anaconda.com/products/individual) + +If you are familiar with the command line you could use [Miniconda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html), which needs very much less disk space than Anaconda. + +After installing conda, run these commands in an `Anaconda Powershell Prompt`: +- `conda create --name openfisca python=3.9` to create an `openfisca` environment. +- `conda activate openfisca` to use your new environment. + +Then, choose one of the following options according to your use case: +- `conda install -c conda-forge -c openfisca openfisca-france-data` for default dependencies, +- or `conda install -c conda-forge -c openfisca openfisca-france-data-test` if you want all the dependencies needed to contribute to the project. + +For information on how we publish to conda, see [.conda/README.md](.conda/README.md). + +### Development installation + You can follow this installation if you want to: * run one of OpenFisca-France-Data scripts, * develop and contribute to OpenFisca-France-Data source code. Please be welcomed! :)