From e0849648a2577033db2483c391971a2bf54e2eed Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 21 Jul 2023 18:24:53 +0200 Subject: [PATCH 01/28] Remove pkg_resources --- openfisca_france_data/__init__.py | 10 ++++++++-- openfisca_france_data/aggregates.py | 11 ++++------- .../erfs/input_data_builder/step_08_final.py | 4 ++-- openfisca_france_data/erfs/old/datatable.py | 6 +++--- tests/test_calibration.py | 7 ++++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 81ef2499..969e5a53 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -1,8 +1,9 @@ import inspect +import importlib import logging import os -import pkg_resources import pandas +from pathlib import Path from openfisca_core import reforms # type: ignore @@ -13,6 +14,11 @@ from openfisca_france_data.model.base import * # noqa analysis:ignore +openfisca_france_data_location = Path( + importlib.metadata.distribution('openfisca-survey-manager').files[0] + ).parent + + log = logging.getLogger(__name__) @@ -206,7 +212,7 @@ def apply(self): COUNTRY_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join( - pkg_resources.get_distribution('openfisca-france-data').location, + openfisca_france_data_location, 'openfisca_france_data', 'plugins', 'aggregates', diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index ae47df5d..9730239f 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -5,12 +5,9 @@ import numpy as np import pandas as pd -import pkg_resources -import os -from datetime import datetime from openfisca_survey_manager.aggregates import AbstractAggregates -from openfisca_france_data import AGGREGATES_DEFAULT_VARS # type: ignore +from openfisca_france_data import openfisca_france_data_location, AGGREGATES_DEFAULT_VARS # type: ignore log = logging.getLogger(__name__) @@ -45,7 +42,7 @@ def load_actual_data(self, year = None): if target_source == "taxipp": taxipp_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", @@ -91,7 +88,7 @@ def load_actual_data(self, year = None): elif target_source == "ines": ines_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", @@ -128,7 +125,7 @@ def load_actual_data(self, year = None): result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") return result - + def to_csv(self, path = None, absolute = True, amount = True, beneficiaries = True, default = 'actual', relative = True, target = "reform"): """Saves the table to csv.""" diff --git a/openfisca_france_data/erfs/input_data_builder/step_08_final.py b/openfisca_france_data/erfs/input_data_builder/step_08_final.py index 3ff828a7..4f7c160c 100644 --- a/openfisca_france_data/erfs/input_data_builder/step_08_final.py +++ b/openfisca_france_data/erfs/input_data_builder/step_08_final.py @@ -10,6 +10,7 @@ from openfisca_survey_manager.temporary import temporary_store_decorator +from openfisca_france_data import openfisca_france_data_location from openfisca_france_data.utils import ( check_structure, control, @@ -20,6 +21,7 @@ set_variables_default_value, ) + log = logging.getLogger(__name__) @@ -202,8 +204,6 @@ def final(temporary_store = None, year = None, check = True): print_id(final2) # # TODO: merging with patrimoine log.info(' traitement des zones apl') - import pkg_resources - openfisca_france_data_location = pkg_resources.get_distribution('openfisca-france-data').location zone_apl_imputation_data_file_path = os.path.join( openfisca_france_data_location, 'openfisca_france_data', diff --git a/openfisca_france_data/erfs/old/datatable.py b/openfisca_france_data/erfs/old/datatable.py index b2eac78b..3de1bee4 100644 --- a/openfisca_france_data/erfs/old/datatable.py +++ b/openfisca_france_data/erfs/old/datatable.py @@ -1,5 +1,4 @@ import os -import pkg_resources import sys import gc @@ -18,8 +17,9 @@ #from openfisca_france.data.sources.config import DATA_DIR -openfisca_france_location = pkg_resources.get_distribution('openfisca-france-data').location -CONFIG_DIR = os.path.join(openfisca_france_location) +from openfisca_france_data import openfisca_france_data_location + +CONFIG_DIR = os.path.join(openfisca_france_data_location) #ERF_HDF5_DATA_DIR = os.path.join(SRC_PATH,'countries','france','data', 'erf') diff --git a/tests/test_calibration.py b/tests/test_calibration.py index c56f0308..dee2ee60 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -1,14 +1,16 @@ -import pkg_resources import os import pytest from openfisca_survey_manager.calibration import Calibration # type: ignore +from openfisca_france_data import openfisca_france_data_location + + @pytest.fixture def location() -> str: - return pkg_resources.get_distribution("openfisca-france-data").location + return openfisca_france_data_location def test_calibration(survey_scenario, fake_input_data, location, year: int = 2009): @@ -34,4 +36,3 @@ def test_calibration(survey_scenario, fake_input_data, location, year: int = 200 pre_cal_weight = survey_scenario.calculate_variable("wprm", period = year) calibration.set_calibrated_weights() assert pre_cal_weight * 1.123 == survey_scenario.calculate_variable("wprm", period = year) - From 0782168b5e2e46c355e22bde0fb4de575549b759 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 21 Jul 2023 19:17:51 +0200 Subject: [PATCH 02/28] Fix importlib metadata import --- openfisca_france_data/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 969e5a53..035770e3 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -1,5 +1,5 @@ import inspect -import importlib +from importlib import metadata import logging import os import pandas @@ -15,7 +15,7 @@ openfisca_france_data_location = Path( - importlib.metadata.distribution('openfisca-survey-manager').files[0] + metadata.distribution('openfisca-survey-manager').files[0] ).parent From 07c027991abd8a09c78fe8faa1e352500d7a74e2 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 25 Jul 2023 12:09:53 +0200 Subject: [PATCH 03/28] Improve package location --- openfisca_france_data/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 035770e3..84388307 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -14,9 +14,7 @@ from openfisca_france_data.model.base import * # noqa analysis:ignore -openfisca_france_data_location = Path( - metadata.distribution('openfisca-survey-manager').files[0] - ).parent +openfisca_france_data_location = Path(__file__).parent.parent log = logging.getLogger(__name__) From 7b84173922fe445192937118e0f07fe51e6d4deb Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 25 Jul 2023 12:17:29 +0200 Subject: [PATCH 04/28] Upgrade survey-manager dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47c9ea66..b45b2cac 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager >=1, <2.0.0", + "openFisca-survey-manager >= 1.1.5, < 2.0.0", "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { From 2c522897701dda04100abadd08452cc9a82eb1b8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 6 Jul 2023 11:03:19 +0200 Subject: [PATCH 05/28] Use specific survey-manager-branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b45b2cac..fb5c0dac 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager >= 1.1.5, < 2.0.0", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { From 25b90c3939bc336d2e42d6638479f44abe77bec9 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 8 Aug 2023 18:01:51 +0200 Subject: [PATCH 06/28] Change percenile calculation --- openfisca_france_data/model/common.py | 6 +++--- openfisca_france_data/model/survey_variables.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 13049ba8..dad888f6 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -190,10 +190,10 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) + method = 2 + decile, values = mark_weighted_percentiles(niveau_de_vie, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) # Alternative method - # method = 2 - # decile, values = mark_weighted_percentiles(niveau_de_vie, labels, pondmen, method, return_quantiles = True) - decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) + # decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) return decile diff --git a/openfisca_france_data/model/survey_variables.py b/openfisca_france_data/model/survey_variables.py index e9cde28c..16ef928b 100644 --- a/openfisca_france_data/model/survey_variables.py +++ b/openfisca_france_data/model/survey_variables.py @@ -4,7 +4,7 @@ class menage_ordinaire(Variable): value_type = int is_period_size_independent = True - default_value = True + default_value = 1 entity = Menage definition_period = YEAR From 973d897aa232340af539ec613067a05364382fc8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 8 Aug 2023 18:06:54 +0200 Subject: [PATCH 07/28] Typo --- openfisca_france_data/model/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index dad888f6..e3591bd3 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -191,7 +191,7 @@ def formula(foyer_fiscal, period): menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) method = 2 - decile, values = mark_weighted_percentiles(niveau_de_vie, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) + decile, values = mark_weighted_percentiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) # Alternative method # decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) return decile From 01a80ccf27bee276eb1bc6f8d2a111f0c9b108af Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 10 Aug 2023 13:00:42 +0200 Subject: [PATCH 08/28] Remove wquantiles dep --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index fb5c0dac..4e15c3f4 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,6 @@ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", - "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { "test": [ From 7b5d45dbf6427a75c5202d8d763e89ef65069158 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 9 Sep 2023 08:06:26 +0200 Subject: [PATCH 09/28] Use standard method to compute decile_rfr_par_part --- openfisca_france_data/model/common.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index e3591bd3..336aa83a 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -228,11 +228,9 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) - # Alternative method # method = 2 - # decile, values = mark_weighted_percentiles(niveau_de_vie, labels, pondmen, method, return_quantiles = True) - decile, values = weighted_quantiles( - rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) + decile, values = mark_weighted_percentiles( + rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) return decile From e9458856ec92ea23e9b68f5a5ac62783ce34a57c Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 9 Sep 2023 08:48:59 +0200 Subject: [PATCH 10/28] Fix missing method for decile_rfr_par_part --- openfisca_france_data/model/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 336aa83a..c798e5f8 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -228,7 +228,7 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) - # method = 2 + method = 2 decile, values = mark_weighted_percentiles( rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) return decile From ab1d32a1fbe8346b65f34d84cbdcef23a70582b1 Mon Sep 17 00:00:00 2001 From: cgl Date: Sat, 9 Sep 2023 10:18:25 +0200 Subject: [PATCH 11/28] Fix update error in tax and benefits system reform --- openfisca_france_data/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 84388307..b0c9b6f6 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -6,6 +6,7 @@ from pathlib import Path from openfisca_core import reforms # type: ignore +from openfisca_core.errors import VariableNameConflictError import openfisca_france # type: ignore @@ -145,7 +146,8 @@ def apply(self): continue try: self.add_variable(variable) - except AttributeError: + except VariableNameConflictError: + log.warning(f"{variable.__name__} has been updated") self.update_variable(variable) From f6fc7a47f11363b75094e8546b3f87f7c74259bc Mon Sep 17 00:00:00 2001 From: cgl Date: Mon, 11 Sep 2023 11:05:31 +0200 Subject: [PATCH 12/28] corrige log --- openfisca_france_data/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index b0c9b6f6..43389933 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -147,7 +147,7 @@ def apply(self): try: self.add_variable(variable) except VariableNameConflictError: - log.warning(f"{variable.__name__} has been updated") + # log.debug(f"{variable.__name__} has been updated in openfisca-france-data") self.update_variable(variable) From 30c6509a98966030aa327090facc822120fc7bea Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 13 Oct 2023 09:50:32 +0200 Subject: [PATCH 13/28] =?UTF-8?q?mets=20=C3=A0=20jour=20la=20d=C3=A9pense?= =?UTF-8?q?=20=C3=A0=20openfisca-survey-manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openfisca_france_data/surveys.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index a595570d..e1853070 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -7,7 +7,7 @@ from openfisca_core.model_api import Enum # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore from openfisca_france_data import base_survey as base # type: ignore -from openfisca_survey_manager.scenarios import AbstractSurveyScenario # type: ignore +from openfisca_survey_manager.scenarios.abstract_scenario import AbstractSurveyScenario # type: ignore log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 4e15c3f4..d0bc3d89 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-sceanrio#egg=openfisca-survey-manager", ], extras_require = { "test": [ From a774ef6dca61c03034b79cd6ef3a9e6d42368314 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 13 Oct 2023 10:28:02 +0200 Subject: [PATCH 14/28] fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0bc3d89..1f84143b 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-sceanrio#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { "test": [ From 40766271ea65ca76129f3148ffb333b25a2ec11f Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 13 Oct 2023 21:16:45 +0100 Subject: [PATCH 15/28] Adapt ErfsSurveyScenario --- openfisca_france_data/surveys.py | 20 +++++++++++++------- tests/test_calibration.py | 17 ++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index e1853070..5d20955b 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -7,12 +7,13 @@ from openfisca_core.model_api import Enum # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore from openfisca_france_data import base_survey as base # type: ignore -from openfisca_survey_manager.scenarios.abstract_scenario import AbstractSurveyScenario # type: ignore +from openfisca_survey_manager.scenarios.reform_scenario import ReformScenario # type: ignore + log = logging.getLogger(__name__) -class AbstractErfsSurveyScenario(AbstractSurveyScenario): +class AbstractErfsSurveyScenario(ReformScenario): """ Parties communes entre ERFS et ERFS PFR @@ -116,10 +117,15 @@ def create( else: survey_scenario = cls(year = year) - survey_scenario.set_tax_benefit_systems( - tax_benefit_system = tax_benefit_system, - baseline_tax_benefit_system = baseline_tax_benefit_system, - ) + if baseline_tax_benefit_system: + survey_scenario.set_tax_benefit_systems(dict( + reform = tax_benefit_system, + baseline = baseline_tax_benefit_system, + )) + else: + survey_scenario.set_tax_benefit_systems(dict( + baseline = tax_benefit_system, + )) survey_scenario.year = year survey_scenario.period = year @@ -162,7 +168,7 @@ def custom_initialize(self, simulation): assert variable in self.used_as_input_variables, \ f"{variable} is not a in the input_varaibles to be used {self.used_as_input_variables}" # noqa: E501 - if self.tax_benefit_system.variables[variable].value_type == Enum: + if simulation.tax_benefit_system.variables[variable].value_type == Enum: permanent_value = simulation.calculate( variable, period = periods.period(self.year).first_month, diff --git a/tests/test_calibration.py b/tests/test_calibration.py index dee2ee60..a9293435 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -24,15 +24,14 @@ def test_calibration(survey_scenario, fake_input_data, location, year: int = 200 survey_scenario.init_from_data(data = dict(input_data_frame = input_data)) # On fait la calibration - calibration = Calibration(survey_scenario) - calibration.parameters["method"] = "linear" - calibration.total_population = calibration.initial_total_population * 1.123 + parameters = dict( + method = "logit", + invlo = 3, + up = 3, + ) - calibration.set_parameters("invlo", 3) - calibration.set_parameters("up", 3) - calibration.set_parameters("method", "logit") - - calibration.calibrate() pre_cal_weight = survey_scenario.calculate_variable("wprm", period = year) - calibration.set_calibrated_weights() + target_entity_count = pre_cal_weight.sum() * 1.123 + survey_scenario.calibrate(target_entity_count = target_entity_count, entity = "menage", parameters = parameters, period = year) + assert pre_cal_weight * 1.123 == survey_scenario.calculate_variable("wprm", period = year) From 75cc1c816d8ec9e237951d7fe7db57b680d4ce67 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 16 Oct 2023 09:58:16 +0100 Subject: [PATCH 16/28] Lint --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f84143b..03c26a56 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { "test": [ From dd2c78a3b4850f18a8e0f4d0f4fcb76007b50f4d Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 16 Oct 2023 15:11:15 +0100 Subject: [PATCH 17/28] Remove unused variable use_baseline --- openfisca_france_data/surveys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index 5d20955b..e6ec6f95 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -189,7 +189,6 @@ def custom_initialize(self, simulation): except ValueError as e: log.debug(f"Dealing with: {e}") if sum(simulation.calculate_add(variable, simulation_period.offset(offset))) != sum(permanent_value): - use_baseline = self.baseline_simulation == simulation simulation.delete_arrays(variable, simulation_period.offset(offset)) simulation.set_input( variable, From 14ea2d2d6226d3f48f46dd3bdff94b181d292faa Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 17 Oct 2023 15:08:37 +0100 Subject: [PATCH 18/28] Migrate aggregate attribute --- openfisca_france_data/aggregates.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 9730239f..e3e6b4e7 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -35,10 +35,10 @@ def __init__(self, survey_scenario = None, target_source = None): super().__init__(survey_scenario = survey_scenario) self.target_source = target_source - def load_actual_data(self, year = None): + def load_actual_data(self, period = None): target_source = self.target_source assert target_source in ["ines", "taxipp", "france_entiere"], "les options possible pour source_cible sont ines, taxipp ou france_entiere" - assert year is not None + assert period is not None if target_source == "taxipp": taxipp_aggregates_file = Path( @@ -59,15 +59,15 @@ def load_actual_data(self, year = None): .rename(columns = {"unnamed: 0": "description"}) .dropna(subset = ["annee 2019", "annee 2018", "annee 2017", "annee 2016"], how = "all") ) - if f"annee {year}" not in df: + if f"annee {period}" not in df: return df = ( - df[["variable_openfisca", f"annee {year}"]] + df[["variable_openfisca", f"annee {period}"]] .dropna() .rename(columns = { "variable_openfisca": "variable", - f"annee {year}": year, + f"annee {period}": period, }) ) @@ -75,13 +75,13 @@ def load_actual_data(self, year = None): df.loc[df.variable.str.startswith("nombre")] .set_index("variable") .rename(index = lambda x : x.replace("nombre_", "")) - .rename(columns = {year: "actual_beneficiaries"}) + .rename(columns = {period: "actual_beneficiaries"}) ) / self.beneficiaries_unit amounts = ( df.loc[~df.variable.str.startswith("nombre")] .set_index("variable") - .rename(columns = {year: "actual_amount"}) + .rename(columns = {period: "actual_amount"}) ) / self.amount_unit result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") @@ -93,7 +93,7 @@ def load_actual_data(self, year = None): "assets", "aggregats", "ines", - f"ines_{year}.json" + f"ines_{period}.json" ) with open(ines_aggregates_file, 'r') as f: @@ -112,7 +112,7 @@ def load_actual_data(self, year = None): "assets", "aggregats", "france_entiere", - f"france_entiere_{year}.json" + f"france_entiere_{period}.json" ) with open(ines_aggregates_file, 'r') as f: @@ -133,7 +133,7 @@ def to_csv(self, path = None, absolute = True, amount = True, beneficiaries = Tr if os.path.isdir(path): now = datetime.now() - file_path = os.path.join(path, 'Aggregates_%s_%s_%s.%s' % (self.target_source,self.year,now.strftime('%d-%m-%Y'), "csv")) + file_path = os.path.join(path, 'Aggregates_%s_%s_%s.%s' % (self.target_source, self.period, now.strftime('%d-%m-%Y'), "csv")) else: file_path = path From 763210e0f8641b89f59e38f6f05b830853d9dae8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 16:52:45 +0200 Subject: [PATCH 19/28] Fix tests --- openfisca_france_data/erfs/scenario.py | 4 +- .../erfs_fpr/get_survey_scenario.py | 4 +- openfisca_france_data/erfs_fpr/scenario.py | 5 +- openfisca_france_data/surveys.py | 25 ++-- tests/conftest.py | 2 +- .../erfs_fpr/integration/test_pivot_table.py | 2 +- tests/test_aggregate.py | 16 +-- tests/test_fake_survey_simulation.py | 124 ++++++++++-------- 8 files changed, 94 insertions(+), 88 deletions(-) diff --git a/openfisca_france_data/erfs/scenario.py b/openfisca_france_data/erfs/scenario.py index 46ad488b..d3d6148a 100644 --- a/openfisca_france_data/erfs/scenario.py +++ b/openfisca_france_data/erfs/scenario.py @@ -41,5 +41,5 @@ class ErfsSurveyScenario(AbstractErfsSurveyScenario): 'zone_apl', ] - def __init__(self, year: int) -> None: - self.year = year + def __init__(self, period: int) -> None: + self.period = period diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 448467fc..6b2ff551 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -126,14 +126,14 @@ def get_survey_scenario( survey_scenario = ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, baseline_tax_benefit_system = baseline_tax_benefit_system, - year = year, + period = year, ) else: assert varying_variable is not None, "You need to specify the varying variable." survey_scenario = ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, baseline_tax_benefit_system = baseline_tax_benefit_system, - year = year, + period = year, ) # taux marginaux !! survey_scenario.variation_factor = variation_factor diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index 7e9711d3..eef10cd5 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -63,8 +63,9 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "wprm_init", ] - def __init__(self, year: int) -> None: - self.year = year + def __init__(self, period: int) -> None: + # self.year = period + self.period = period @classmethod def build_input_data(cls, year: int) -> None: diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index e6ec6f95..1fb0db71 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -62,7 +62,7 @@ def build_input_data_from_test_case(self, test_case_scenario): ) array_by_variable = dict() - period = periods.period(str(self.year)) + period = periods.period(str(self.period)) for variable in self.used_as_input_variables: array_by_variable[variable] = simulation.calculate_add( @@ -88,10 +88,10 @@ def create( input_data_type = None, reform = None, reform_key = None, - year: int = None, + period: int = None, ): - assert year is not None + assert period is not None assert not ((reform is not None) and (reform_key is not None)) reform_is_provided = (reform is not None) or (reform_key is not None) @@ -112,10 +112,10 @@ def create( tax_benefit_system = reform if input_data_type is not None: - survey_scenario = cls(input_data_type = input_data_type, year = year) + survey_scenario = cls(input_data_type = input_data_type, period = period) else: - survey_scenario = cls(year = year) + survey_scenario = cls(period = period) if baseline_tax_benefit_system: survey_scenario.set_tax_benefit_systems(dict( @@ -127,8 +127,7 @@ def create( baseline = tax_benefit_system, )) - survey_scenario.year = year - survey_scenario.period = year + survey_scenario.period = period return survey_scenario @@ -156,27 +155,27 @@ def custom_initialize(self, simulation): "primes_fonction_publique", "retraite_brute", "retraite_imposable", - "salaire_de_base", + "salaire_imposable", "traitement_indiciaire_brut", # 'chomage_brut', ] - three_year_span_variables = input_variables + computed_variables_used_as_input + three_period_span_variables = input_variables + computed_variables_used_as_input - simulation_period = periods.period(self.year) - for variable in three_year_span_variables: + simulation_period = periods.period(self.period) + for variable in three_period_span_variables: assert variable in self.used_as_input_variables, \ f"{variable} is not a in the input_varaibles to be used {self.used_as_input_variables}" # noqa: E501 if simulation.tax_benefit_system.variables[variable].value_type == Enum: permanent_value = simulation.calculate( variable, - period = periods.period(self.year).first_month, + period = periods.period(self.period).first_month, ) else: permanent_value = simulation.calculate_add( variable, - period = self.year, + period = self.period, ) for offset in [-1, -2]: diff --git a/tests/conftest.py b/tests/conftest.py index 1b6cc218..933e6350 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,7 @@ def survey_scenario(tax_benefit_system: TaxBenefitSystem): def _survey_scenario(year: int) -> ErfsFprSurveyScenario: return ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) return _survey_scenario diff --git a/tests/erfs_fpr/integration/test_pivot_table.py b/tests/erfs_fpr/integration/test_pivot_table.py index 4ba92652..81b0bc04 100644 --- a/tests/erfs_fpr/integration/test_pivot_table.py +++ b/tests/erfs_fpr/integration/test_pivot_table.py @@ -60,4 +60,4 @@ def get_survey_scenario(kind = 'erfs_fpr', year = None): year = 2009 SurveyScenario = ErfsSurveyScenario - return SurveyScenario.create(year = year) + return SurveyScenario.create(period = year) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 61adb5a4..1e6d3fc0 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -8,12 +8,13 @@ ) -@pytest.mark.skip( - reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", - ) def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 2009): # On ititialise le survey scenario - survey_scenario = ErfsFprSurveyScenario.create(year) + survey_scenario = ErfsFprSurveyScenario.create( + tax_benefit_system = france_data_tax_benefit_system, + + period = year, + ) # On charge les données input_data = fake_input_data(year) @@ -23,16 +24,13 @@ def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 20 # On calcule les agrégats aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') - aggregates.compute_aggregates(use_baseline = False) + aggregates.compute_aggregates() return aggregates.base_data_frame -@pytest.mark.skip( - reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", - ) def test_erfs_fpr_aggregates_reform(fake_input_data, year:int = 2013): survey_scenario = ErfsFprSurveyScenario.create( - year = year, + period = year, reform_key = 'plf2015', baseline_tax_benefit_system = france_data_tax_benefit_system, ) diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 47694fcf..5faa7b79 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -10,29 +10,13 @@ from openfisca_france_data.reforms.old_openfisca_france_reforms.plf2015 import plf2015 # type: ignore -@pytest.fixture -def fake_calibration(tax_benefit_system): - def _fake_calibration(fake_input_data: pandas.DataFrame, year: int) -> Calibration: - input_data_frame = fake_input_data(year) - - survey_scenario = ErfsSurveyScenario.create( - tax_benefit_system = tax_benefit_system, - year = year, - ) - - survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) - - calibration = Calibration(survey_scenario = survey_scenario, period = year) - calibration.set_parameters("invlo", 3) - calibration.set_parameters("up", 3) - calibration.set_parameters("method", "logit") - - return calibration - - return _fake_calibration +parameters = dict( + invlo = 3, + up = 3, + method = "logit", + ) -@pytest.mark.skip(reason = "AssertionError: [0. 0.] != [20000. 10000.]") def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = 2006): input_data_frame = fake_input_data(year) assert input_data_frame.salaire_imposable.loc[0] == 20000 @@ -40,7 +24,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = survey_scenario = ErfsSurveyScenario.create( tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) @@ -48,7 +32,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert (input_data_frame.salaire_imposable.loc[0] == 20000).all() assert (input_data_frame.salaire_imposable.loc[1] == 10000).all() - simulation = survey_scenario.simulation + simulation = survey_scenario.simulations["baseline"] salaire_imposable = simulation.calculate_add("salaire_imposable", period = year) @@ -70,7 +54,10 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert (sal_2005 == sal_2006).all() for year, month in itertools.product(range(2003, 2004), range(1, 13)): - period = f"{year}-{month}" + if month < 10: + period = f"{year}-0{month}" + else: + period = f"{year}-{month}" assert ( simulation @@ -78,11 +65,13 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = ).all() for year, month in itertools.product(range(2004, 2007), range(1, 13)): - period = f"{year}-{month}" + if month < 10: + period = f"{year}-0{month}" + else: + period = f"{year}-{month}" assert ( - simulation - .calculate("salaire_imposable", period = period) == sal_2006 / 12 + simulation.calculate("salaire_imposable", period = period) == sal_2006 / 12 ).all() data_frame_by_entity = survey_scenario.create_data_frame_by_entity( @@ -106,66 +95,85 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "weight_familles", "revenu_disponible", "impo", - ] + ], ) return data_frame_by_entity, simulation -@pytest.mark.skip( - reason = "AttributeError: 'Calibration' object has no attribute 'simulation'", - ) -def test_fake_calibration_float(fake_calibration, year: int = 2006): - calibration = fake_calibration(year) - calibration.total_population = calibration.initial_total_population * 1.123 +def test_fake_calibration_float(tax_benefit_system, fake_input_data, year: int = 2006): + input_data_frame = fake_input_data(year) + survey_scenario = ErfsSurveyScenario.create( + tax_benefit_system = tax_benefit_system, + period = year, + ) + survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) + + initial_count = survey_scenario.compute_aggregate("revenu_disponible", aggfunc = "count", period = year) + target_entity_count = initial_count * 1.123 - revenu_disponible_target = 7e6 - calibration.set_target_margin("revenu_disponible", revenu_disponible_target) - calibration.calibrate() - calibration.set_calibrated_weights() + revenu_disponible_target = 7.7e6 + target_margins_by_variable = dict(revenu_disponible = revenu_disponible_target) - simulation = calibration.survey_scenario.simulation + survey_scenario.calibrate( + period = year, + target_margins_by_variable = target_margins_by_variable, + target_entity_count = target_entity_count, + parameters=parameters, + ) assert_near( - simulation.calculate("wprm", period=year).sum(), - calibration.total_population, + survey_scenario.calculate_variable("wprm", period=year).sum(), + target_entity_count, absolute_error_margin = None, relative_error_margin = 0.00001, ) assert_near( ( - + simulation.calculate("revenu_disponible", period = year) - * simulation.calculate("wprm", period = year) + + survey_scenario.calculate_variable("revenu_disponible", period = year) + * survey_scenario.calculate_variable("wprm", period = year) ).sum(), revenu_disponible_target, absolute_error_margin = None, relative_error_margin = 0.00001, ) - @pytest.mark.skip( - reason = "ValueError: Length of values does not match length of index", + reason = "ValueError: Unable to compute variable 'age' for period 2006: 'age' must be computed for a whole month. Should use an ge with year definition period", ) -def test_fake_calibration_age(fake_calibration, fake_input_data, year: int = 2006): - calibration = fake_calibration(fake_input_data, year) - survey_scenario = calibration.survey_scenario - calibration.total_population = calibration.initial_total_population * 1.123 - calibration.set_target_margin("age", [95, 130]) - calibration.calibrate() - calibration.set_calibrated_weights() +def test_fake_calibration_age(tax_benefit_system, fake_input_data, year: int = 2006): + input_data_frame = fake_input_data(year) + period = year + survey_scenario = ErfsSurveyScenario.create( + tax_benefit_system = tax_benefit_system, + period = period, + ) + survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) - simulation = survey_scenario.simulation + initial_count = survey_scenario.compute_aggregate("wprm", aggfunc = "count", period = year) + target_entity_count = initial_count * 1.123 + + target_margins_by_variable = { + "age": [95, 130] + } + + survey_scenario.calibrate( + period = period, + target_margins_by_variable = target_margins_by_variable, + target_entity_count = target_entity_count, + parameters=parameters, + ) assert_near( - simulation.calculate("wprm").sum(), - calibration.total_population, + survey_scenario.calculate_variable("wprm", period=period).sum(), + target_entity_count, absolute_error_margin = None, relative_error_margin = 0.00001, ) - age = (survey_scenario.simulation.calculate("age"),) - weight_individus = survey_scenario.simulation.calculate("weight_individus") + age = survey_scenario.calculate_variable("age", period = period) + weight_individus = survey_scenario.calculate_variable("weight_individus", period = year) for category, target in calibration.margins_by_variable["age"]["target"].items(): actual = ((age == category) * weight_individus).sum() / weight_individus.sum() @@ -197,7 +205,7 @@ def test_calculate_irpp_before_and_after_plf2015( survey_scenario = ErfsSurveyScenario.create( tax_benefit_system = reform, baseline_tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) # On charge les données From cce63a79128560544dfb16bf5356b58438e36d95 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 17:01:03 +0200 Subject: [PATCH 20/28] Fix tests --- openfisca_france_data/surveys.py | 2 +- tests/test_fake_survey_simulation.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index 1fb0db71..32e9b156 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -155,7 +155,7 @@ def custom_initialize(self, simulation): "primes_fonction_publique", "retraite_brute", "retraite_imposable", - "salaire_imposable", + "salaire_de_base", "traitement_indiciaire_brut", # 'chomage_brut', ] diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 5faa7b79..cf0f38d2 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -44,14 +44,14 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert age[0] == 77 assert age[1] == 37 - sal_2003 = simulation.calculate_add("salaire_imposable", period = 2003) - sal_2004 = simulation.calculate_add("salaire_imposable", period = 2004) - sal_2005 = simulation.calculate_add("salaire_imposable", period = 2005) - sal_2006 = simulation.calculate_add("salaire_imposable", period = 2006) + retraite_2003 = simulation.calculate_add("retraite_imposable", period = 2003) + retraite_2004 = simulation.calculate_add("retraite_imposable", period = 2004) + retraite_2005 = simulation.calculate_add("retraite_imposable", period = 2005) + retraite_2006 = simulation.calculate_add("retraite_imposable", period = 2006) - assert (sal_2003 == 0).all() - assert (sal_2004 == sal_2006).all() - assert (sal_2005 == sal_2006).all() + assert (retraite_2003 == 0).all() + assert (retraite_2004 == retraite_2006).all() + assert (retraite_2005 == retraite_2006).all() for year, month in itertools.product(range(2003, 2004), range(1, 13)): if month < 10: @@ -61,7 +61,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert ( simulation - .calculate_add("salaire_imposable", period = period) == 0 + .calculate_add("retraite_imposable", period = period) == 0 ).all() for year, month in itertools.product(range(2004, 2007), range(1, 13)): @@ -71,7 +71,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = period = f"{year}-{month}" assert ( - simulation.calculate("salaire_imposable", period = period) == sal_2006 / 12 + simulation.calculate("retraite_imposable", period = period) == retraite_2006 / 12 ).all() data_frame_by_entity = survey_scenario.create_data_frame_by_entity( @@ -80,6 +80,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "activite", "rsa_base_ressources_i", "menage_ordinaire_individus", + "retraite_imposable", "salaire_imposable", "salaire_net", "autonomie_financiere", @@ -97,7 +98,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "impo", ], ) - + BIM return data_frame_by_entity, simulation From 54bab67225f2f2ec0703ff107d264cb693af5c8c Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 17:40:50 +0200 Subject: [PATCH 21/28] Remove leftover --- tests/test_fake_survey_simulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index cf0f38d2..9eacf4d0 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -98,7 +98,6 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "impo", ], ) - BIM return data_frame_by_entity, simulation From 16868db0df1709b49f4ceeb69065a12875563166 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 19 Oct 2023 14:35:59 +0200 Subject: [PATCH 22/28] Fix path --- openfisca_france_data/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index e3e6b4e7..91ac656f 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -107,7 +107,7 @@ def load_actual_data(self, period = None): elif target_source == "france_entiere": ines_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", From e67d037e143754de4ffbf10fa851b8e6ae260103 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 24 Oct 2023 15:24:03 +0200 Subject: [PATCH 23/28] fix import os --- openfisca_france_data/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 91ac656f..79d0ab02 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -3,7 +3,7 @@ import json from pathlib import Path -import numpy as np +import os import pandas as pd from openfisca_survey_manager.aggregates import AbstractAggregates From b5d1182a87e97e157400f50a0dc45c58eb351b30 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 24 Oct 2023 15:28:43 +0200 Subject: [PATCH 24/28] fix import datetime --- openfisca_france_data/aggregates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 79d0ab02..1b1b9351 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -4,6 +4,7 @@ from pathlib import Path import os +from datetime import datetime import pandas as pd from openfisca_survey_manager.aggregates import AbstractAggregates From d9796bf28d016c0193c004e7947bf9ec12d1940d Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 24 Oct 2023 15:52:10 +0200 Subject: [PATCH 25/28] Fix typo (space). --- openfisca_france_data/aggregates.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 1b1b9351..f46a5ebc 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -121,9 +121,13 @@ def load_actual_data(self, period = None): result = pd.DataFrame(data['data']).drop(['source'], axis = 1) result['actual_beneficiaries'] = result. actual_beneficiaries / self.beneficiaries_unit - result['actual_amount'] = result. actual_amount / self.amount_unit + result['actual_amount'] = result.actual_amount / self.amount_unit - result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") + result = result[[ + "variable", + "actual_amount", + "actual_beneficiaries", + ]].set_index("variable") return result From 49a873c29a9b9e17f906bdb29babae75dfbce19d Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 2 Nov 2023 19:11:43 +0100 Subject: [PATCH 26/28] =?UTF-8?q?update=20d=C3=A9pendance=20openfisca-fran?= =?UTF-8?q?ce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 03c26a56..20eff165 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ python_requires = ">=3.9", install_requires = [ "multipledispatch >=0.6.0, <1.0.0", - "OpenFisca-France >=150.0.0, <154.0.0", + "OpenFisca-France >=150.0.0, <155.0.0", "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { From af576e69f344cf85469cac79cd75c4c10371a5f8 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 7 Nov 2023 17:30:07 +0100 Subject: [PATCH 27/28] =?UTF-8?q?bump=20+=20update=20d=C3=A9pendance=20ope?= =?UTF-8?q?nfisca-survey-manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++++ setup.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa3c413..bfd72c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### 3.0.0 [#241](https://github.com/openfisca/openfisca-france-data/pull/241) +- Breaking changes + +Adapte le dépôt au passage à openfisca-survey-manager 2.0.0 qui constitue une refactorisation de l'objet survey-scenario et des simulations qui sont dedans. Cela concerne donc les parties de ce dépôts qui héritent d'objets d'openfisca-survey-manager : +- `openfisca_france_data/aggregates.py` +- `openfisca_france_data/surveys.py` +Les autres modifications sont des adaptions syntaxique mineurs du fait de cette adaptation + ### 2.0.7 [#239](https://github.com/openfisca/openfisca-france-data/pull/239/files) * New features - Ajoute des nouveaux agrégats pour FranceAggregates diff --git a/setup.py b/setup.py index 20eff165..a49ead52 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.7", + version = "3.0.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <155.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager >=2.0.0, <2.1.0", ], extras_require = { "test": [ From ea591f1546370ccd6e0e1747aea6b540927aaea6 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 7 Nov 2023 21:31:16 +0100 Subject: [PATCH 28/28] fix config file directory --- openfisca_france_data/erfs_fpr/get_survey_scenario.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 6b2ff551..ed626143 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -10,6 +10,7 @@ from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario from openfisca_france_data import france_data_tax_benefit_system +from openfisca_survey_manager import default_config_files_directory from openfisca_france_data.model.id_variables import ( idmen_original, @@ -102,6 +103,7 @@ def get_survey_scenario( variation_factor: float = 0.03, varying_variable: str = None, survey_name: str = "input", + config_files_directory : str = default_config_files_directory, ) -> ErfsFprSurveyScenario: """Helper pour créer un `ErfsFprSurveyScenario`. @@ -153,6 +155,8 @@ def get_survey_scenario( input_data_table_by_entity_by_period = input_data_table_by_entity_by_period, survey = survey_name ) + data["config_files_directory"] = config_files_directory + # Les données peuvent venir en différents formats : #