Cleanup tests

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+### 0.8.4 [1679](https://github.com/openfisca/openfisca-france-data/pull/169)
+
+- Cleanup tests
+
 ### 0.8.3 [167](https://github.com/openfisca/openfisca-france-data/pull/167)
 
 - Fix `test_build_cerfa_fields_by_column_name`

openfisca_france_data/aggregates.py

@@ -432,7 +432,13 @@ def get_data_frame(self,
 
 def load_actual_data(year = None):
     assert year is not None
+
+    if not Config:
+        log.info("No actual data available")
+        return
+
     parser = Config()
+
     # Cotisations CSG -CRDS
     try:
         directory = os.path.join(

openfisca_france_data/erfs/scenario.py

@@ -42,6 +42,5 @@ class ErfsSurveyScenario(AbstractErfsSurveyScenario):
         'zone_apl',
         ]
 
-    def __init__(self, year = None):
-        assert year is not None
+    def __init__(self, year: int) -> None:
         self.year = year

openfisca_france_data/erfs_fpr/input_data_builder/__init__.py

@@ -3,6 +3,8 @@
 
 import logging
 
+from multipledispatch import dispatch  # type: ignore
+
 from openfisca_france_data.erfs_fpr.input_data_builder import (
     step_01_preprocessing as preprocessing,
     # step_02_imputation_loyer as imputation_loyer,

openfisca_france_data/surveys.py

@@ -2,10 +2,13 @@
 
 
 import logging
-import numpy as np
-import pandas as pd
+import pandas
+import numpy
+
+from typing import Optional
 
 from openfisca_core import periods, simulations  # type: ignore
+from openfisca_core.taxbenefitsystems import TaxBenefitSystem  # type: ignore
 from openfisca_france_data import base_survey as base  # type: ignore
 from openfisca_survey_manager.scenarios import AbstractSurveyScenario  # type: ignore
 
@@ -69,7 +72,7 @@ def build_input_data_from_test_case(self, test_case_scenario):
         for ident in ["idmen", "idfoy", "idfam"]:
             array_by_variable[ident] = range(axe["count"])
 
-        input_data_frame = pd.DataFrame(array_by_variable)
+        input_data_frame = pandas.DataFrame(array_by_variable)
 
         for qui in ["quimen", "quifoy", "quifam"]:
             input_data_frame[qui] = 0
@@ -79,12 +82,12 @@ def build_input_data_from_test_case(self, test_case_scenario):
     @classmethod
     def create(
             cls,
+            tax_benefit_system: Optional[TaxBenefitSystem] = None,
+            baseline_tax_benefit_system: Optional[TaxBenefitSystem] = None,
             input_data_type = None,
-            baseline_tax_benefit_system = None,
             reform = None,
             reform_key = None,
-            tax_benefit_system = None,
-            year = None,
+            year: int = None,
             ):
 
         assert year is not None
@@ -204,11 +207,11 @@ def new_simulation_from_array_dict(
 
     for role_var in ["quifam", "quifoy", "quimen"]:
         if role_var not in array_dict:
-            array_dict[role_var] = np.zeros(global_count, dtype = int)
+            array_dict[role_var] = numpy.zeros(global_count, dtype = int)
 
     for id_var in ["idfam", "idfoy", "idmen"]:
         if id_var not in array_dict:
-            array_dict[id_var] = np.arange(global_count, dtype = int)
+            array_dict[id_var] = numpy.arange(global_count, dtype = int)
 
     column_by_name = tax_benefit_system.variables
 
@@ -250,6 +253,6 @@ def new_simulation_from_array_dict(
         assert array.size == entity.count, \
             f"Bad size for {column_name}: {array.size} instead of {entity.count}"
 
-        holder.array = np.array(array, dtype = holder.variable.dtype)
+        holder.array = numpy.array(array, dtype = holder.variable.dtype)
 
     return simulation

setup.py

@@ -11,8 +11,8 @@
 
 setup(
     name = "OpenFisca-France-Data",
-    version = "0.8.3",
-    description = "OpenFisca-France module to work with French survey data (ERFS and ERFS-FPR)",
+    version = "0.8.4",
+    description = "OpenFisca-France module to work with French survey data",
     long_description = long_description,
     author = "OpenFisca Team",
     author_email = "contact@openfisca.fr",
@@ -46,6 +46,7 @@
             "mypy >= 0.670, < 1.0.0",
             "pytest >= 4.3.0, < 5.0.0",
             "pytest-cov >= 2.6.0, < 3.0.0",
+            "toolz >= 0.9.0, < 1.0.0",
             ],
         },
     packages = find_packages(exclude = ("docs", "tests")),

tests/conftest.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+
+import os
+import pytest
+import pandas
+
+from openfisca_core.taxbenefitsystems import TaxBenefitSystem  # type: ignore
+from openfisca_france_data import france_data_tax_benefit_system  # type: ignore
+from openfisca_france_data.erfs_fpr.scenario import (  # type: ignore
+    ErfsFprSurveyScenario,
+    )
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+
+hdf5_file_realpath = os.path.join(
+    current_dir,
+    "fixtures",
+    "erfs",
+    "fake_openfisca_input_data.h5",
+    )
+
+csv_file_realpath = os.path.join(
+    current_dir,
+    "fixtures",
+    "erfs",
+    "fake_openfisca_input_data.csv",
+    )
+
+
+@pytest.fixture
+def tax_benefit_system() -> TaxBenefitSystem:
+    return france_data_tax_benefit_system
+
+
+@pytest.fixture
+def survey_scenario(tax_benefit_system: TaxBenefitSystem):
+    def _survey_scenario(year: int) -> ErfsFprSurveyScenario:
+        return ErfsFprSurveyScenario.create(
+            tax_benefit_system = tax_benefit_system,
+            year = year,
+            )
+
+    return _survey_scenario
+
+
+@pytest.fixture
+def fake_input_data():
+    def _fake_input_data_frame(year: int) -> pandas.DataFrame:
+        try:
+            input_data_frame = pandas.read_hdf(hdf5_file_realpath, key = str(year))
+
+        except Exception:
+            input_data_frame = pandas.read_csv(csv_file_realpath)
+
+        input_data_frame.rename(
+            columns = dict(
+                sali = "salaire_imposable",
+                choi = "chomage_imposable",
+                rsti = "retraite_imposable",
+                ),
+            inplace = True,
+            )
+
+        input_data_frame.loc[0, "salaire_imposable"] = 20000
+        input_data_frame.loc[1, "salaire_imposable"] = 10000
+
+        for idx in [2, 6]:
+            input_data_frame.loc[idx] = input_data_frame.loc[1].copy()
+            input_data_frame.loc[idx, "salaire_imposable"] = 0
+            input_data_frame.loc[idx, "quifam"] = idx
+            input_data_frame.loc[idx, "quifoy"] = idx
+            input_data_frame.loc[idx, "quimen"] = idx
+
+            if idx < 4:
+                input_data_frame.loc[idx, "age"] = 10
+
+            else:
+                input_data_frame.loc[idx, "age"] = 24
+                input_data_frame.loc[idx, "age"] = 24
+
+        input_data_frame.reset_index(inplace = True)
+
+        return input_data_frame
+
+    return _fake_input_data_frame

tests/erfs_fpr/integration/test_af.py

@@ -1,13 +1,11 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
-#%%
 
-from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario
 
+from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario
+from openfisca_france_data.reforms.inversion_directe_salaires import inversion_directe_salaires
 
-survey_scenario = get_survey_scenario(year = 2012, reform_key = 'inversion_directe_salaires')
+survey_scenario = get_survey_scenario(year = 2012, reform = inversion_directe_salaires)
 
-#%%
 data_frame_by_entity = survey_scenario.create_data_frame_by_entity(
     variables = [
         'af_base',

tests/erfs_fpr/integration/test_al.py

@@ -1,15 +1,11 @@
-#! /usr/bin/env python2
 # -*- coding: utf-8 -*-
 
-#%%
-
 
 from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario
+from openfisca_france_data.reforms.inversion_directe_salaires import inversion_directe_salaires
 
+survey_scenario = get_survey_scenario(year = 2012, reform = inversion_directe_salaires)
 
-survey_scenario = get_survey_scenario(year = 2012, reform_key = 'inversion_directe_salaires')
-
-#%%
 data_frame_by_entity = survey_scenario.create_data_frame_by_entity(
     variables = [
         'age',

tests/erfs_fpr/integration/test_impot_revenu.py

@@ -1,22 +1,13 @@
-#! /usr/bin/env python2
 # -*- coding: utf-8 -*-
 
-#%%
-
 
 import os
 
 from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario
+from openfisca_france_data.reforms.inversion_directe_salaires import inversion_directe_salaires
 
+survey_scenario = get_survey_scenario(year = 2012, reform = inversion_directe_salaires)
 
-is_travis = 'TRAVIS' in os.environ
-
-if is_travis:
-    exit()
-
-survey_scenario = get_survey_scenario(year = 2012, reform_key = 'inversion_directe_salaires')
-
-#%%
 data_frame_by_entity = survey_scenario.create_data_frame_by_entity(
     variables = [
         'irpp',

tests/erfs_fpr/integration/test_salaire_imposable.py

@@ -1,13 +1,13 @@
-#! /usr/bin/env python2
 # -*- coding: utf-8 -*-
 
-#%%
 
 from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario
+from openfisca_france_data.reforms.inversion_directe_salaires import inversion_directe_salaires
+
+survey_scenario = get_survey_scenario(year = 2012, reform = inversion_directe_salaires)
 
 
 def test_inversion():
-    survey_scenario = get_survey_scenario(year = 2012, reform_key = 'inversion_directe_salaires')
     survey_scenario.summarize_variable('salaire_de_base', force_compute = True)
 
     survey_scenario.summarize_variable('salaire_imposable', force_compute = True)

tests/test_aggregate.py

@@ -3,28 +3,39 @@
 
 import pytest
 
-from openfisca_france_data.aggregates import Aggregates
-from openfisca_france_data.erfs.scenario import ErfsSurveyScenario
-from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario
+from openfisca_france_data.aggregates import Aggregates  # type: ignore
+from openfisca_france_data.erfs_fpr.scenario import (  # type: ignore
+    ErfsFprSurveyScenario,
+    )
 
 
-@pytest.mark.skip(reason = 'assert False != (None is not None)')
-def test_erfs_survey_simulation(year = 2009):
-    survey_scenario = ErfsSurveyScenario.create(year = year)
+@pytest.mark.skip(reason = "ValueError: NumPy boolean array indexing assignment...")
+def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 2009):
+    # On ititialise le survey scenario
+    survey_scenario = survey_scenario(year)
+
+    # On charge les données
+    input_data = fake_input_data(year)
+
+    # On initialise le survey scenario
+    survey_scenario.init_from_data(data = dict(input_data_frame = input_data))
+
+    # On calcule les agrégats
     aggregates = Aggregates(survey_scenario = survey_scenario)
-    aggregates.compute_aggregates()
+    aggregates.compute_aggregates(use_baseline = False)
     return aggregates.base_data_frame
 
 
-@pytest.mark.skip(reason = "TypeError: create() got an unexpected keyword argument 'data_year'")
-def test_erfs_aggregates_reform():
-    '''
-    test aggregates value with data
-    :param year: year of data and simulation to test agregates
-    :param reform: optional argument, put an openfisca_france.refoms object, default None
-    '''
-    survey_scenario = ErfsFprSurveyScenario.create(data_year = 2012, year = 2015, reform_key = 'plf2015')
+@pytest.mark.skip(
+    reason = "TypeError: create() got an unexpected keyword argument 'data_year'",
+    )
+def test_erfs_fpr_aggregates_reform():
+    survey_scenario = ErfsFprSurveyScenario.create(
+        data_year = 2012,
+        year = 2015,
+        reform_key = 'plf2015',
+        )
+
     aggregates = Aggregates(survey_scenario = survey_scenario)
     base_data_frame = aggregates.compute_aggregates()
-
     return aggregates, base_data_frame

tests/test_calibration.py

@@ -6,21 +6,30 @@
 
 import pytest
 
-from openfisca_survey_manager.calibration import Calibration
-from openfisca_france_data.erfs.scenario import ErfsSurveyScenario
+from openfisca_survey_manager.calibration import Calibration  # type: ignore
 
 
 @pytest.fixture
-def location():
-    return pkg_resources.get_distribution('openfisca-france-data').location
+def location() -> str:
+    return pkg_resources.get_distribution("openfisca-france-data").location
 
 
-@pytest.mark.skip(reason = 'assert None is not None')
-def test_calibration():
-    year = 2009
-    survey_scenario = ErfsSurveyScenario().create(year = year)
-    calibration = Calibration(survey_scenario)
-    calibration.parameters['method'] = 'linear'
+@pytest.mark.skip(
+    reason = "'Calibration' object has no attribute 'set_inputs_margins_from_file'",
+    )
+def test_calibration(survey_scenario, fake_input_data, location, year: int = 2009):
+    # On ititialise le survey scenario
+    survey_scenario = survey_scenario(year)
+
+    # On charge les données
+    input_data = fake_input_data(year)
+
+    # On initialise le survey scenario
+    survey_scenario.init_from_data(data = dict(input_data_frame = input_data))
+
+    # On fait la calibration
+    calibration = Calibration(survey_scenario, period = year)
+    calibration.parameters["method"] = "linear"
     calibration.total_population = calibration.initial_total_population * 1.123
 
     filename = os.path.join(
@@ -31,7 +40,8 @@ def test_calibration():
         )
 
     calibration.set_inputs_margins_from_file(filename, 2006)
-    calibration.set_parameters('invlo', 3)
-    calibration.set_parameters('up', 3)
-    calibration.set_parameters('method', 'logit')
-    calibration.calibrate()
+    calibration.set_parameters("invlo", 3)
+    calibration.set_parameters("up", 3)
+    calibration.set_parameters("method", "logit")
+
+    assert calibration.calibrate()