Make package compatible with Python 3

.circleci/config.yml

@@ -135,12 +135,8 @@ jobs:
           name: Run tests
           command: |
             pip install --upgrade pip twine wheel
-            pip install . --upgrade
-            pip install --editable git+https://github.com/etalab/biryani.git@master#egg=Biryani
-            pip install --editable git+https://github.com/openfisca/country-template.git@master#egg=OpenFisca-Country-Template
-            pip install --editable git+https://github.com/openfisca/extension-template.git@master#egg=OpenFisca-Extension-Template
             pip install --editable .[test] --upgrade
-            nosetests core/test_base_functions.py
+            make test
 
   deploy:
     working_directory: ~/openfisca-core

CHANGELOG.md

@@ -1,9 +1,10 @@
 # Changelog
 
-### 23.0.2 [#645](https://github.com/openfisca/openfisca-core/pull/645)
+### 23.1.0 [#660](https://github.com/openfisca/openfisca-core/pull/660)
 
-* Technical improvement
-* Details:
+Make package compatible with Python 3
+
+### 23.0.2 [#645](https://github.com/openfisca/openfisca-core/pull/645)
 
 Start adapting OpenFisca to Python 3
 - Imports are now all absolute imports

openfisca_core/columns.py

@@ -139,7 +139,7 @@ def transform_value_to_json(self, value, use_label = False):
         if isinstance(value, dict):
             return collections.OrderedDict(
                 (str(period), self.transform_dated_value_to_json(dated_value, use_label = use_label))
-                for period, dated_value in value.iteritems()
+                for period, dated_value in value.items()
                 )
         return self.transform_dated_value_to_json(value, use_label = use_label)
 

openfisca_core/commons.py

@@ -2,7 +2,7 @@
 
 # The following two variables and the is_unicode function are there to bridge string types across Python 2 & 3
 unicode_type = u"".__class__
-basestring_type = (str, unicode_type)
+basestring_type = (b"".__class__, unicode_type)
 
 
 def to_unicode(string):
@@ -18,7 +18,7 @@ def to_unicode(string):
         return string
 
     # Next line only gets triggered if the code is run in python 2
-    return unicode(string, 'utf-8')
+    return string.decode('utf-8')
 
 
 class Dummy(object):
@@ -37,9 +37,10 @@ def empty_clone(original):
 
 
 def stringify_array(array):
-    """Generate a clean string representation of a NumPY array.
+    """
+        Generate a clean string representation of a NumPY array.
     """
     return u'[{}]'.format(u', '.join(
-        unicode(cell)
+        to_unicode(cell)
         for cell in array
         )) if array is not None else u'None'

openfisca_core/conv.py

@@ -66,7 +66,7 @@ def embed_error(value, error_key, error):
         return None
     if isinstance(value, dict):
         if isinstance(error, dict):
-            for child_key, child_error in error.iteritems():
+            for child_key, child_error in error.items():
                 child_error = embed_error(value.get(child_key), error_key, child_error)
                 if child_error is not None:
                     value.setdefault(error_key, {})[child_key] = child_error
@@ -75,13 +75,13 @@ def embed_error(value, error_key, error):
         return None
     if isinstance(value, list) and isinstance(error, dict):
         if all(isinstance(key, int) and 0 <= key < len(value) for key in error):
-            for child_index, child_error in error.iteritems():
+            for child_index, child_error in error.items():
                 child_error = embed_error(value[child_index], error_key, child_error)
                 if child_error is not None:
                     return error
             return None
         if all(isinstance(key, basestring) and key.isdigit() and 0 <= int(key) < len(value) for key in error):
-            for child_key, child_error in error.iteritems():
+            for child_key, child_error in error.items():
                 child_error = embed_error(value[int(child_key)], error_key, child_error)
                 if child_error is not None:
                     return error

openfisca_core/data_storage.py

@@ -56,7 +56,7 @@ def delete(self, period = None):
 
         self._arrays = {
             period_item: value
-            for period_item, value in self._arrays.iteritems()
+            for period_item, value in self._arrays.items()
             if not period.contains(period_item)
             }
 
@@ -73,10 +73,10 @@ def get_memory_usage(self):
 
         nb_arrays = sum([
             len(array_or_dict) if isinstance(array_or_dict, dict) else 1
-            for array_or_dict in self._arrays.itervalues()
+            for array_or_dict in self._arrays.values()
             ])
 
-        array = self._arrays.values()[0]
+        array = list(self._arrays.values())[0]
         if isinstance(array, dict):
             array = array.values()[0]
         return dict(
@@ -153,7 +153,7 @@ def delete(self, period = None):
         if period is not None:
             self._files = {
                 period_item: value
-                for period_item, value in self._files.iteritems()
+                for period_item, value in self._files.items()
                 if not period.contains(period_item)
                 }
 

openfisca_core/decompositionsxml.py

@@ -21,7 +21,7 @@ def transform_node_xml_json_to_json(node_xml_json, root = True):
         node_json['@context'] = u'https://openfisca.fr/contexts/decomposition.jsonld'
     node_json['@type'] = 'Node'
     children_json = []
-    for key, value in node_xml_json.iteritems():
+    for key, value in node_xml_json.items():
         if key == 'color':
             node_json['color'] = [
                 int(color)

openfisca_core/entities.py

@@ -71,7 +71,7 @@ def init_variable_values(self, entity_object, entity_id):
             try:
                 self.check_variable_defined_for_entity(variable_name)
             except ValueError as e:  # The variable is defined for another entity
-                raise SituationParsingError(path_in_json, e.message)
+                raise SituationParsingError(path_in_json, e.args[0])
             except VariableNotFound as e:  # The variable doesn't exist
                 raise SituationParsingError(path_in_json, e.message, code = 404)
 
@@ -85,7 +85,7 @@ def init_variable_values(self, entity_object, entity_id):
                 try:
                     period = make_period(date)
                 except ValueError as e:
-                    raise SituationParsingError(path_in_json, e.message)
+                    raise SituationParsingError(path_in_json, e.args[0])
                 if value is not None:
                     array = holder.buffer.get(period)
                     if array is None:
@@ -137,11 +137,11 @@ def clone(self, new_simulation):
         new = self.__class__(new_simulation)
         new_dict = new.__dict__
 
-        for key, value in self.__dict__.iteritems():
+        for key, value in self.__dict__.items():
             if key == '_holders':
                 new_dict[key] = {
                     name: holder.clone(new)
-                    for name, holder in self._holders.iteritems()
+                    for name, holder in self._holders.items()
                     }
             elif key != 'simulation':
                 new_dict[key] = value
@@ -151,7 +151,7 @@ def clone(self, new_simulation):
     def __getattr__(self, attribute):
         projector = get_projector_from_shortcut(self, attribute)
         if not projector:
-            raise Exception(u"Entity {} has no attribute {}".format(self.key, attribute))
+            raise AttributeError(u"Entity {} has no attribute {}".format(self.key, attribute))
         return projector
 
     @classmethod
@@ -246,12 +246,12 @@ def get_holder(self, variable_name):
     def get_memory_usage(self, variables = None):
         holders_memory_usage = {
             variable_name: holder.get_memory_usage()
-            for variable_name, holder in self._holders.iteritems()
+            for variable_name, holder in self._holders.items()
             if variables is None or variable_name in variables
             }
 
         total_memory_usage = sum(
-            holder_memory_usage['total_nb_bytes'] for holder_memory_usage in holders_memory_usage.itervalues()
+            holder_memory_usage['total_nb_bytes'] for holder_memory_usage in holders_memory_usage.values()
             )
 
         return dict(
@@ -410,9 +410,6 @@ def init_from_json(self, entities_json):
                     unallocated_person, self.simulation.persons.plural, self.key)
                 )
 
-        #  Deprecated attribute used by deprecated projection opertors, such as sum_by_entity
-        self.roles_count = self.members_legacy_role.max() + 1
-
     def init_members(self, roles_json, entity_id):
         for role_id, role_definition in roles_json.items():
             check_type(role_definition, list, [self.plural, entity_id, role_id])
@@ -467,12 +464,24 @@ def members_role(self, members_role):
 
     @property
     def roles_count(self):
+        warnings.warn(' '.join([
+            u"entity.roles_count is deprecated.",
+            u"Since OpenFisca Core 23.0, this attribute has strictly no effect, and it is not necessary to set it."
+            ]),
+            Warning
+            )
         if self._roles_count is None:
             self._roles_count = self.members_legacy_role.max() + 1
         return self._roles_count
 
     @roles_count.setter
     def roles_count(self, value):
+        warnings.warn(' '.join([
+            u"entity.roles_count is deprecated.",
+            u"Since OpenFisca Core 23.0, this attribute has strictly no effect, and it is not necessary to set it."
+            ]),
+            Warning
+            )
         self._roles_count = value
 
     @property

openfisca_core/errors.py

@@ -4,7 +4,8 @@
 
 
 class VariableNotFound(Exception):
-    """Exception raised when a variable has been queried but is not defined in the TaxBenefitSystem.
+    """
+        Exception raised when a variable has been queried but is not defined in the TaxBenefitSystem.
     """
 
     def __init__(self, variable_name, tax_benefit_system):
@@ -26,4 +27,5 @@ def __init__(self, variable_name, tax_benefit_system):
             u"Look at its changelog to learn about renames and removals and update your code. If it is an official package,",
             u"it is probably available on <https://github.com/openfisca/{0}/blob/master/CHANGELOG.md>.".format(country_package_name)
             ])
-        Exception.__init__(self, message.encode('utf-8'))
+        self.message = message
+        Exception.__init__(self, self.message.encode('utf-8'))

openfisca_core/holders.py

@@ -4,6 +4,7 @@
 from __future__ import division
 import logging
 import os
+import sys
 import warnings
 
 import numpy as np
@@ -53,7 +54,7 @@ def clone(self, entity):
         new = empty_clone(self)
         new_dict = new.__dict__
 
-        for key, value in self.__dict__.iteritems():
+        for key, value in self.__dict__.items():
             if key not in ('entity', 'formula', 'simulation'):
                 new_dict[key] = value
 
@@ -168,9 +169,11 @@ def set_input(self, period, array):
                 error_message
                 )
         if self.variable.is_neutralized:
+            warning_message = u"You cannot set a value for the variable {}, as it has been neutralized. The value you provided ({}) will be ignored.".format(self.variable.name, array)
+            if sys.version_info < (3, 0):
+                warning_message = warning_message.encode('utf-8')
             return warnings.warn(
-                u"You cannot set a value for the variable {}, as it has been neutralized. The value you provided ({}) will be ignored."
-                .format(self.variable.name, array).encode('utf-8'),
+                warning_message,
                 Warning
                 )
         if self.variable.set_input:
@@ -267,10 +270,10 @@ def extra_params_to_json_key(extra_params, period):
                 for cell in array.tolist()
                 ]
         value_json = {}
-        for period, array_or_dict in self._memory_storage._arrays.iteritems():
+        for period, array_or_dict in self._memory_storage._arrays.items():
             if type(array_or_dict) == dict:
                 values_dict = {}
-                for extra_params, array in array_or_dict.iteritems():
+                for extra_params, array in array_or_dict.items():
                     extra_params_key = extra_params_to_json_key(extra_params, period)
                     values_dict[str(extra_params_key)] = [
                         transform_dated_value_to_json(cell, use_label = use_label)
@@ -283,10 +286,10 @@ def extra_params_to_json_key(extra_params, period):
                     for cell in array_or_dict.tolist()
                     ]
         if self._disk_storage:
-            for period, file_or_dict in self._disk_storage._files.iteritems():
+            for period, file_or_dict in self._disk_storage._files.items():
                 if type(file_or_dict) == dict:
                     values_dict = {}
-                    for extra_params, file in file_or_dict.iteritems():
+                    for extra_params, file in file_or_dict.items():
                         extra_params_key = extra_params_to_json_key(extra_params, period)
                         values_dict[str(extra_params_key)] = [
                             transform_dated_value_to_json(cell, use_label = use_label)
@@ -303,15 +306,16 @@ def extra_params_to_json_key(extra_params, period):
     # Legacy method called by to_value_json
     def get_extra_param_names(self, period):
         formula = self.variable.get_formula(period)
-        return formula.func_code.co_varnames[3:]
+        return formula.__code__.co_varnames[3:]
 
 
 class PeriodMismatchError(ValueError):
     def __init__(self, variable_name, period, definition_period, message):
         self.variable_name = variable_name
         self.period = period
         self.definition_period = definition_period
-        ValueError.__init__(self, message)
+        self.message = message
+        ValueError.__init__(self, self.message)
 
 
 def set_input_dispatch_by_period(holder, period, array):

openfisca_core/indexed_enums.py

@@ -19,6 +19,8 @@ def __init__(self, name):
     # Bypass the slow Enum.__eq__
     __eq__ = object.__eq__
 
+    __hash__ = object.__hash__  # In Python 3, __hash__ must be defined if __eq__ is defined to stay hashable
+
     @classmethod
     def encode(cls, array):
         """

openfisca_core/json_to_test_case.py

@@ -41,7 +41,7 @@ def check_variable(value, key):
             raise ValueError(u"Invalid value {} for variable {}. Error: {}".format(value, key, error).encode('utf-8'))
         entity_json[key] = value
 
-    for key, value in entity_json.iteritems():
+    for key, value in entity_json.items():
         if key == 'id':
             check_id(value)
         elif valid_roles.get(key) is not None:
@@ -53,7 +53,7 @@ def check_variable(value, key):
             from .taxbenefitsystems import VariableNotFound
             raise VariableNotFound(key, tax_benefit_system)
 
-    for role in valid_roles.itervalues():
+    for role in valid_roles.values():
         if role.max != 1 and entity_json.get(role.plural) is None:  # by convention, if no one in the entity has a given non-unique role, it should be [] in the JSON
             entity_json[role.plural] = []
 
@@ -70,7 +70,7 @@ def check_entities_and_role(test_case, tax_benefit_system, state):
 
     test_case = deepcopy(test_case)  # Avoid side-effects on other references to test_case
     entity_classes = {entity_class.plural: entity_class for entity_class in tax_benefit_system.entities}
-    for entity_type_name, entities in test_case.iteritems():
+    for entity_type_name, entities in test_case.items():
         if entity_classes.get(entity_type_name) is None:
             raise ValueError(u"Invalid entity name: {}".format(entity_type_name).encode('utf-8'))
         entities, error = conv.pipe(
@@ -95,7 +95,7 @@ def check_entities_and_role(test_case, tax_benefit_system, state):
         for entity_json in entities:
             check_entity_fields(entity_json, entity_class, valid_roles, tax_benefit_system)
 
-    for entity_class in entity_classes.itervalues():
+    for entity_class in entity_classes.values():
         if test_case.get(entity_class.plural) is None:
             test_case[entity_class.plural] = []  # by convention, all entities must be declared in the test_case