Add IXMP_DATA / execute all tests in temporary directories

NOTICE.rst

@@ -1,7 +1,7 @@
 User Guidelines and Notice
 ==========================
 
-   Copyright 2017-18 IIASA Energy Program
+   Copyright © 2017–2019 IIASA Energy Program
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -46,7 +46,7 @@ the ix modeling platform  for scientific publications or technical reports:
   Michael Pimmer, Nikolay Kushin, Adriano Vinca, Alessio Mastrucci,
   Keywan Riahi, and Volker Krey.
   "The MESSAGEix Integrated Assessment Model and the ix modeling platform".
-  *Environmental Modelling & Software* 112:143-156, 2019. 
+  *Environmental Modelling & Software* 112:143-156, 2019.
   doi: `10.1016/j.envsoft.2018.11.012`_
   electronic pre-print available at `pure.iiasa.ac.at/15157/`_.
 
@@ -63,7 +63,7 @@ or scientific analysis using the MESSAGEix framework.
 B) Developing new model instances
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Researchers are welcome to develop new model instances using the |MESSAGEix| framework 
+Researchers are welcome to develop new model instances using the |MESSAGEix| framework
 for their own research interests. However, any such model must be named "MESSAGEix xxx" or "MESSAGEix-xxx",
 where 'xxx' is replaced by the name of the country/region, institutional affiliation or a similar identifying name.
 For example, the national model for South Africa developed by Orthofer et al. [1] is called "MESSAGEix South Africa".

README.md

@@ -14,7 +14,7 @@ examples for a generic model instance based on Dantzig's transport problem.
 
 ## License
 
-Copyright 2017-18 IIASA Energy Program
+Copyright © 2017–2019 IIASA Energy Program
 
 The platform package is licensed under the Apache License, Version 2.0 (the
 "License"); you may not use the files in this repository except in compliance

RELEASE_NOTES.md

@@ -1,6 +1,7 @@
 
 # Next Release
 
+- [#130](https://github.com/iiasa/ixmp/pull/130): Recognize `IXMP_DATA` environment variable for configuration and local databases.
 - [#129](https://github.com/iiasa/ixmp/pull/129): Fully implement `Scenario.clone()` across platforms (databases).
 - [#128](https://github.com/iiasa/ixmp/pull/128): New module `ixmp.testing` for reuse of testing utilities.
 - [#125](https://github.com/iiasa/ixmp/pull/125): Add functions to view and add regions for IAMC-style timeseries data.

ixmp/__init__.py

@@ -8,8 +8,6 @@
 from ixmp import (
     model_settings,
     utils,
-    default_paths,
-    config,
 )
 
 model_settings.register_model(

ixmp/cli.py

@@ -6,19 +6,21 @@
 def config():
     # construct cli
     parser = argparse.ArgumentParser()
-    db_config_path = 'Set default directory for database connection ' + \
-        'and configuration files.'
+    db_config_path = ('Set default directory for database connection and '
+                      'configuration files.')
     parser.add_argument('--db_config_path', help=db_config_path, default=None)
-    default_dbprops_file = 'Set default properties file for database connection.'
+    default_dbprops_file = ('Set default properties file for database '
+                            ' connection.')
     parser.add_argument('--default_dbprops_file',
                         help=default_dbprops_file, default=None)
     args = parser.parse_args()
 
-    # do the config
-    ixmp.config.config(
-        db_config_path=args.db_config_path,
-        default_dbprops_file=args.default_dbprops_file,
-    )
+    # Store the user-supplied configuration values
+    ixmp.config._config.set('DB_CONFIG_PATH', args.db_config_path)
+    ixmp.config._config.set('DEFAULT_DBPROPS_FILE', args.default_dbprops_file)
+
+    # Save the configuration to file
+    ixmp.config._config.save()
 
 
 def import_timeseries():

ixmp/config.py

@@ -1,50 +1,206 @@
+from itertools import chain
 import json
 import os
+try:
+    from pathlib import Path
+except ImportError:
+    # Python 2.7 compatibility
+    from pathlib2 import Path
+    FileNotFoundError = OSError
 
-from ixmp.default_path_constants import CONFIG_PATH
 from ixmp.utils import logger
 
 
-def get(key):
-    """Return key from configuration file"""
-    # TODO return sensible defaults even if the user has not given ixmp-config
-    if not os.path.exists(CONFIG_PATH):
-        raise RuntimeError(
-            'ixmp has not been configured, do so with `$ ixmp-config -h`')
+class Config(object):
+    """Configuration for ixmp.
 
-    with open(CONFIG_PATH, mode='r') as f:
-        data = json.load(f)
+    When imported, :mod:`ixmp` reads a configuration file `config.json` in the
+    first of the following directories:
 
-    if key not in data:
-        raise RuntimeError(
-            '{} is not configured, do so with `$ ixmp-config -h`'.format(key))
+    1. `IXMP_DATA`, if defined.
+    2. `${XDG_DATA_HOME}/ixmp`, if defined.
+    3. `$HOME/.local/share/ixmp`.
+    4. `$HOME/.local/ixmp` (used by ixmp <= 1.1).
 
-    return data[key]
+    The file may define either or both of the following configuration keys, in
+    JSON format:
 
+    - `DB_CONFIG_PATH`: location for database properties files. A
+      :class:`ixmp.Platform` instantiated with a relative path name for the
+      `dbprops` argument will locate the file first in the current working
+      directory, then in `DB_CONFIG_PATH`, then in the four directories above.
+    - `DEFAULT_DBPROPS_FILE`: path to a default database properties file. A
+      :class:`ixmp.Platform` instantiated with no arguments will use this file.
+    - `DEFAULT_LOCAL_DB_PATH`: path to a directory where a local directory
+      should be created. A :class:`ixmp.Platform` instantiated with
+      `dbtype='HSQLDB'` will create or reuse a database in this path.
 
-def config(db_config_path=None, default_dbprops_file=None):
-    """Update configuration file with new values"""
-    config = {}
+    Parameters
+    ----------
+    read : bool
+        Read `config.json` on startup.
 
-    if db_config_path:
-        db_config_path = os.path.abspath(os.path.expanduser(db_config_path))
-        config['DB_CONFIG_PATH'] = db_config_path
+    """
+    # User configuration keys
+    _keys = [
+        'DB_CONFIG_PATH',
+        'DEFAULT_DBPROPS_FILE',
+        'DEFAULT_LOCAL_DB_PATH',
+    ]
 
-    if default_dbprops_file:
-        default_dbprops_file = os.path.abspath(
-            os.path.expanduser(default_dbprops_file))
-        config['DEFAULT_DBPROPS_FILE'] = default_dbprops_file
+    def __init__(self, read=True):
+        # Default values
+        self.clear()
 
-    if os.path.exists(CONFIG_PATH):
-        with open(CONFIG_PATH, mode='r') as f:
-            data = json.load(f)
-        data.update(config)
-        config = data
+        # Read configuration from file; store the path at which it was located
+        if read:
+            self.read()
 
-    if config:
-        dirname = os.path.dirname(CONFIG_PATH)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
-        with open(CONFIG_PATH, mode='w') as f:
-            logger().info('Updating configuration file: {}'.format(CONFIG_PATH))
-            json.dump(config, f)
+    def _iter_paths(self):
+        """Yield recognized paths, in order of priority."""
+        try:
+            yield 'environment (IXMP_DATA)', Path(os.environ['IXMP_DATA'])
+        except KeyError:
+            pass
+
+        try:
+            yield 'environment (XDG_DATA_HOME)', \
+                Path(os.environ['XDG_DATA_HOME'], 'ixmp')
+        except KeyError:
+            pass
+
+        yield 'default', Path.home() / '.local' / 'share' / 'ixmp'
+        yield 'default (ixmp<=1.1)', Path.home() / '.local' / 'ixmp'
+
+    def _locate(self, filename=None, dirs=[]):
+        """Locate an existing *filename* in the ixmp config directories.
+
+        If *filename* is None (the default), only directories are located.
+        If *dirs* are provided, they are tried in order before the ixmp config
+        directories.
+        """
+        tried = []
+        dirs = map(lambda d: ('arg', d), dirs)
+        for label, directory in chain(dirs, self._iter_paths()):
+            try:
+                directory = Path(directory)
+            except TypeError:
+                # e.g. 'DB_CONFIG_PATH' via find_dbprops() is None
+                continue
+
+            # fix for R users (was in default_path_constants.py)
+            parts = filter(lambda p: p != 'Documents', directory.parts)
+            directory = Path(*parts)
+
+            if filename:
+                # Locate a specific file
+                if (directory / filename).exists():
+                    return directory / filename
+                else:
+                    tried.append(str(directory))
+            else:
+                # Locate an existing directory
+                if directory.exists():
+                    return directory
+                else:
+                    tried.append(str(directory))
+
+        if filename:
+            raise FileNotFoundError('Could not find {} in {!r}'
+                                    .format(filename, tried))
+        else:
+            raise FileNotFoundError('Could not find any of {!r}'.format(tried))
+
+    def read(self):
+        """Try to read configuration keys from file.
+
+        If successful, the configuration key 'CONFIG_PATH' is set to the path
+        of the file.
+        """
+        try:
+            config_path = self._locate('config.json')
+            contents = config_path.read_text()
+            self.values.update(json.loads(contents))
+            self.values['CONFIG_PATH'] = config_path
+        except FileNotFoundError:
+            pass
+        except json.JSONDecodeError:
+            print(config_path, contents)
+            raise
+
+    # Public methods
+
+    def get(self, key):
+        """Return the value of a configuration *key*."""
+        return self.values[key]
+
+    def set(self, key, value):
+        """Set configuration *key* to *value*."""
+        assert key in self.values
+        if value is None:
+            return
+        self.values[key] = value
+
+    def clear(self):
+        """Clear all configuration keys by setting their values to None."""
+        self.values = {key: None for key in self._keys}
+
+        # Set 'DEFAULT_LOCAL_DB_PATH'
+        # Use the first identifiable path
+        _, config_dir = next(self._iter_paths())
+        self.values['DEFAULT_LOCAL_DB_PATH'] = (config_dir / 'localdb'
+                                                'default')
+
+    def save(self):
+        """Write configuration keys to file.
+
+        `config.json` is created in the first of the ixmp configuration
+        directories that exists. Only non-null values are written.
+        """
+        # Use the first identifiable path
+        _, config_dir = next(self._iter_paths())
+        path = config_dir / 'config.json'
+
+        # TODO merge with existing configuration
+
+        # Make the directory to contain the configuration file
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write the file
+        logger().info('Updating configuration file: {}'.format(path))
+        # str() here is for py2 compatibility
+        with open(str(path), 'w') as f:
+            json.dump({k: str(self.values[k]) for k in self._keys if
+                       self.values[k] is not None}, f)
+
+    def find_dbprops(self, fname):
+        """Return the absolute path to a database properties file.
+
+        Searches for a file named *fname*, first in the current working
+        directory (`.`), then in the ixmp default location.
+
+        Parameters
+        ----------
+        fname : str
+            Name of a database properties file to locate.
+
+        Returns
+        -------
+        str
+            Absolute path to *fname*.
+
+        Raises
+        ------
+        FileNotFoundError
+            *fname* is not found in any of the search paths.
+        """
+        if fname is None:
+            # Use the default
+            return self.get('DEFAULT_DBPROPS_FILE')
+        else:
+            # Look in the current directory first, then the config directories
+            return self._locate(fname,
+                                dirs=[Path.cwd(), self.get('DB_CONFIG_PATH')])
+
+
+_config = Config()

ixmp/core.py

@@ -13,8 +13,7 @@
 
 import ixmp as ix
 from ixmp import model_settings
-from ixmp.default_path_constants import DEFAULT_LOCAL_DB_PATH
-from ixmp.default_paths import default_dbprops_file, find_dbprops
+from ixmp.config import _config
 from ixmp.utils import logger, islistable
 
 # %% default settings for column headers
@@ -98,14 +97,13 @@ def __init__(self, dbprops=None, dbtype=None, jvmargs=None):
         try:
             # if no dbtype is specified, launch Platform with properties file
             if dbtype is None:
-                dbprops = default_dbprops_file() if dbprops is None \
-                    else find_dbprops(dbprops)
-                logger().info("launching ixmp.Platform using config file at"
+                dbprops = _config.find_dbprops(dbprops)
+                logger().info("launching ixmp.Platform using config file at "
                               "'{}'".format(dbprops))
                 self._jobj = java.ixmp.Platform("Python", str(dbprops))
             # if dbtype is specified, launch Platform with local database
             elif dbtype == 'HSQLDB':
-                dbprops = dbprops or DEFAULT_LOCAL_DB_PATH
+                dbprops = dbprops or _config.get('DEFAULT_LOCAL_DB_PATH')
                 logger().info("launching ixmp.Platform with local {} database "
                               "at '{}'".format(dbtype, dbprops))
                 self._jobj = java.ixmp.Platform("Python", str(dbprops), dbtype)