From b76ef44ba899ae8c84b4197e99c58bacd4443c11 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 10:24:35 +0100 Subject: [PATCH 01/12] Add a `list_platforms()` function --- pyam/iiasa.py | 23 +++++++++++++++++++++-- pyam/str.py | 10 ++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index e422e9a9e..fea2c977c 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -12,13 +12,14 @@ import pandas as pd import requests import yaml +from ixmp4.cli import utils from ixmp4.conf import settings from ixmp4.conf.auth import ManagerAuth from requests.auth import AuthBase from pyam.core import IamDataFrame from pyam.logging import deprecation_warning -from pyam.str import is_str +from pyam.str import is_str, shorten from pyam.utils import ( IAMC_IDX, META_IDX, @@ -35,13 +36,31 @@ You are connected to the {} scenario explorer hosted by IIASA. If you use this data in any published format, please cite the data as provided in the explorer guidelines: {} -""".replace("\n", "") +""".replace( + "\n", "" +) IXMP4_LOGIN = "Please run `ixmp4 login ` in a console" # path to local configuration settings DEFAULT_IIASA_CREDS = Path("~").expanduser() / ".local" / "pyam" / "iiasa.yaml" +def list_platforms(): + + platforms = ixmp4.conf.settings.manager.list_platforms() + utils.echo("IIASA platform".ljust(20) + "Access".ljust(10) + "Notice\n") + + for p in platforms: + utils.important(shorten(p.name, 20), nl=False) + utils.echo(str(p.accessibility.value.lower()).ljust(10), nl=False) + + if p.notice is not None: + utils.echo(shorten(p.notice, 55), nl=False) + utils.echo() + + utils.info("\n" + str(len(platforms)) + " total \n") + + def set_config(user, password, file=None): raise DeprecationWarning(f"This method is deprecated. {IXMP4_LOGIN}.") diff --git a/pyam/str.py b/pyam/str.py index 286ac38d2..321ed0702 100644 --- a/pyam/str.py +++ b/pyam/str.py @@ -65,18 +65,21 @@ def _count_pipes(val): # test = lambda x: level == x if x is not None else False def test(x): return level == x if x is not None else False + elif level[-1] == "-": level = int(level[:-1]) # test = lambda x: level >= x if x is not None else False def test(x): return level >= x if x is not None else False + elif level[-1] == "+": level = int(level[:-1]) # test = lambda x: level <= x if x is not None else False def test(x): return level <= x if x is not None else False + else: raise ValueError("Unknown level type: `{}`".format(level)) @@ -141,3 +144,10 @@ def escape_regexp(s): def is_str(x): """Returns True if x is a string""" return isinstance(x, six.string_types) + + +def shorten(value: str, length: int): + """Shorten a string to a given length adding `...`""" + if len(value) > length - 4: + value = value[: length - 4] + "..." + return value.ljust(length) From cc34c4a6164fb613cc0f8c6dad173292d30b7c13 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 10:26:43 +0100 Subject: [PATCH 02/12] Add a warning to legacy `valid_connections` --- pyam/iiasa.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index fea2c977c..8542ec957 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -216,6 +216,10 @@ def _connection_map(self): @lru_cache() def valid_connections(self): """Return available resources (database API connections)""" + logger.warning( + "IIASA is migrating to a new database infrastructure based on the ixmp4 package." + "Please use `pyam.iiasa.list_platforms()` to list available ixmp4 databases." + ) return list(self._connection_map.keys()) def connect(self, name): From 6a61546298d33fdf294824a980fac9b146d383e9 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 10:40:40 +0100 Subject: [PATCH 03/12] Add `pyam.iiasa.list_platforms()` to docs --- docs/api/iiasa.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/api/iiasa.rst b/docs/api/iiasa.rst index 77ea0ca2a..5533775db 100644 --- a/docs/api/iiasa.rst +++ b/docs/api/iiasa.rst @@ -38,10 +38,15 @@ You will be prompted to enter your password. *Scenario Apps* instances ------------------------- -Coming soon... +The *Scenario Apps* use the |ixmp4| package as a database backend. +You can list all available ixmp4 platforms hosted by IIASA using the following: -*Scenario Explorer* instances ------------------------------ +.. autofunction:: list_platforms + :noindex: + + +*Scenario Explorer* instances (legacy service) +---------------------------------------------- The *Scenario Explorer* infrastructure developed by the Scenario Services and Scientific Software team was developed and used for projects from 2018 until 2023. From 752b359f6e2e4cc5c4e192d49a1a40e286206218 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 12:58:00 +0100 Subject: [PATCH 04/12] Add to release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9b3bde499..9b2207f69 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,6 @@ # Next release +- [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.list_platforms()` function - [#826](https://github.com/IAMconsortium/pyam/pull/826) Add `read_ixmp4()` function and extend integration test - [#825](https://github.com/IAMconsortium/pyam/pull/825) Add support for Python 3.12 - [#824](https://github.com/IAMconsortium/pyam/pull/824) Update ixmp4 requirement to >=0.7.1 From 263c41f1130287fe2bba51275bf6cb4ff9e1390c Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 13:22:18 +0100 Subject: [PATCH 05/12] Make ruff --- pyam/iiasa.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 8542ec957..b89cf8f90 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -46,6 +46,7 @@ def list_platforms(): + """Print all available ixmp4 platforms hosted by IIASA""" platforms = ixmp4.conf.settings.manager.list_platforms() utils.echo("IIASA platform".ljust(20) + "Access".ljust(10) + "Notice\n") @@ -217,8 +218,8 @@ def _connection_map(self): def valid_connections(self): """Return available resources (database API connections)""" logger.warning( - "IIASA is migrating to a new database infrastructure based on the ixmp4 package." - "Please use `pyam.iiasa.list_platforms()` to list available ixmp4 databases." + "IIASA is migrating to a database infrastructure using the ixmp4 package." + "Use `pyam.iiasa.list_platforms()` to list available ixmp4 databases." ) return list(self._connection_map.keys()) From d6af89bfb485c9450e74bd77867eb7268619eced Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 15:23:21 +0100 Subject: [PATCH 06/12] Update the iiasa-tutorial --- docs/api/iiasa.rst | 1 - docs/tutorials/iiasa.ipynb | 93 ++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/docs/api/iiasa.rst b/docs/api/iiasa.rst index 5533775db..3cc550e98 100644 --- a/docs/api/iiasa.rst +++ b/docs/api/iiasa.rst @@ -44,7 +44,6 @@ You can list all available ixmp4 platforms hosted by IIASA using the following: .. autofunction:: list_platforms :noindex: - *Scenario Explorer* instances (legacy service) ---------------------------------------------- diff --git a/docs/tutorials/iiasa.ipynb b/docs/tutorials/iiasa.ipynb index 4a87af9db..61aeb3e17 100644 --- a/docs/tutorials/iiasa.ipynb +++ b/docs/tutorials/iiasa.ipynb @@ -4,23 +4,40 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Read directly from IIASA data resources\n", + "# Query data from the IIASA database infrastructure\n", "\n", - "The IIASA *Energy, Climate, and Environment* Program hosts a suite of **Scenario Explorer** instances and related infrastructure to support analysis of integrated-assessment pathways in IPCC reports and model comparison projects. \n", - "High-profile use cases include the [IAMC 1.5°C Scenario Explorer hosted by IIASA](https://data.ece.iiasa.ac.at/iamc-1.5c-explorer) supporting the *IPCC Special Report on Global Warming of 1.5°C* (SR15) and the Horizon 2020 project [CD-LINKS](https://data.ece.iiasa.ac.at/cd-links).\n", + "The IIASA *Energy, Climate, and Environment* Program hosts a suite of **Scenario Explorer** instances and related database infrastructure to support analysis of integrated-assessment pathways in IPCC reports and model comparison projects. \n", + "High-profile use cases include the [AR6 Scenario Explorer hosted by IIASA](https://data.ece.iiasa.ac.at/ar6) supporting the *IPCC' Sixth Assessment Report* (AR6) and the Horizon 2020 project [ENGAGE](https://data.ece.iiasa.ac.at/engage).\n", "\n", - "IIASA's [modeling platform infrastructure](http://software.ene.iiasa.ac.at/ixmp-server) and the Scenario Explorer UI is not only a great resource on its own, but it also allows the underlying datasets to be directly queried.\n", - "**pyam** takes advantage of this ability to allow you to easily pull data and work with it in your Python data processing and analysis workflow." + "IIASA's [modeling platform infrastructure](http://docs.ece.iiasa.ac.at/) and the scenario apps are not only a great resource on its own, but it also allows the underlying datasets to be queried directly via a Rest API.\n", + "The **pyam** package takes advantage of this ability to allow you to easily pull data and work with it in your Python data processing and analysis workflow." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Connecting to a data resource (aka the database API of a Scenario Explorer instance)\n", + "## Access and permission management for project-internal databases\n", "\n", - "Accessing a data resource is done via a **Connection** object.\n", - "By default, your can connect to all public Scenario Explorer instances. " + "By default, your can connect to all public scenario database instances.\n", + "\n", + "If you have credentials to connect to a private, project-internal database instance, \n", + "you can store this information by running the following command in a console:\n", + "\n", + "```\n", + "ixmp4 login \n", + "\n", + "```\n", + "\n", + "You will be prompted to enter your password.\n", + "\n", + "
\n", + "\n", + "Your username and password will be saved locally in plain-text for future use!\n", + "\n", + "
\n", + "\n", + "When connecting to a database, **pyam** will automatically search for the configuration in a known location." ] }, { @@ -29,41 +46,69 @@ "metadata": {}, "outputs": [], "source": [ - "import pyam\n", - "\n", - "conn = pyam.iiasa.Connection()\n", - "conn.valid_connections" + "import pyam" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "If you have credentials to connect to a non-public or restricted Scenario Explorer instance,\n", - "you can store this information by running the following command in a console:\n", + "## Connecting to ixmp4 database instances hosted by IIASA\n", "\n", - "```\n", - "ixmp4 login \n", - "\n", - "```\n", - "\n", - "You will be prompted to enter your password.\n", + "The *Scenario Apps* use the **ixmp4** package as a database backend.\n", "\n", "
\n", "\n", - "Your username and password will be saved locally in plain-text for future use!\n", + "The *Scenario Services* team is currently migrating database instances from the legacy *Scenario Explorer* infrastructure \n", + "to the new *Scenario Apps* services based on the **ixmp4** package.\n", "\n", "
\n", "\n", + "You can list all available ixmp4 platforms hosted by IIASA using the following function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pyam.iiasa.list_platforms()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connecting to a (legacy) Scenario Explorer database\n", "\n", - "\n", - "When initializing a new **Connection** instance, **pyam** will automatically search for the configuration in a known location." + "Accessing the database connected to a **Scenario Explorer** is done via a **Connection** object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "conn = pyam.iiasa.Connection()\n", + "conn.valid_connections" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "## Reading data from a database hosted by IIASA" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The **pyam** package can read both from **ixmp4** databases (connected to a *Scenario App*) \n", + "and the (legacy) **Scenario Explorer** infrastructure with the same function.\n", + "\n", "In this example, we will be retrieving data from the *IAMC 1.5°C Scenario Explorer hosted by IIASA*\n", "([link](https://data.ece.iiasa.ac.at/iamc-1.5c-explorer)),\n", "which provides the quantitative scenario ensemble underpinning\n", @@ -316,7 +361,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.11.0" } }, "nbformat": 4, From 0485091dfc1fd8951a2b5e5f2737c36d4cd03a5e Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 18:07:23 +0100 Subject: [PATCH 07/12] Rename to avoid name-conflict with ixmp4 --- docs/api/iiasa.rst | 3 +-- docs/tutorials/iiasa.ipynb | 2 +- pyam/iiasa.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/api/iiasa.rst b/docs/api/iiasa.rst index 3cc550e98..1c63692ff 100644 --- a/docs/api/iiasa.rst +++ b/docs/api/iiasa.rst @@ -41,8 +41,7 @@ You will be prompted to enter your password. The *Scenario Apps* use the |ixmp4| package as a database backend. You can list all available ixmp4 platforms hosted by IIASA using the following: -.. autofunction:: list_platforms - :noindex: +.. autofunctions:: platforms *Scenario Explorer* instances (legacy service) ---------------------------------------------- diff --git a/docs/tutorials/iiasa.ipynb b/docs/tutorials/iiasa.ipynb index 61aeb3e17..273822da4 100644 --- a/docs/tutorials/iiasa.ipynb +++ b/docs/tutorials/iiasa.ipynb @@ -73,7 +73,7 @@ "metadata": {}, "outputs": [], "source": [ - "pyam.iiasa.list_platforms()" + "pyam.iiasa.platforms()" ] }, { diff --git a/pyam/iiasa.py b/pyam/iiasa.py index b89cf8f90..a670fa865 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -45,13 +45,13 @@ DEFAULT_IIASA_CREDS = Path("~").expanduser() / ".local" / "pyam" / "iiasa.yaml" -def list_platforms(): +def platforms() -> None: """Print all available ixmp4 platforms hosted by IIASA""" - platforms = ixmp4.conf.settings.manager.list_platforms() + _platforms = ixmp4.conf.settings.manager.list_platforms() utils.echo("IIASA platform".ljust(20) + "Access".ljust(10) + "Notice\n") - for p in platforms: + for p in _platforms: utils.important(shorten(p.name, 20), nl=False) utils.echo(str(p.accessibility.value.lower()).ljust(10), nl=False) @@ -59,7 +59,7 @@ def list_platforms(): utils.echo(shorten(p.notice, 55), nl=False) utils.echo() - utils.info("\n" + str(len(platforms)) + " total \n") + utils.info("\n" + str(len(_platforms)) + " total \n") def set_config(user, password, file=None): From bc48b7a493a4d4b860facef2966ef5e301d2620f Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 18:07:35 +0100 Subject: [PATCH 08/12] Add a smoke test --- tests/test_iiasa.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 9be63c0ef..943521d20 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -55,6 +55,11 @@ ) +def test_platforms(): + # test that the function does not raise an error + iiasa.platforms() + + def test_unknown_conn(): # connecting to an unknown API raises an error match = "You do not have access to instance 'foo' or it does not exist." From 59fbb455215d7b182f547e72e5fd1163339921a1 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 18:07:45 +0100 Subject: [PATCH 09/12] Add ixmp4 to conf.py --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index c020d8711..d414b1b65 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -330,6 +330,7 @@ "pandas_datareader": ("https://pandas-datareader.readthedocs.io/en/stable", None), "unfccc_di_api": ("https://unfccc-di-api.readthedocs.io/en/stable", None), "nomenclature": ("https://nomenclature-iamc.readthedocs.io/en/stable", None), + "ixmp4": ("https://docs.ece.iiasa.ac.at/projects/ixmp4/en/stable", None), } # Set up the plotting gallery with plotly scraper From 6e538226829f79d015308c25652d9f46e16d217a Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 18:10:30 +0100 Subject: [PATCH 10/12] Add see-also pointing to (non-existing) ixmp4 docs --- pyam/iiasa.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyam/iiasa.py b/pyam/iiasa.py index a670fa865..9a41c94c8 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -46,7 +46,12 @@ def platforms() -> None: - """Print all available ixmp4 platforms hosted by IIASA""" + """Print all available ixmp4 platforms hosted by IIASA + + See Also + -------- + ixmp4.conf.settings.manager.list_platforms + """ _platforms = ixmp4.conf.settings.manager.list_platforms() utils.echo("IIASA platform".ljust(20) + "Access".ljust(10) + "Notice\n") From b1d66a3616c1f8a8d69fc537a46cf2b9fd242172 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 18:14:24 +0100 Subject: [PATCH 11/12] Add an assertion on the print-statement for @phackstock --- tests/test_iiasa.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py index 943521d20..9e72dd9b8 100644 --- a/tests/test_iiasa.py +++ b/tests/test_iiasa.py @@ -55,9 +55,13 @@ ) -def test_platforms(): +def test_platforms(capsys): # test that the function does not raise an error iiasa.platforms() + assert ( + "public-test public This is a public ixmp4 test instance" + in capsys.readouterr().out + ) def test_unknown_conn(): From 5d6509599e424619eefb0062949d618a09e6a783 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Mon, 4 Mar 2024 19:47:19 +0100 Subject: [PATCH 12/12] Fix changed function name --- RELEASE_NOTES.md | 2 +- pyam/iiasa.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9b2207f69..fe80a0d64 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # Next release -- [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.list_platforms()` function +- [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.platforms()` function for a list of available platforms - [#826](https://github.com/IAMconsortium/pyam/pull/826) Add `read_ixmp4()` function and extend integration test - [#825](https://github.com/IAMconsortium/pyam/pull/825) Add support for Python 3.12 - [#824](https://github.com/IAMconsortium/pyam/pull/824) Update ixmp4 requirement to >=0.7.1 diff --git a/pyam/iiasa.py b/pyam/iiasa.py index 9a41c94c8..39de83851 100644 --- a/pyam/iiasa.py +++ b/pyam/iiasa.py @@ -46,7 +46,7 @@ def platforms() -> None: - """Print all available ixmp4 platforms hosted by IIASA + """Print a list of available ixmp4 platforms hosted by IIASA See Also -------- @@ -224,7 +224,7 @@ def valid_connections(self): """Return available resources (database API connections)""" logger.warning( "IIASA is migrating to a database infrastructure using the ixmp4 package." - "Use `pyam.iiasa.list_platforms()` to list available ixmp4 databases." + "Use `pyam.iiasa.platforms()` to list available ixmp4 databases." ) return list(self._connection_map.keys())