Remove namespace packaging and hardcode elements (attempt 2)

Namespace packages are constant source of problems for users. The python
packaging ecosystem around splitting packages across namespaces is
fragile at the best of times and can often leave a you with an
environment that isn't recoverable (especially when mixing install
methods). There is also a performance hit whenever there is a piece of
the namespace we allow external packages to extend since it requires
doing a full python path search which can be slow depending on the
backing I/O and the number of paths in sys.path for an environment. This
commit starts the process of addressing this by removing the arbitrary
namespace hook points and hard coding the element namespace maps via a
custom import loader at the root of the namespace. This has 2 advantages
it removes the use of namespace packages so the fragility and
performance impact are fixed since every element will be renamed to use
'qiskit_' instead of 'qiskit.', but it also makes it explicit where we
extend the namespace. The previous method allowed any package to extend
qiskit.* and qiskit.providers.* with whatever they wanted.

We'll need to coordinate updating the elements with this merging,
because it is a breaking change for each element (although not for end
users).

A potential follow on is to add a plugin interface for 3rd party
providers like what was proposed in Qiskit#1465 so that we can make external
providers externally discoverable without needing to add manual hook
points moving forward (this was done for backwards compat with the aqt
and honeywell provider).

This is a second attempt at removing namespace packaging. The first
attempt in PR Qiskit#4767 was merged and had to be reverted because there were
some circular import error issues that needed to be resolved. Since
having this in terra blocks CI for all the qiskit elements a revert was
necessary to unblock developement for the entire project while those
were resolved.

qiskit/__init__.py

@@ -12,59 +12,91 @@
 
 # pylint: disable=wrong-import-order,invalid-name,wrong-import-position
 
-
 """Main Qiskit public functionality."""
 
-
-import pkgutil
 import sys
 import warnings
 import os
 
 # First, check for required Python and API version
 from . import util
 
+# Extend namespace for backwards compat
+from qiskit import namespace
+new_meta_path = []
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aer', 'qiskit.providers.aer'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_ignis', 'qiskit.ignis'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqua', 'qiskit.aqua'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqua.chemistry', 'qiskit.chemistry'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqua.finance', 'qiskit.finance'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqua.ml', 'qiskit.ml'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqua.optimization', 'qiskit.optimization'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_ibmq_provider', 'qiskit.providers.ibmq'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_aqt_provider', 'qiskit.providers.aqt'))
+new_meta_path.append(namespace.QiskitElementImport(
+    'qiskit_honeywell_provider', 'qiskit.providers.honeywell'))
+# Add Qiskit importers to meta_path before PathFinder in the default
+# sys.meta_path to avoid the miss penalty on trying to import a module
+# which does not exist
+old_meta_path = sys.meta_path
+sys.meta_path = old_meta_path[:-1] + new_meta_path + [old_meta_path[-1]]
+
 # qiskit errors operator
-from qiskit.exceptions import QiskitError
+from qiskit.exceptions import QiskitError  # noqa
 
 # The main qiskit operators
-from qiskit.circuit import ClassicalRegister
-from qiskit.circuit import QuantumRegister
-from qiskit.circuit import AncillaRegister
-from qiskit.circuit import QuantumCircuit
+from qiskit.circuit import ClassicalRegister  # noqa
+from qiskit.circuit import QuantumRegister  # noqa
+from qiskit.circuit import AncillaRegister  # noqa
+from qiskit.circuit import QuantumCircuit  # noqa
 
 # user config
-from qiskit import user_config as _user_config
+from qiskit import user_config as _user_config  # noqa
 
 # The qiskit.extensions.x imports needs to be placed here due to the
 # mechanism for adding gates dynamically.
-import qiskit.extensions
-import qiskit.circuit.measure
-import qiskit.circuit.reset
-
-# Allow extending this namespace. Please note that currently this line needs
-# to be placed *before* the wrapper imports or any non-import code AND *before*
-# importing the package you want to allow extensions for (in this case `backends`).
-__path__ = pkgutil.extend_path(__path__, __name__)
+import qiskit.extensions  # noqa
+import qiskit.circuit.measure  # noqa
+import qiskit.circuit.reset  # noqa
 
 # Please note these are global instances, not modules.
-from qiskit.providers.basicaer import BasicAer
+from qiskit.providers.basicaer import BasicAer  # noqa
 
 _config = _user_config.get_config()
 
+from qiskit.execute import execute  # noqa
+from qiskit.compiler import transpile, assemble, schedule  # noqa
+
+from .version import __version__  # noqa
+from .version import _get_qiskit_versions  # noqa
+
+__qiskit_version__ = _get_qiskit_versions()
+
 # Try to import the Aer provider if installed.
 try:
-    from qiskit.providers.aer import Aer
+    from qiskit.providers import aer
+    Aer = aer.Aer
 except ImportError:
     suppress_warnings = os.environ.get('QISKIT_SUPPRESS_PACKAGING_WARNINGS', '')
     if suppress_warnings.upper() != 'Y':
         if not _config.get('suppress_packaging_warnings') or suppress_warnings.upper() == 'N':
             warnings.warn('Could not import the Aer provider from the qiskit-aer '
                           'package. Install qiskit-aer or check your installation.',
                           RuntimeWarning)
+
 # Try to import the IBMQ provider if installed.
 try:
-    from qiskit.providers.ibmq import IBMQ
+    from qiskit.providers import ibmq
+    IBMQ = ibmq.IBMQ
 except ImportError:
     suppress_warnings = os.environ.get('QISKIT_SUPPRESS_PACKAGING_WARNINGS', '')
     if suppress_warnings.upper() != 'Y':
@@ -74,13 +106,3 @@
                           'qiskit-ibmq-provider or check your installation.',
                           RuntimeWarning)
 
-# Moved to after IBMQ and Aer imports due to import issues
-# with other modules that check for IBMQ (tools)
-from qiskit.execute import execute  # noqa
-from qiskit.compiler import transpile, assemble, schedule  # noqa
-
-from .version import __version__  # noqa
-from .version import _get_qiskit_versions  # noqa
-
-
-__qiskit_version__ = _get_qiskit_versions()

qiskit/namespace.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2020.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+# pylint: disable=unused-argument
+
+"""Module for utilities to manually construct qiskit namespace"""
+
+import sys
+from importlib.abc import MetaPathFinder, Loader
+import importlib
+
+
+class QiskitLoader(Loader):
+    """Load qiskit element as a namespace package."""
+    def __init__(self, new_package, old_namespace):
+        super().__init__()
+        self.new_package = new_package
+        self.old_namespace = old_namespace
+
+    def module_repr(self, module):
+        return repr(module)
+
+    def load_module(self, fullname):
+        old_name = fullname
+        names = fullname.split(".")
+        new_namespace_names = self.new_package.split('.')
+        old_namespace_names = self.old_namespace.split('.')
+        fullname = ".".join(
+            new_namespace_names + names[len(old_namespace_names):])
+        module = importlib.import_module(fullname)
+        sys.modules[old_name] = module
+        return module
+
+
+class QiskitElementImport(MetaPathFinder):
+    """Meta importer to enable unified qiskit namespace."""
+    def __init__(self, new_package, old_namespace):
+        super().__init__()
+        self.new_package = new_package
+        self.old_namespace = old_namespace
+
+    def find_spec(self, fullname, path=None, target=None):
+        """Return the ModuleSpec for Qiskit element."""
+        if fullname.startswith(self.old_namespace):
+            return importlib.util.spec_from_loader(
+                fullname,
+                QiskitLoader(self.new_package, self.old_namespace),
+                origin='qiskit')
+        return None

qiskit/providers/__init__.py

@@ -47,15 +47,9 @@
    JobTimeoutError
 """
 
-import pkgutil
-
 from .basebackend import BaseBackend
 from .baseprovider import BaseProvider
 from .basejob import BaseJob
 from .exceptions import (JobError, JobTimeoutError, QiskitBackendNotFoundError,
                          BackendPropertyError, BackendConfigurationError)
 from .jobstatus import JobStatus
-
-
-# Allow extending this namespace.
-__path__ = pkgutil.extend_path(__path__, __name__)

qiskit/tools/jupyter/job_watcher.py

@@ -176,5 +176,6 @@ def qiskit_disable_job_watcher(self, line='', cell=None):
         _JOB_WATCHER.stop_viewer()
 
 
-# The Jupyter job watcher instance
-_JOB_WATCHER = JobWatcher()
+if HAS_IBMQ:
+    # The Jupyter job watcher instance
+    _JOB_WATCHER = JobWatcher()

qiskit/version.py

@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-# pylint: disable=no-name-in-module,broad-except,cyclic-import
+# pylint: disable=no-name-in-module,broad-except,cyclic-import,import-error
 
 """Contains the terra version."""
 
@@ -85,22 +85,22 @@ def _get_qiskit_versions():
     out_dict = {}
     out_dict['qiskit-terra'] = __version__
     try:
-        from qiskit.providers import aer
+        import qiskit_aer as aer
         out_dict['qiskit-aer'] = aer.__version__
     except Exception:
         out_dict['qiskit-aer'] = None
     try:
-        from qiskit import ignis
+        import qiskit_ignis as ignis
         out_dict['qiskit-ignis'] = ignis.__version__
     except Exception:
         out_dict['qiskit-ignis'] = None
     try:
-        from qiskit.providers import ibmq
+        import qiskit_ibmq_provider as ibmq
         out_dict['qiskit-ibmq-provider'] = ibmq.__version__
     except Exception:
         out_dict['qiskit-ibmq-provider'] = None
     try:
-        from qiskit import aqua
+        import qiskit_aqua as aqua
         out_dict['qiskit-aqua'] = aqua.__version__
     except Exception:
         out_dict['qiskit-aqua'] = None