From ffa174f9de17d0dfa7416141e6514ee8102753af Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 11:19:32 +0100 Subject: [PATCH 01/18] new workflow test --- .github/workflows/{pr.yml => ci.yml} | 95 ++++++++++++++-------------- .github/workflows/main.yml | 92 --------------------------- 2 files changed, 48 insertions(+), 139 deletions(-) rename .github/workflows/{pr.yml => ci.yml} (71%) delete mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/pr.yml b/.github/workflows/ci.yml similarity index 71% rename from .github/workflows/pr.yml rename to .github/workflows/ci.yml index bf78a9b69..009ffeb19 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/ci.yml @@ -10,66 +10,35 @@ # copyright notice, and modified files need to carry a notice indicating # that they have been altered from the originals. -name: PR-Test -on: pull_request +name: CI +on: + [ push, pull_request ] jobs: - tests: - name: pr-tests-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["macOS-latest", "ubuntu-latest", "windows-latest"] - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_TESTS: skip_online - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make coverage - - name: Upload test coverage report - uses: actions/upload-artifact@v2 - with: - name: coverage_report-${{ matrix.python-version }}-${{ matrix.os }} - path: htmlcov - lint: - name: lint & mypy + code-quality: + name: Run code quality checks runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install Deps + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -c constraints.txt -e . pip install -U -c constraints.txt -r requirements-dev.txt + - name: Run black + run: make style - name: Run lint - run: make style && make lint + run: make lint - name: Run mypy run: make mypy if: ${{ !cancelled() }} - doc: - name: doc + documentation: + name: Generate documentation runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -79,16 +48,48 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install Deps + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -U tox sudo apt install -y graphviz pip install -c constraints.txt -e . - - name: Build docs + - name: Build documentation run: tox -edocs - - name: Upload docs + - name: Upload documentation uses: actions/upload-artifact@v2 with: name: html_docs path: docs/_build/html + unit-tests: + # only kick-off test cases when basic code quality checks succeed + needs: [ "code-quality" , "documentation" ] + name: Run unit tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [ 3.7, 3.8, 3.9 ] + os: [ "macOS-latest", "ubuntu-latest", "windows-latest" ] + env: + LOG_LEVEL: DEBUG + STREAM_LOG: True + QISKIT_TESTS: skip_online + QISKIT_IN_PARALLEL: True + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -c constraints.txt -e . + pip install -U -c constraints.txt -r requirements-dev.txt + - name: Run unit tests + run: make coverage + - name: Upload unit test coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage_report-${{ matrix.python-version }}-${{ matrix.os }} + path: htmlcov \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 1aeceeacd..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,92 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -name: Push-Test -on: push -jobs: - lint: - name: lint & mypy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run lint - run: make style && make lint - - name: Run mypy - run: make mypy - if: ${{ !cancelled() }} - doc: - name: doc - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -U tox - sudo apt install -y graphviz - pip install -c constraints.txt -e . - - name: Build docs - run: tox -edocs - - name: Upload docs - uses: actions/upload-artifact@v2 - with: - name: html_docs - path: docs/_build/html - tests-mac: - name: push-tests-mac-python${{ matrix.python-version }} - runs-on: macOS-latest - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make coverage - - name: Upload test coverage report - uses: actions/upload-artifact@v2 - with: - name: coverage_report-${{ matrix.python-version }} - path: htmlcov From a9286aed7c67c99719c5f2eb40b2bcbd2477aee7 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 11:43:53 +0100 Subject: [PATCH 02/18] new workflow test --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009ffeb19..6cd40fc6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ on: [ push, pull_request ] jobs: code-quality: + if: ${{ 'skip' == 'me' }} name: Run code quality checks runs-on: ubuntu-latest steps: @@ -38,7 +39,8 @@ jobs: run: make mypy if: ${{ !cancelled() }} documentation: - name: Generate documentation + if: ${{ 'skip' == 'me' }} + name: Build documentation runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -62,6 +64,7 @@ jobs: name: html_docs path: docs/_build/html unit-tests: + if: ${{ 'skip' == 'me' }} # only kick-off test cases when basic code quality checks succeed needs: [ "code-quality" , "documentation" ] name: Run unit tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) @@ -77,6 +80,8 @@ jobs: QISKIT_IN_PARALLEL: True steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -91,5 +96,41 @@ jobs: - name: Upload unit test coverage report uses: actions/upload-artifact@v2 with: - name: coverage_report-${{ matrix.python-version }}-${{ matrix.os }} + name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + path: htmlcov + integration-tests: + name: Run integration tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [ 3.9 ] + os: [ "ubuntu-latest" ] + environment: [ "legacy-production" ] + environment: ${{ matrix.environment }} + env: + QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }} + QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }} + QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }} + LOG_LEVEL: DEBUG + STREAM_LOG: True + QISKIT_IN_PARALLEL: True + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -c constraints.txt -e . + pip install -U -c constraints.txt -r requirements-dev.txt + - name: Run integration tests + run: make coverage + - name: Upload integration test coverage report + uses: actions/upload-artifact@v2 + with: + name: Integration test coverage report (Python ${{ matrix.environment }}, ${{ matrix.os }}) path: htmlcov \ No newline at end of file From 10d5a97cd2cbe77e786657f2b5691f2fcca44932 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 11:46:07 +0100 Subject: [PATCH 03/18] new workflow test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cd40fc6a..396cbd65b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) path: htmlcov integration-tests: - name: Run integration tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + name: Run integration tests against ${{ matrix.environment }} (Python ${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: matrix: From 3bb407d418b545d0d2e5dcedbd4a7d37a224babe Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 11:53:58 +0100 Subject: [PATCH 04/18] new workflow test --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 396cbd65b..4b1722426 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,13 +99,13 @@ jobs: name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) path: htmlcov integration-tests: - name: Run integration tests against ${{ matrix.environment }} (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + name: Run integration tests (${{ matrix.environment }}) runs-on: ${{ matrix.os }} strategy: matrix: python-version: [ 3.9 ] os: [ "ubuntu-latest" ] - environment: [ "legacy-production" ] + environment: [ "legacy-production", "cloud-production" ] environment: ${{ matrix.environment }} env: QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }} @@ -132,5 +132,5 @@ jobs: - name: Upload integration test coverage report uses: actions/upload-artifact@v2 with: - name: Integration test coverage report (Python ${{ matrix.environment }}, ${{ matrix.os }}) + name: Integration test coverage report (${{ matrix.environment }}) path: htmlcov \ No newline at end of file From 0d0b9f460247ae41aac59714256a77d7210f4219 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 16:40:40 +0100 Subject: [PATCH 05/18] prepare decorators for running tests separately against legacy/cloud --- test/utils/decorators.py | 330 +++++++++++---------------------------- 1 file changed, 87 insertions(+), 243 deletions(-) diff --git a/test/utils/decorators.py b/test/utils/decorators.py index 6cbbe7998..891ed2bed 100644 --- a/test/utils/decorators.py +++ b/test/utils/decorators.py @@ -10,289 +10,133 @@ # copyright notice, and modified files need to carry a notice indicating # that they have been altered from the originals. -"""Decorators for using with IBM Provider unit tests. - - Environment variables used by the decorators: - * QISKIT_IBM_API_TOKEN: default API token to use. - * QISKIT_IBM_API_URL: default API url to use. - * QISKIT_IBM_HGP: default hub/group/project to use. - * QISKIT_IBM_PRIVATE_HGP: hub/group/project to use for private jobs. - * QISKIT_IBM_DEVICE: default device to use. - * QISKIT_IBM_USE_STAGING_CREDENTIALS: True if use staging credentials. - * QISKIT_IBM_STAGING_API_TOKEN: staging API token to use. - * QISKIT_IBM_STAGING_API_URL: staging API url to use. - * QISKIT_IBM_STAGING_HGP: staging hub/group/project to use. - * QISKIT_IBM_STAGING_DEVICE: staging device to use. - * QISKIT_IBM_STAGING_PRIVATE_HGP: staging hub/group/project to use for private jobs. -""" +"""Decorators used by unit tests.""" import os +from dataclasses import dataclass from functools import wraps +from typing import Optional, List, Any from unittest import SkipTest -from typing import Optional, List, Union from qiskit.test.testing_options import get_test_options -from qiskit_ibm_runtime import IBMRuntimeService +from qiskit_ibm_runtime import IBMRuntimeService from ..mock.fake_runtime_service import FakeRuntimeService -def requires_online_access(func): - """Decorator that signals whether online access is needed.""" - - @wraps(func) - def _wrapper(*args, **kwargs): - if get_test_options()["skip_online"]: - raise SkipTest("Skipping online tests") - return func(*args, **kwargs) - - return _wrapper - - -def requires_qe_access(func): - """Test requires legacy access.""" - - @wraps(func) - def _wrapper(obj, *args, **kwargs): - token, url, _ = _get_token_url_instance("legacy") - kwargs.update({"qe_token": token, "qe_url": url}) - return func(obj, *args, **kwargs) - - return _wrapper - - -def requires_multiple_hgps(func): - """Test requires a public and premium hgp.""" +def run_legacy_and_cloud_fake(func): + """Decorator that runs a test using both legacy and cloud fake services.""" @wraps(func) - def _wrapper(*args, **kwargs): - service = _get_service("legacy") - hgps = list(service._hgps.keys()) - if len(hgps) < 2: - raise SkipTest("Test require at least 2 hub/group/project.") - - # Get open access hgp - open_hgp = hgps[-1] - premium_hgp = hgps[0] - kwargs.update( - { - "service": service, - "open_hgp": open_hgp, - "premium_hgp": premium_hgp, - } + def _wrapper(self, *args, **kwargs): + legacy_service = FakeRuntimeService( + auth="legacy", token="my_token", instance="h/g/p" ) - return func(*args, **kwargs) - - return _wrapper - - -def requires_legacy_service(func): - """Test requires legacy online API.""" - - @wraps(func) - def _wrapper(*args, **kwargs): - token, url, instance = _get_token_url_instance("legacy") - service = IBMRuntimeService( - auth="legacy", token=token, url=url, instance=instance + cloud_service = FakeRuntimeService( + auth="cloud", token="my_token", instance="crn:123" ) - kwargs.update({"service": service, "instance": instance}) - return func(*args, **kwargs) + for service in [legacy_service, cloud_service]: + with self.subTest(service=service.auth): + kwargs["service"] = service + func(self, *args, **kwargs) return _wrapper -def requires_cloud_service(func): - """Test requires cloud online API.""" - - @wraps(func) - def _wrapper(*args, **kwargs): - token, url, instance = _get_token_url_instance("cloud") - service = IBMRuntimeService( - auth="cloud", token=token, url=url, instance=instance - ) - kwargs.update({"service": service, "instance": instance}) - return func(*args, **kwargs) +def _get_integration_test_config(): + token, url, instance = ( + os.getenv("QISKIT_IBM_TOKEN"), + os.getenv("QISKIT_IBM_URL"), + os.getenv("QISKIT_IBM_INSTANCE"), + ) + auth: Any = "legacy" if url.find("quantum-computing.ibm.com") >= 0 else "cloud" + return auth, token, url, instance - return _wrapper +def run_integration_test(func): + """Decorator that injects preinitialized service and device parameters. -def requires_cloud_legacy_services(func): - """Test requires cloud online API.""" + To be used in combinatino with the integration_test_setup decorator function.""" @wraps(func) - def _wrapper(*args, **kwargs): - cloud_token, cloud_url, cloud_instance = _get_token_url_instance("cloud") - cloud_service = IBMRuntimeService( - auth="cloud", token=cloud_token, url=cloud_url, instance=cloud_instance - ) - legacy_token, legacy_url, legacy_instance = _get_token_url_instance("legacy") - legacy_service = IBMRuntimeService( - auth="legacy", token=legacy_token, url=legacy_url, instance=legacy_instance - ) - - kwargs.update({"services": [cloud_service, legacy_service]}) - return func(*args, **kwargs) + def _wrapper(self, *args, **kwargs): + with self.subTest(service=self.dependencies.service): + if self.dependencies.service: + kwargs["service"] = self.dependencies.service + if self.dependencies.device: + kwargs["device"] = self.dependencies.device + func(self, *args, **kwargs) return _wrapper -def requires_provider(func): - """Decorator that signals the test uses the online API, via a custom hub/group/project. - - This decorator delegates into the `requires_qe_access` decorator, but - instead of the credentials it appends a `provider` argument to the decorated - function. It also appends the custom `hub`, `group` and `project` arguments. +def integration_test_setup( + supported_auth: Optional[List[str]] = None, + resolve_least_busy_device: Optional[bool] = False, +): + """Returns a decorator for integration test initialization. Args: - func (callable): test function to be decorated. + supported_auth: a list of supported auth types that this test supports + resolve_least_busy_device: to resolve the least busy device and return it via the + test dependencies Returns: - callable: the decorated function. + A decorator that handles initialization of integration test dependencies. """ + if supported_auth is None: + supported_auth = ["cloud", "legacy"] - @wraps(func) - @requires_qe_access - def _wrapper(*args, **kwargs): - token = kwargs.pop("qe_token") - url = kwargs.pop("qe_url") - service = IBMRuntimeService(auth="legacy", token=token, url=url) - hub, group, project = _get_custom_hgp() - kwargs.update( - {"service": service, "hub": hub, "group": group, "project": project} - ) - return func(*args, **kwargs) + if get_test_options()["skip_online"]: + raise SkipTest("Skipping integration test.") - return _wrapper - - -def requires_cloud_legacy_devices(func): - """Test requires both cloud and legacy devices.""" - - @wraps(func) - def _wrapper(obj, *args, **kwargs): - - devices = [] - token, url, instance = _get_token_url_instance("cloud") - service = IBMRuntimeService( - auth="cloud", token=token, url=url, instance=instance - ) - # TODO use real device when cloud supports it - devices.append(service.least_busy(min_num_qubits=5)) + auth, token, url, instance = _get_integration_test_config() + if not all([auth, token, url]): + raise Exception("Configuration Issue") - token, url, instance = _get_token_url_instance("legacy") - service = IBMRuntimeService( - auth="legacy", token=token, url=url, instance=instance + if auth not in supported_auth: + raise SkipTest( + f"Skipping integration test. Test does not support auth type {auth}" ) - devices.append( - service.least_busy(simulator=False, min_num_qubits=5, instance=instance) - ) - - kwargs.update({"devices": devices}) - return func(obj, *args, **kwargs) - - return _wrapper - -@requires_online_access -def _get_token_url_instance(auth): - # TODO: Change this once we start using different environments - if auth == "cloud": - if os.getenv("QISKIT_IBM_USE_STAGING_CREDENTIALS", ""): - return ( - os.getenv("QISKIT_IBM_STAGING_CLOUD_TOKEN"), - os.getenv("QISKIT_IBM_STAGING_CLOUD_URL"), - os.getenv("QISKIT_IBM_STAGING_CLOUD_CRN"), + def _decorator(func): + @wraps(func) + def _wrapper(self, *args, **kwargs): + service = IBMRuntimeService( + auth=auth, token=token, url=url, instance=instance ) - return ( - os.getenv("QISKIT_IBM_CLOUD_TOKEN"), - os.getenv("QISKIT_IBM_CLOUD_URL"), - os.getenv("QISKIT_IBM_CLOUD_CRN"), - ) - - if os.getenv("QISKIT_IBM_USE_STAGING_CREDENTIALS", ""): - # Special case: instead of using the standard credentials mechanism, - # load them from different environment variables. This assumes they - # will always be in place, as is used by the CI setup. - return ( - os.getenv("QISKIT_IBM_STAGING_API_TOKEN"), - os.getenv("QISKIT_IBM_STAGING_API_URL"), - os.getenv("QISKIT_IBM_STAGING_HGP"), - ) - - return ( - os.getenv("QISKIT_IBM_API_TOKEN"), - os.getenv("QISKIT_IBM_API_URL"), - os.getenv("QISKIT_IBM_HGP"), - ) - - -def _get_service(auth: str) -> Union[List, IBMRuntimeService]: - """Return service(s). - - Args: - auth: Service type, ``cloud``, ``legacy``, or ``both``. - - Returns: - Runtime service(s) - """ - if auth in ["cloud", "legacy"]: - token, url, instance = _get_token_url_instance(auth) - return IBMRuntimeService(auth=auth, token=token, url=url, instance=instance) - - services = [] - for auth_ in ["cloud", "legacy"]: - token, url, instance = _get_token_url_instance(auth_) - services.append( - IBMRuntimeService(auth=auth_, token=token, url=url, instance=instance) - ) - return services - - -def _get_custom_hgp() -> Optional[str]: - """Get a custom hub/group/project - - Gets the hub/group/project set in QISKIT_IBM_STAGING_HGP for staging env or - QISKIT_IBM_HGP for production env. - - Returns: - Custom hub/group/project or ``None`` if not set. - """ - hgp = ( - os.getenv("QISKIT_IBM_STAGING_HGP", None) - if os.getenv("QISKIT_IBM_USE_STAGING_CREDENTIALS", "") - else os.getenv("QISKIT_IBM_HGP", None) - ) - return hgp - - -def run_legacy_and_cloud_fake(func): - """Decorator that runs a test using both legacy and cloud fake services.""" - - @wraps(func) - def _wrapper(self, *args, **kwargs): - legacy_service = FakeRuntimeService( - auth="legacy", token="my_token", instance="h/g/p" - ) - cloud_service = FakeRuntimeService( - auth="cloud", token="my_token", instance="crn:123" - ) - for service in [legacy_service, cloud_service]: - with self.subTest(service=service.auth): - kwargs["service"] = service - func(self, *args, **kwargs) - - return _wrapper + device = None + if resolve_least_busy_device: + if auth == "cloud": + # TODO use real device when cloud supports it + device = service.least_busy(min_num_qubits=5) + if auth == "legacy": + device = service.least_busy( + simulator=False, min_num_qubits=5, instance=instance + ) + + dependencies = IntegrationTestDependencies( + service=service, + auth=auth, + token=token, + url=url, + instance=instance, + device=device, + ) + kwargs["dependencies"] = dependencies + func(self, *args, **kwargs) + return _wrapper -def run_cloud_legacy_real(func): - """Decorator that runs a test using both legacy and cloud real services.""" + return _decorator - @wraps(func) - def _wrapper(self, *args, **kwargs): - for service in self.services: - with self.subTest(service=service.auth): - kwargs["service"] = service - func(self, *args, **kwargs) - return _wrapper +@dataclass +class IntegrationTestDependencies: + service: IBMRuntimeService + instance: Optional[str] + token: str + auth: str + url: str + device: Any From 9f8539181fbc2738c83fa7076f2118804ce87a46 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 17:07:10 +0100 Subject: [PATCH 06/18] [ci skip] use new decorators (part 1) --- test/test_account_client.py | 46 ++++++------- test/test_proxies.py | 125 +++++++++++++++++++++--------------- test/utils/decorators.py | 15 +++-- 3 files changed, 106 insertions(+), 80 deletions(-) diff --git a/test/test_account_client.py b/test/test_account_client.py index 97399c771..79324f28c 100644 --- a/test/test_account_client.py +++ b/test/test_account_client.py @@ -21,7 +21,7 @@ from .ibm_test_case import IBMTestCase from .mock.http_server import SimpleServer, ClientErrorHandler from .utils.account import custom_envs, no_envs -from .utils.decorators import requires_qe_access +from .utils.decorators import integration_test_setup, IntegrationTestDependencies class TestAccountClient(IBMTestCase): @@ -90,52 +90,52 @@ def test_client_error(self): class TestAuthClient(IBMTestCase): """Tests for the AuthClient.""" - @requires_qe_access - def test_valid_login(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_valid_login(self, dependencies: IntegrationTestDependencies): """Test valid authentication.""" - client = self._init_auth_client(qe_token, qe_url) + client = self._init_auth_client(dependencies.token, dependencies.url) self.assertTrue(client.access_token) - @requires_qe_access - def test_url_404(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_url_404(self, dependencies: IntegrationTestDependencies): """Test login against a 404 URL""" - url_404 = re.sub(r"/api.*$", "/api/TEST_404", qe_url) + url_404 = re.sub(r"/api.*$", "/api/TEST_404", dependencies.url) with self.assertRaises(ApiError): - _ = self._init_auth_client(qe_token, url_404) + _ = self._init_auth_client(dependencies.token, url_404) - @requires_qe_access - def test_invalid_token(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_token(self, dependencies: IntegrationTestDependencies): """Test login using invalid token.""" qe_token = "INVALID_TOKEN" with self.assertRaises(ApiError): - _ = self._init_auth_client(qe_token, qe_url) + _ = self._init_auth_client(qe_token, dependencies.url) - @requires_qe_access - def test_url_unreachable(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_url_unreachable(self, dependencies: IntegrationTestDependencies): """Test login against an invalid (malformed) URL.""" qe_url = "INVALID_URL" with self.assertRaises(ApiError): - _ = self._init_auth_client(qe_token, qe_url) + _ = self._init_auth_client(dependencies.token, qe_url) - @requires_qe_access - def test_api_version(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_api_version(self, dependencies: IntegrationTestDependencies): """Check the version of the QX API.""" - client = self._init_auth_client(qe_token, qe_url) + client = self._init_auth_client(dependencies.token, dependencies.url) version = client.api_version() self.assertIsNotNone(version) - @requires_qe_access - def test_user_urls(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_user_urls(self, dependencies: IntegrationTestDependencies): """Check the user urls of the QX API.""" - client = self._init_auth_client(qe_token, qe_url) + client = self._init_auth_client(dependencies.token, dependencies.url) user_urls = client.user_urls() self.assertIsNotNone(user_urls) self.assertTrue("http" in user_urls and "ws" in user_urls) - @requires_qe_access - def test_user_hubs(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_user_hubs(self, dependencies: IntegrationTestDependencies): """Check the user hubs of the QX API.""" - client = self._init_auth_client(qe_token, qe_url) + client = self._init_auth_client(dependencies.token, dependencies.url) user_hubs = client.user_hubs() self.assertIsNotNone(user_hubs) for user_hub in user_hubs: diff --git a/test/test_proxies.py b/test/test_proxies.py index c2cc2dfa1..70cb38d21 100644 --- a/test/test_proxies.py +++ b/test/test_proxies.py @@ -24,7 +24,7 @@ from qiskit_ibm_runtime.api.exceptions import RequestsApiError from qiskit_ibm_runtime.proxies import ProxyConfiguration from .ibm_test_case import IBMTestCase -from .utils.decorators import requires_qe_access, requires_cloud_service +from .utils.decorators import IntegrationTestDependencies, integration_test_setup ADDRESS = "127.0.0.1" PORT = 8085 @@ -55,11 +55,13 @@ def tearDown(self): # wait for the process to terminate self.proxy_process.wait() - @requires_cloud_service - def test_proxies_cloud_runtime_client(self, service, instance): + @integration_test_setup(supported_auth=["cloud"]) + def test_proxies_cloud_runtime_client( + self, dependencies: IntegrationTestDependencies + ): """Should reach the proxy using RuntimeClient.""" # pylint: disable=unused-argument - params = service._client_params + params = dependencies.service._client_params params.proxies = ProxyConfiguration(urls=VALID_PROXIES) client = RuntimeClient(params) client.list_programs(limit=1) @@ -68,18 +70,20 @@ def test_proxies_cloud_runtime_client(self, service, instance): proxy_output = self.proxy_process.stdout.read().decode("utf-8") self.assertIn(api_line, proxy_output) - @requires_qe_access - def test_proxies_legacy_runtime_client(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_proxies_legacy_runtime_client( + self, dependencies: IntegrationTestDependencies + ): """Should reach the proxy using RuntimeClient.""" service = IBMRuntimeService( auth="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies={"urls": VALID_PROXIES}, ) service.programs(limit=1) - auth_line = pproxy_desired_access_log_line(qe_url) + auth_line = pproxy_desired_access_log_line(dependencies.url) api_line = list(service._hgps.values())[0]._api_client._session.base_url api_line = pproxy_desired_access_log_line(api_line) self.proxy_process.terminate() # kill to be able of reading the output @@ -90,19 +94,19 @@ def test_proxies_legacy_runtime_client(self, qe_token, qe_url): # Check if the API call (querying providers list) went through proxy. self.assertIn(api_line, proxy_output) - @requires_qe_access - def test_proxies_account_client(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_proxies_account_client(self, dependencies: IntegrationTestDependencies): """Should reach the proxy using AccountClient.""" service = IBMRuntimeService( auth="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies={"urls": VALID_PROXIES}, ) self.proxy_process.terminate() # kill to be able of reading the output - auth_line = pproxy_desired_access_log_line(qe_url) + auth_line = pproxy_desired_access_log_line(dependencies.url) api_line = list(service._hgps.values())[0]._api_client._session.base_url api_line = pproxy_desired_access_log_line(api_line) proxy_output = self.proxy_process.stdout.read().decode("utf-8") @@ -112,14 +116,16 @@ def test_proxies_account_client(self, qe_token, qe_url): # Check if the API call (querying providers list) went through proxy. self.assertIn(api_line, proxy_output) - @requires_qe_access - def test_proxies_authclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_proxies_authclient(self, dependencies: IntegrationTestDependencies): """Should reach the proxy using AuthClient.""" - pproxy_desired_access_log_line_ = pproxy_desired_access_log_line(qe_url) + pproxy_desired_access_log_line_ = pproxy_desired_access_log_line( + dependencies.url + ) params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls=VALID_PROXIES), ) @@ -131,13 +137,14 @@ def test_proxies_authclient(self, qe_token, qe_url): self.proxy_process.stdout.read().decode("utf-8"), ) - # pylint: disable=unused-argument - @requires_qe_access - def test_proxies_versionclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_proxies_versionclient(self, dependencies: IntegrationTestDependencies): """Should reach the proxy using IBMVersionFinder.""" - pproxy_desired_access_log_line_ = pproxy_desired_access_log_line(qe_url) + pproxy_desired_access_log_line_ = pproxy_desired_access_log_line( + dependencies.url + ) - version_finder = VersionClient(qe_url, proxies=VALID_PROXIES) + version_finder = VersionClient(dependencies.url, proxies=VALID_PROXIES) version_finder.version() self.proxy_process.terminate() # kill to be able of reading the output @@ -146,13 +153,15 @@ def test_proxies_versionclient(self, qe_token, qe_url): self.proxy_process.stdout.read().decode("utf-8"), ) - @requires_qe_access - def test_invalid_proxy_port_runtime_client(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_port_runtime_client( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using RuntimeClient.""" params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls=INVALID_PORT_PROXIES), ) with self.assertRaises(RequestsApiError) as context_manager: @@ -160,13 +169,15 @@ def test_invalid_proxy_port_runtime_client(self, qe_token, qe_url): client.list_programs(limit=1) self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - @requires_qe_access - def test_invalid_proxy_port_authclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_port_authclient( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using AuthClient.""" params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls=INVALID_PORT_PROXIES), ) with self.assertRaises(RequestsApiError) as context_manager: @@ -174,23 +185,28 @@ def test_invalid_proxy_port_authclient(self, qe_token, qe_url): self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - # pylint: disable=unused-argument - @requires_qe_access - def test_invalid_proxy_port_versionclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_port_versionclient( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using VersionClient.""" with self.assertRaises(RequestsApiError) as context_manager: - version_finder = VersionClient(qe_url, proxies=INVALID_PORT_PROXIES) + version_finder = VersionClient( + dependencies.url, proxies=INVALID_PORT_PROXIES + ) version_finder.version() self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - @requires_qe_access - def test_invalid_proxy_address_runtime_client(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_address_runtime_client( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using RuntimeClient.""" params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls=INVALID_ADDRESS_PROXIES), ) with self.assertRaises(RequestsApiError) as context_manager: @@ -199,13 +215,15 @@ def test_invalid_proxy_address_runtime_client(self, qe_token, qe_url): self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - @requires_qe_access - def test_invalid_proxy_address_authclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_address_authclient( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using AuthClient.""" params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls=INVALID_ADDRESS_PROXIES), ) with self.assertRaises(RequestsApiError) as context_manager: @@ -213,18 +231,21 @@ def test_invalid_proxy_address_authclient(self, qe_token, qe_url): self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - @requires_qe_access - def test_invalid_proxy_address_versionclient(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_invalid_proxy_address_versionclient( + self, dependencies: IntegrationTestDependencies + ): """Should raise RequestApiError with ProxyError using VersionClient.""" - # pylint: disable=unused-argument with self.assertRaises(RequestsApiError) as context_manager: - version_finder = VersionClient(qe_url, proxies=INVALID_ADDRESS_PROXIES) + version_finder = VersionClient( + dependencies.url, proxies=INVALID_ADDRESS_PROXIES + ) version_finder.version() self.assertIsInstance(context_manager.exception.__cause__, ProxyError) - @requires_qe_access - def test_proxy_urls(self, qe_token, qe_url): + @integration_test_setup(supported_auth=["legacy"], init_service=False) + def test_proxy_urls(self, dependencies: IntegrationTestDependencies): """Test different forms of the proxy urls.""" test_urls = [ "http://{}:{}".format(ADDRESS, PORT), @@ -235,8 +256,8 @@ def test_proxy_urls(self, qe_token, qe_url): with self.subTest(proxy_url=proxy_url): params = ClientParameters( auth_type="legacy", - token=qe_token, - url=qe_url, + token=dependencies.token, + url=dependencies.url, proxies=ProxyConfiguration(urls={"https": proxy_url}), ) version_finder = VersionClient( diff --git a/test/utils/decorators.py b/test/utils/decorators.py index 891ed2bed..93ee419c7 100644 --- a/test/utils/decorators.py +++ b/test/utils/decorators.py @@ -72,12 +72,15 @@ def _wrapper(self, *args, **kwargs): def integration_test_setup( supported_auth: Optional[List[str]] = None, + init_service: Optional[bool] = True, resolve_least_busy_device: Optional[bool] = False, ): """Returns a decorator for integration test initialization. Args: supported_auth: a list of supported auth types that this test supports + init_service: to initialize the IBMRuntimeService based on the current environment + configuration and return it via the test dependencies resolve_least_busy_device: to resolve the least busy device and return it via the test dependencies @@ -102,12 +105,14 @@ def integration_test_setup( def _decorator(func): @wraps(func) def _wrapper(self, *args, **kwargs): - service = IBMRuntimeService( - auth=auth, token=token, url=url, instance=instance - ) + service = None + if init_service: + service = IBMRuntimeService( + auth=auth, token=token, url=url, instance=instance + ) device = None - if resolve_least_busy_device: + if service and resolve_least_busy_device: if auth == "cloud": # TODO use real device when cloud supports it device = service.least_busy(min_num_qubits=5) @@ -117,11 +122,11 @@ def _wrapper(self, *args, **kwargs): ) dependencies = IntegrationTestDependencies( - service=service, auth=auth, token=token, url=url, instance=instance, + service=service, device=device, ) kwargs["dependencies"] = dependencies From 1ae508abeda946d748fab012262515cbaa6f77c9 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 17:29:39 +0100 Subject: [PATCH 07/18] [ci skip] use new decorators (part 2) --- test/ibm_test_case.py | 59 +++++++++++++++--------------- test/test_backend_serialization.py | 29 ++++++++------- test/test_integration_backend.py | 20 +++++----- test/test_integration_program.py | 28 +++++++------- test/test_jupyter.py | 3 -- test/utils/decorators.py | 16 -------- 6 files changed, 70 insertions(+), 85 deletions(-) diff --git a/test/ibm_test_case.py b/test/ibm_test_case.py index 763e999e7..ae98b7a68 100644 --- a/test/ibm_test_case.py +++ b/test/ibm_test_case.py @@ -24,7 +24,7 @@ from qiskit_ibm_runtime.exceptions import IBMNotAuthorizedError from .utils.utils import setup_test_logging -from .utils.decorators import requires_cloud_legacy_services +from .utils.decorators import IntegrationTestDependencies, integration_test_setup from .utils.templates import RUNTIME_PROGRAM, RUNTIME_PROGRAM_METADATA, PROGRAM_PREFIX @@ -67,12 +67,13 @@ class IBMIntegrationTestCase(IBMTestCase): """Custom integration test case for use with the Qiskit IBM Runtime.""" @classmethod - @requires_cloud_legacy_services - def setUpClass(cls, services): + @integration_test_setup() + def setUpClass(cls, dependencies: IntegrationTestDependencies): """Initial class level setup.""" # pylint: disable=arguments-differ super().setUpClass() - cls.services = services + cls.dependencies = dependencies + cls.service = dependencies.service def setUp(self) -> None: """Test level setup.""" @@ -84,18 +85,17 @@ def tearDown(self) -> None: """Test level teardown.""" super().tearDown() # Delete programs - for service in self.services: - for prog in self.to_delete[service.auth]: - with suppress(Exception): - service.delete_program(prog) + service = self.service + for prog in self.to_delete[service.auth]: + with suppress(Exception): + service.delete_program(prog) # Cancel and delete jobs. - for service in self.services: - for job in self.to_cancel[service.auth]: - with suppress(Exception): - job.cancel() - with suppress(Exception): - service.delete_job(job.job_id) + for job in self.to_cancel[service.auth]: + with suppress(Exception): + job.cancel() + with suppress(Exception): + service.delete_job(job.job_id) def _upload_program( self, @@ -135,11 +135,11 @@ def tearDownClass(cls) -> None: super().tearDownClass() # Delete default program. with suppress(Exception): - for service in cls.services: - service.delete_program(cls.program_ids[service.auth]) - cls.log.debug( - "Deleted %s program %s", service.auth, cls.program_ids[service.auth] - ) + service = cls.service + service.delete_program(cls.program_ids[service.auth]) + cls.log.debug( + "Deleted %s program %s", service.auth, cls.program_ids[service.auth] + ) @classmethod def _create_default_program(cls): @@ -148,21 +148,20 @@ def _create_default_program(cls): metadata["name"] = PROGRAM_PREFIX cls.program_ids = {} cls.sim_backends = {} - for service in cls.services: - try: - prog_id = service.upload_program( - data=RUNTIME_PROGRAM, metadata=metadata - ) - cls.log.debug("Uploaded %s program %s", service.auth, prog_id) - cls.program_ids[service.auth] = prog_id - except IBMNotAuthorizedError: - raise unittest.SkipTest("No upload access.") + service = cls.service + try: + prog_id = service.upload_program(data=RUNTIME_PROGRAM, metadata=metadata) + cls.log.debug("Uploaded %s program %s", service.auth, prog_id) + cls.program_ids[service.auth] = prog_id + except IBMNotAuthorizedError: + raise unittest.SkipTest("No upload access.") @classmethod def _find_sim_backends(cls): """Find a simulator backend for each service.""" - for service in cls.services: - cls.sim_backends[service.auth] = service.backends(simulator=True)[0].name() + cls.sim_backends[cls.service.auth] = cls.service.backends(simulator=True)[ + 0 + ].name() def _run_program( self, diff --git a/test/test_backend_serialization.py b/test/test_backend_serialization.py index 3fa81051b..2f4e1eb47 100644 --- a/test/test_backend_serialization.py +++ b/test/test_backend_serialization.py @@ -17,27 +17,30 @@ import dateutil.parser from .ibm_test_case import IBMTestCase -from .utils.decorators import requires_cloud_legacy_services, run_cloud_legacy_real +from .utils.decorators import ( + integration_test_setup, + run_integration_test, + IntegrationTestDependencies, +) class TestSerialization(IBMTestCase): """Test data serialization.""" @classmethod - @requires_cloud_legacy_services - def setUpClass(cls, services): + @integration_test_setup() + def setUpClass(cls, dependencies: IntegrationTestDependencies): """Initial class level setup.""" # pylint: disable=arguments-differ super().setUpClass() - cls.services = services - cls.instances = {} - for serv in services: - cls.instances[serv.auth] = serv._account.instance + cls.dependencies = dependencies + cls.service = dependencies.service + cls.instance = dependencies.instance - @run_cloud_legacy_real + @run_integration_test def test_backend_configuration(self, service): """Test deserializing backend configuration.""" - instance = self.instances[service.auth] if service.auth == "legacy" else None + instance = self.instance if service.auth == "legacy" else None backends = service.backends( operational=True, simulator=False, instance=instance ) @@ -62,10 +65,10 @@ def test_backend_configuration(self, service): backend.configuration().to_dict(), good_keys, good_keys_prefixes ) - @run_cloud_legacy_real + @run_integration_test def test_pulse_defaults(self, service): """Test deserializing backend configuration.""" - instance = self.instances[service.auth] if service.auth == "legacy" else None + instance = self.instance if service.auth == "legacy" else None backends = service.backends( operational=True, open_pulse=True, instance=instance ) @@ -79,10 +82,10 @@ def test_pulse_defaults(self, service): with self.subTest(backend=backend): self._verify_data(backend.defaults().to_dict(), good_keys) - @run_cloud_legacy_real + @run_integration_test def test_backend_properties(self, service): """Test deserializing backend properties.""" - instance = self.instances[service.auth] if service.auth == "legacy" else None + instance = self.instance if service.auth == "legacy" else None backends = service.backends( operational=True, simulator=False, instance=instance ) diff --git a/test/test_integration_backend.py b/test/test_integration_backend.py index ba138ab2a..025f099fd 100644 --- a/test/test_integration_backend.py +++ b/test/test_integration_backend.py @@ -15,16 +15,13 @@ from unittest import SkipTest from .ibm_test_case import IBMIntegrationTestCase -from .utils.decorators import ( - run_cloud_legacy_real, - requires_cloud_legacy_devices, -) +from .utils.decorators import run_integration_test class TestIntegrationBackend(IBMIntegrationTestCase): """Integration tests for backend functions.""" - @run_cloud_legacy_real + @run_integration_test def test_backends(self, service): """Test getting all backends.""" backends = service.backends() @@ -36,7 +33,7 @@ def test_backends(self, service): f"backend_names={backend_names}", ) - @run_cloud_legacy_real + @run_integration_test def test_get_backend(self, service): """Test getting a backend.""" backends = service.backends() @@ -48,13 +45,18 @@ class TestIBMBackend(IBMIntegrationTestCase): """Test ibm_backend module.""" @classmethod - @requires_cloud_legacy_devices - def setUpClass(cls, devices): + def setUpClass(cls): """Initial class level setup.""" # pylint: disable=arguments-differ # pylint: disable=no-value-for-parameter super().setUpClass() - cls.devices = devices + if cls.dependencies.auth == "cloud": + # TODO use real device when cloud supports it + cls.device = cls.dependencies.service.least_busy(min_num_qubits=5) + if cls.dependencies.auth == "legacy": + cls.device = cls.dependencies.service.least_busy( + simulator=False, min_num_qubits=5, instance=cls.dependencies.instance + ) def test_backend_status(self): """Check the status of a real chip.""" diff --git a/test/test_integration_program.py b/test/test_integration_program.py index e50c6bdd3..f39e5efdb 100644 --- a/test/test_integration_program.py +++ b/test/test_integration_program.py @@ -23,14 +23,14 @@ ) from .ibm_test_case import IBMIntegrationTestCase -from .utils.decorators import run_cloud_legacy_real +from .utils.decorators import run_integration_test from .utils.templates import RUNTIME_PROGRAM, PROGRAM_PREFIX class TestIntegrationProgram(IBMIntegrationTestCase): """Integration tests for runtime modules.""" - @run_cloud_legacy_real + @run_integration_test def test_list_programs(self, service): """Test listing programs.""" program_id = self._upload_program(service) @@ -43,7 +43,7 @@ def test_list_programs(self, service): found = True self.assertTrue(found, f"Program {program_id} not found!") - @run_cloud_legacy_real + @run_integration_test def test_list_programs_with_limit_skip(self, service): """Test listing programs with limit and skip.""" for _ in range(4): @@ -58,7 +58,7 @@ def test_list_programs_with_limit_skip(self, service): self.assertIn(all_ids[1], some_ids) self.assertIn(all_ids[2], some_ids) - @run_cloud_legacy_real + @run_integration_test def test_list_program(self, service): """Test listing a single program.""" program_id = self._upload_program(service) @@ -66,7 +66,7 @@ def test_list_program(self, service): self.assertEqual(program_id, program.program_id) self._validate_program(program) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_program_data(self, service): """Test retrieving program data""" program_id = self._upload_program(service) @@ -74,7 +74,7 @@ def test_retrieve_program_data(self, service): self.assertEqual(RUNTIME_PROGRAM, program.data) self._validate_program(program) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_unauthorized_program_data(self, service): """Test retrieving program data when user is not the program author""" programs = service.programs() @@ -88,7 +88,7 @@ def test_retrieve_unauthorized_program_data(self, service): with self.assertRaises(IBMNotAuthorizedError): return not_mine.data - @run_cloud_legacy_real + @run_integration_test def test_upload_program(self, service): """Test uploading a program.""" max_execution_time = 3000 @@ -100,7 +100,7 @@ def test_upload_program(self, service): self.assertTrue(program) self.assertEqual(max_execution_time, program.max_execution_time) - @run_cloud_legacy_real + @run_integration_test def test_upload_program_file(self, service): """Test uploading a program using a file.""" temp_fp = tempfile.NamedTemporaryFile(mode="w", delete=False) @@ -118,7 +118,7 @@ def test_upload_program_file(self, service): not os.environ.get("QISKIT_IBM_USE_STAGING_CREDENTIALS", ""), "Only runs on staging", ) - @run_cloud_legacy_real + @run_integration_test def test_upload_public_program(self, service): """Test uploading a public program.""" max_execution_time = 3000 @@ -137,7 +137,7 @@ def test_upload_public_program(self, service): not os.environ.get("QISKIT_IBM_USE_STAGING_CREDENTIALS", ""), "Only runs on staging", ) - @run_cloud_legacy_real + @run_integration_test def test_set_visibility(self, service): """Test setting the visibility of a program.""" program_id = self._upload_program(service) @@ -152,7 +152,7 @@ def test_set_visibility(self, service): # Verify changed self.assertNotEqual(start_vis, end_vis) - @run_cloud_legacy_real + @run_integration_test def test_delete_program(self, service): """Test deleting program.""" program_id = self._upload_program(service) @@ -160,7 +160,7 @@ def test_delete_program(self, service): with self.assertRaises(RuntimeProgramNotFound): service.program(program_id, refresh=True) - @run_cloud_legacy_real + @run_integration_test def test_double_delete_program(self, service): """Test deleting a deleted program.""" program_id = self._upload_program(service) @@ -168,7 +168,7 @@ def test_double_delete_program(self, service): with self.assertRaises(RuntimeProgramNotFound): service.delete_program(program_id) - @run_cloud_legacy_real + @run_integration_test def test_update_program_data(self, service): """Test updating program data.""" program_v1 = """ @@ -184,7 +184,7 @@ def main(backend, user_messenger, **kwargs): service.update_program(program_id=program_id, data=program_v2) self.assertEqual(program_v2, service.program(program_id).data) - @run_cloud_legacy_real + @run_integration_test def test_update_program_metadata(self, service): """Test updating program metadata.""" program_id = self._upload_program(service) diff --git a/test/test_jupyter.py b/test/test_jupyter.py index bc1826358..b60b69632 100644 --- a/test/test_jupyter.py +++ b/test/test_jupyter.py @@ -22,7 +22,6 @@ from qiskit_ibm_runtime.jupyter.dashboard.utils import BackendWithProviders from .ibm_test_case import IBMTestCase -from .utils.decorators import requires_provider @unittest.skip("Skip until jupyter is done") @@ -30,7 +29,6 @@ class TestBackendInfo(IBMTestCase): """Test backend information Jupyter widget.""" @classmethod - @requires_provider def setUpClass(cls, service, hub, group, project): # pylint: disable=arguments-differ super().setUpClass() @@ -78,7 +76,6 @@ class TestIBMDashboard(IBMTestCase): """Test backend information Jupyter widget.""" @classmethod - @requires_provider def setUpClass(cls, service, hub, group, project): # pylint: disable=arguments-differ super().setUpClass() diff --git a/test/utils/decorators.py b/test/utils/decorators.py index 93ee419c7..5b137a3eb 100644 --- a/test/utils/decorators.py +++ b/test/utils/decorators.py @@ -63,8 +63,6 @@ def _wrapper(self, *args, **kwargs): with self.subTest(service=self.dependencies.service): if self.dependencies.service: kwargs["service"] = self.dependencies.service - if self.dependencies.device: - kwargs["device"] = self.dependencies.device func(self, *args, **kwargs) return _wrapper @@ -73,7 +71,6 @@ def _wrapper(self, *args, **kwargs): def integration_test_setup( supported_auth: Optional[List[str]] = None, init_service: Optional[bool] = True, - resolve_least_busy_device: Optional[bool] = False, ): """Returns a decorator for integration test initialization. @@ -110,24 +107,12 @@ def _wrapper(self, *args, **kwargs): service = IBMRuntimeService( auth=auth, token=token, url=url, instance=instance ) - - device = None - if service and resolve_least_busy_device: - if auth == "cloud": - # TODO use real device when cloud supports it - device = service.least_busy(min_num_qubits=5) - if auth == "legacy": - device = service.least_busy( - simulator=False, min_num_qubits=5, instance=instance - ) - dependencies = IntegrationTestDependencies( auth=auth, token=token, url=url, instance=instance, service=service, - device=device, ) kwargs["dependencies"] = dependencies func(self, *args, **kwargs) @@ -144,4 +129,3 @@ class IntegrationTestDependencies: token: str auth: str url: str - device: Any From db4686a9feced9f60caa2db6d3312455d16f10ec Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 17:31:10 +0100 Subject: [PATCH 08/18] enable CI --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b1722426..96fa6755c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,6 @@ on: [ push, pull_request ] jobs: code-quality: - if: ${{ 'skip' == 'me' }} name: Run code quality checks runs-on: ubuntu-latest steps: @@ -39,7 +38,6 @@ jobs: run: make mypy if: ${{ !cancelled() }} documentation: - if: ${{ 'skip' == 'me' }} name: Build documentation runs-on: ubuntu-latest steps: @@ -64,7 +62,6 @@ jobs: name: html_docs path: docs/_build/html unit-tests: - if: ${{ 'skip' == 'me' }} # only kick-off test cases when basic code quality checks succeed needs: [ "code-quality" , "documentation" ] name: Run unit tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) @@ -99,6 +96,8 @@ jobs: name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) path: htmlcov integration-tests: + # only kick-off resource intensive integration tests if unit tests and all basic checks succeeded + needs: [ "unit-tests" ] name: Run integration tests (${{ matrix.environment }}) runs-on: ${{ matrix.os }} strategy: From 02d0d38d006f42d5d831acbdfd396ad9f78bdc9a Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Thu, 20 Jan 2022 17:35:10 +0100 Subject: [PATCH 09/18] only run integration tests on push events --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96fa6755c..d5e57c30e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,7 @@ jobs: name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) path: htmlcov integration-tests: + if: ${{ github.event_name == 'push' }} # only kick-off resource intensive integration tests if unit tests and all basic checks succeeded needs: [ "unit-tests" ] name: Run integration tests (${{ matrix.environment }}) From d01629c9638323dc968aa1b4fa31228685b2d9ab Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 12:00:22 +0100 Subject: [PATCH 10/18] fix lint --- test/test_basic_server_paths.py | 27 ++++++++++++++++++------ test/test_integration_interim_results.py | 18 ++++++++-------- test/test_integration_retrieve_job.py | 18 ++++++++-------- test/utils/decorators.py | 4 +--- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/test/test_basic_server_paths.py b/test/test_basic_server_paths.py index 52be3c2e1..116997687 100644 --- a/test/test_basic_server_paths.py +++ b/test/test_basic_server_paths.py @@ -13,23 +13,34 @@ """Tests that hit all the basic server endpoints using both a public and premium h/g/p.""" from .ibm_test_case import IBMTestCase -from .utils.decorators import requires_multiple_hgps +from .utils.decorators import integration_test_setup, IntegrationTestDependencies class TestBasicServerPaths(IBMTestCase): """Test the basic server endpoints using both a public and premium provider.""" @classmethod - @requires_multiple_hgps - def setUpClass(cls, service, open_hgp, premium_hgp): + @integration_test_setup(supported_auth=["legacy"]) + def setUpClass(cls, dependencies: IntegrationTestDependencies): # pylint: disable=arguments-differ super().setUpClass() - cls.service = service # Dict[str, IBMRuntimeService] - cls.hgps = [open_hgp, premium_hgp] + cls.service = dependencies.service + cls.hgps = list(dependencies.service._hgps.keys()) + + def _require_2_hgps(self): + if len(self.hgps) < 2: + self.skipTest("Test require at least 2 hub/group/project.") + + def _get_hgps(self): + open_hgp = self.hgps[-1] + premium_hgp = self.hgps[0] + return [open_hgp, premium_hgp] def test_device_properties_and_defaults(self): """Test device properties and defaults.""" - for hgp in self.hgps: + self._require_2_hgps() + + for hgp in self._get_hgps(): with self.subTest(hgp=hgp): pulse_backends = self.service.backends( simulator=False, operational=True, instance=hgp @@ -45,7 +56,9 @@ def test_device_properties_and_defaults(self): def test_device_status(self): """Test device status.""" - for hgp in self.hgps: + self._require_2_hgps() + + for hgp in self._get_hgps(): with self.subTest(hgp=hgp): backend = self.service.backends( simulator=False, operational=True, instance=hgp diff --git a/test/test_integration_interim_results.py b/test/test_integration_interim_results.py index a904229e2..cd5c4d8e0 100644 --- a/test/test_integration_interim_results.py +++ b/test/test_integration_interim_results.py @@ -17,7 +17,7 @@ from qiskit.providers.jobstatus import JobStatus from .ibm_test_case import IBMIntegrationJobTestCase -from .utils.decorators import run_cloud_legacy_real +from .utils.decorators import run_integration_test from .utils.utils import cancel_job_safe, wait_for_status from .mock.proxy_server import MockProxyServer, use_proxies @@ -25,7 +25,7 @@ class TestIntegrationInterimResults(IBMIntegrationJobTestCase): """Integration tests for interim result functions.""" - @run_cloud_legacy_real + @run_integration_test def test_interim_result_callback(self, service): """Test interim result callback.""" @@ -53,7 +53,7 @@ def result_callback(job_id, interim_result): self.assertFalse(callback_err) self.assertIsNotNone(job._ws_client._server_close_code) - @run_cloud_legacy_real + @run_integration_test def test_stream_results(self, service): """Test stream_results method.""" @@ -77,7 +77,7 @@ def result_callback(job_id, interim_result): self.assertFalse(callback_err) self.assertIsNotNone(job._ws_client._server_close_code) - @run_cloud_legacy_real + @run_integration_test def test_stream_results_done(self, service): """Test streaming interim results after job is done.""" @@ -95,7 +95,7 @@ def result_callback(job_id, interim_result): self.assertFalse(called_back) self.assertIsNotNone(job._ws_client._server_close_code) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_interim_results(self, service): """Test retrieving interim results with API endpoint""" int_res = "foo" @@ -104,7 +104,7 @@ def test_retrieve_interim_results(self, service): interim_results = job.interim_results() self.assertIn(int_res, interim_results[0]) - @run_cloud_legacy_real + @run_integration_test def test_callback_error(self, service): """Test error in callback method.""" @@ -130,7 +130,7 @@ def result_callback(job_id, interim_result): self.assertEqual(iterations - 1, final_it) self.assertIsNotNone(job._ws_client._server_close_code) - @run_cloud_legacy_real + @run_integration_test def test_callback_cancel_job(self, service): """Test canceling a running job while streaming results.""" @@ -161,7 +161,7 @@ def result_callback(job_id, interim_result): self.assertIsNotNone(job._ws_client._server_close_code) self.assertLess(final_it, iterations) - @run_cloud_legacy_real + @run_integration_test def test_websocket_proxy(self, service): """Test connecting to websocket via proxy.""" @@ -178,7 +178,7 @@ def result_callback(job_id, interim_result): # pylint: disable=unused-argument self.assertTrue(callback_called) - @run_cloud_legacy_real + @run_integration_test def test_websocket_proxy_invalid_port(self, service): """Test connecting to websocket via invalid proxy port.""" diff --git a/test/test_integration_retrieve_job.py b/test/test_integration_retrieve_job.py index 8c4e73037..26fdbe1d9 100644 --- a/test/test_integration_retrieve_job.py +++ b/test/test_integration_retrieve_job.py @@ -17,14 +17,14 @@ from qiskit.providers.jobstatus import JobStatus from .ibm_test_case import IBMIntegrationJobTestCase -from .utils.decorators import run_cloud_legacy_real +from .utils.decorators import run_integration_test from .utils.utils import wait_for_status, get_real_device class TestIntegrationRetrieveJob(IBMIntegrationJobTestCase): """Integration tests for job retrieval functions.""" - @run_cloud_legacy_real + @run_integration_test def test_retrieve_job_queued(self, service): """Test retrieving a queued job.""" real_device = get_real_device(service) @@ -35,7 +35,7 @@ def test_retrieve_job_queued(self, service): self.assertEqual(job.job_id, rjob.job_id) self.assertEqual(self.program_ids[service.auth], rjob.program_id) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_job_running(self, service): """Test retrieving a running job.""" job = self._run_program(service, iterations=10) @@ -44,7 +44,7 @@ def test_retrieve_job_running(self, service): self.assertEqual(job.job_id, rjob.job_id) self.assertEqual(self.program_ids[service.auth], rjob.program_id) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_job_done(self, service): """Test retrieving a finished job.""" job = self._run_program(service) @@ -53,7 +53,7 @@ def test_retrieve_job_done(self, service): self.assertEqual(job.job_id, rjob.job_id) self.assertEqual(self.program_ids[service.auth], rjob.program_id) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_all_jobs(self, service): """Test retrieving all jobs.""" job = self._run_program(service) @@ -67,7 +67,7 @@ def test_retrieve_all_jobs(self, service): break self.assertTrue(found, f"Job {job.job_id} not returned.") - @run_cloud_legacy_real + @run_integration_test def test_retrieve_jobs_limit(self, service): """Test retrieving jobs with limit.""" jobs = [] @@ -82,7 +82,7 @@ def test_retrieve_jobs_limit(self, service): rjob_ids.issubset(job_ids), f"Submitted: {job_ids}, Retrieved: {rjob_ids}" ) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_pending_jobs(self, service): """Test retrieving pending jobs (QUEUED, RUNNING).""" job = self._run_program(service, iterations=10) @@ -102,7 +102,7 @@ def test_retrieve_pending_jobs(self, service): f"Pending job {job.job_id} not retrieved.", ) - @run_cloud_legacy_real + @run_integration_test def test_retrieve_returned_jobs(self, service): """Test retrieving returned jobs (COMPLETED, FAILED, CANCELLED).""" job = self._run_program(service) @@ -117,7 +117,7 @@ def test_retrieve_returned_jobs(self, service): break self.assertTrue(found, f"Returned job {job.job_id} not retrieved.") - @run_cloud_legacy_real + @run_integration_test def test_retrieve_jobs_by_program_id(self, service): """Test retrieving jobs by Program ID.""" program_id = self._upload_program(service) diff --git a/test/utils/decorators.py b/test/utils/decorators.py index 5b137a3eb..5c1c52641 100644 --- a/test/utils/decorators.py +++ b/test/utils/decorators.py @@ -75,11 +75,9 @@ def integration_test_setup( """Returns a decorator for integration test initialization. Args: - supported_auth: a list of supported auth types that this test supports + supported_auth: a list of auth types that this test supports init_service: to initialize the IBMRuntimeService based on the current environment configuration and return it via the test dependencies - resolve_least_busy_device: to resolve the least busy device and return it via the - test dependencies Returns: A decorator that handles initialization of integration test dependencies. From 761cfef3b3d569cfedf21ff8c537a04f9864af2c Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 12:19:38 +0100 Subject: [PATCH 11/18] fix unit tests --- test/test_integration_job.py | 38 ++++++++++++++++++------------------ test/utils/decorators.py | 32 +++++++++++++++++------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/test/test_integration_job.py b/test/test_integration_job.py index ac6074621..b970ba42d 100644 --- a/test/test_integration_job.py +++ b/test/test_integration_job.py @@ -26,7 +26,7 @@ ) from .ibm_test_case import IBMIntegrationJobTestCase -from .utils.decorators import run_cloud_legacy_real +from .utils.decorators import run_integration_test from .utils.serialization import ( get_complex_types, SerializableClassDecoder, @@ -38,7 +38,7 @@ class TestIntegrationJob(IBMIntegrationJobTestCase): """Integration tests for job functions.""" - @run_cloud_legacy_real + @run_integration_test def test_run_program(self, service): """Test running a program.""" job = self._run_program(service, final_result="foo") @@ -47,7 +47,7 @@ def test_run_program(self, service): self.assertEqual("foo", result) @slow_test - @run_cloud_legacy_real + @run_integration_test def test_run_program_real_device(self, service): """Test running a program.""" device = get_real_device(service) @@ -62,7 +62,7 @@ def test_run_program_cloud_no_backend(self): job = self._run_program(service, backend="") self.assertTrue(job.backend, f"Job {job.job_id} has no backend.") - @run_cloud_legacy_real + @run_integration_test def test_run_program_log_level(self, service): """Test running with a custom log level.""" levels = ["INFO", "ERROR"] @@ -77,7 +77,7 @@ def test_run_program_log_level(self, service): f"Job log is {job.logs()}", ) - @run_cloud_legacy_real + @run_integration_test def test_run_program_failed(self, service): """Test a failed program execution.""" job = self._run_program(service, inputs={}) @@ -92,7 +92,7 @@ def test_run_program_failed(self, service): job.result() self.assertIn("KeyError", str(err_cm.exception)) - @run_cloud_legacy_real + @run_integration_test def test_run_program_failed_ran_too_long(self, service): """Test a program that failed since it ran longer than maximum execution time.""" max_execution_time = 60 @@ -114,7 +114,7 @@ def test_run_program_failed_ran_too_long(self, service): with self.assertRaises(RuntimeJobFailureError): job.result() - @run_cloud_legacy_real + @run_integration_test def test_cancel_job_queued(self, service): """Test canceling a queued job.""" real_device = get_real_device(service) @@ -127,7 +127,7 @@ def test_cancel_job_queued(self, service): rjob = service.job(job.job_id) self.assertEqual(rjob.status(), JobStatus.CANCELLED) - @run_cloud_legacy_real + @run_integration_test def test_cancel_job_running(self, service): """Test canceling a running job.""" job = self._run_program(service, iterations=3) @@ -138,7 +138,7 @@ def test_cancel_job_running(self, service): rjob = service.job(job.job_id) self.assertEqual(rjob.status(), JobStatus.CANCELLED) - @run_cloud_legacy_real + @run_integration_test def test_cancel_job_done(self, service): """Test canceling a finished job.""" job = self._run_program(service) @@ -146,7 +146,7 @@ def test_cancel_job_done(self, service): with self.assertRaises(RuntimeInvalidStateError): job.cancel() - @run_cloud_legacy_real + @run_integration_test def test_delete_job(self, service): """Test deleting a job.""" sub_tests = [JobStatus.RUNNING, JobStatus.DONE] @@ -158,7 +158,7 @@ def test_delete_job(self, service): with self.assertRaises(RuntimeJobNotFound): service.job(job.job_id) - @run_cloud_legacy_real + @run_integration_test def test_delete_job_queued(self, service): """Test deleting a queued job.""" real_device = get_real_device(service) @@ -169,7 +169,7 @@ def test_delete_job_queued(self, service): with self.assertRaises(RuntimeJobNotFound): service.job(job.job_id) - @run_cloud_legacy_real + @run_integration_test def test_final_result(self, service): """Test getting final result.""" final_result = get_complex_types() @@ -180,14 +180,14 @@ def test_final_result(self, service): rresults = service.job(job.job_id).result(decoder=SerializableClassDecoder) self.assertEqual(final_result, rresults) - @run_cloud_legacy_real + @run_integration_test def test_job_status(self, service): """Test job status.""" job = self._run_program(service, iterations=1) time.sleep(random.randint(1, 5)) self.assertTrue(job.status()) - @run_cloud_legacy_real + @run_integration_test def test_job_inputs(self, service): """Test job inputs.""" interim_results = get_complex_types() @@ -198,26 +198,26 @@ def test_job_inputs(self, service): rinterim_results = rjob.inputs["interim_results"] self._assert_complex_types_equal(interim_results, rinterim_results) - @run_cloud_legacy_real + @run_integration_test def test_job_backend(self, service): """Test job backend.""" job = self._run_program(service) self.assertEqual(self.sim_backends[service.auth], job.backend.name()) - @run_cloud_legacy_real + @run_integration_test def test_job_program_id(self, service): """Test job program ID.""" job = self._run_program(service) self.assertEqual(self.program_ids[service.auth], job.program_id) - @run_cloud_legacy_real + @run_integration_test def test_wait_for_final_state(self, service): """Test wait for final state.""" job = self._run_program(service) job.wait_for_final_state() self.assertEqual(JobStatus.DONE, job.status()) - @run_cloud_legacy_real + @run_integration_test def test_job_creation_date(self, service): """Test job creation date.""" job = self._run_program(service, iterations=1) @@ -228,7 +228,7 @@ def test_job_creation_date(self, service): for rjob in rjobs: self.assertTrue(rjob.creation_date) - @run_cloud_legacy_real + @run_integration_test def test_job_logs(self, service): """Test job logs.""" job = self._run_program(service, final_result="foo") diff --git a/test/utils/decorators.py b/test/utils/decorators.py index 5c1c52641..4d8cc6a62 100644 --- a/test/utils/decorators.py +++ b/test/utils/decorators.py @@ -82,24 +82,26 @@ def integration_test_setup( Returns: A decorator that handles initialization of integration test dependencies. """ - if supported_auth is None: - supported_auth = ["cloud", "legacy"] - - if get_test_options()["skip_online"]: - raise SkipTest("Skipping integration test.") - - auth, token, url, instance = _get_integration_test_config() - if not all([auth, token, url]): - raise Exception("Configuration Issue") - - if auth not in supported_auth: - raise SkipTest( - f"Skipping integration test. Test does not support auth type {auth}" - ) def _decorator(func): @wraps(func) def _wrapper(self, *args, **kwargs): + _supported_auth = ( + ["cloud", "legacy"] if supported_auth is None else supported_auth + ) + + if get_test_options()["skip_online"]: + raise SkipTest("Skipping integration test.") + + auth, token, url, instance = _get_integration_test_config() + if not all([auth, token, url]): + raise Exception("Configuration Issue") + + if auth not in _supported_auth: + raise SkipTest( + f"Skipping integration test. Test does not support auth type {auth}" + ) + service = None if init_service: service = IBMRuntimeService( @@ -122,6 +124,8 @@ def _wrapper(self, *args, **kwargs): @dataclass class IntegrationTestDependencies: + """Integration test dependencies.""" + service: IBMRuntimeService instance: Optional[str] token: str From aa64a0d2b34560472dba0749cc21543e00a5ae57 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 13:57:51 +0100 Subject: [PATCH 12/18] update scheduled job runs --- .github/workflows/ci.yml | 10 +- .github/workflows/cron-prod.yml | 107 -------------------- .github/workflows/cron-slow-terra-main.yml | 45 --------- .github/workflows/cron-slow.yml | 44 --------- .github/workflows/cron-staging.yml | 110 --------------------- .github/workflows/integration-tests.yml | 60 +++++++++++ 6 files changed, 65 insertions(+), 311 deletions(-) delete mode 100644 .github/workflows/cron-prod.yml delete mode 100644 .github/workflows/cron-slow-terra-main.yml delete mode 100644 .github/workflows/cron-slow.yml delete mode 100644 .github/workflows/cron-staging.yml create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5e57c30e..72b60471d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,6 @@ jobs: run: make lint - name: Run mypy run: make mypy - if: ${{ !cancelled() }} documentation: name: Build documentation runs-on: ubuntu-latest @@ -64,7 +63,7 @@ jobs: unit-tests: # only kick-off test cases when basic code quality checks succeed needs: [ "code-quality" , "documentation" ] - name: Run unit tests (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + name: Run unit tests - python${{ matrix.python-version }}-${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -91,15 +90,16 @@ jobs: - name: Run unit tests run: make coverage - name: Upload unit test coverage report + if: uses: actions/upload-artifact@v2 with: - name: Unit test coverage report (Python ${{ matrix.python-version }}, ${{ matrix.os }}) + name: Unit test coverage report - python${{ matrix.python-version }}-${{ matrix.os }} path: htmlcov integration-tests: if: ${{ github.event_name == 'push' }} # only kick-off resource intensive integration tests if unit tests and all basic checks succeeded needs: [ "unit-tests" ] - name: Run integration tests (${{ matrix.environment }}) + name: Run integration tests - ${{ matrix.environment }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -132,5 +132,5 @@ jobs: - name: Upload integration test coverage report uses: actions/upload-artifact@v2 with: - name: Integration test coverage report (${{ matrix.environment }}) + name: Integration test coverage report - ${{ matrix.environment }} path: htmlcov \ No newline at end of file diff --git a/.github/workflows/cron-prod.yml b/.github/workflows/cron-prod.yml deleted file mode 100644 index f1652c586..000000000 --- a/.github/workflows/cron-prod.yml +++ /dev/null @@ -1,107 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -name: Cron-prod -on: - schedule: - - cron: '0 4 * * *' -jobs: - test1: - name: tests1-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test1 - test2: - name: tests2-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test2 - test3: - name: tests3-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test3 diff --git a/.github/workflows/cron-slow-terra-main.yml b/.github/workflows/cron-slow-terra-main.yml deleted file mode 100644 index 35da9cc17..000000000 --- a/.github/workflows/cron-slow-terra-main.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -name: Slow-test-terra-main -on: - schedule: - - cron: '0 7 * * *' -jobs: - terra-main: - name: terra-main - runs-on: macOS-latest - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_TESTS: run_slow - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - pip install -U git+https://github.com/Qiskit/qiskit-terra.git - - name: Run Tests - run: make test diff --git a/.github/workflows/cron-slow.yml b/.github/workflows/cron-slow.yml deleted file mode 100644 index 5c0433585..000000000 --- a/.github/workflows/cron-slow.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -name: Slow-test -on: - schedule: - - cron: '0 7 * * *' -jobs: - slow-test: - name: slow-test - runs-on: macOS-latest - env: - QISKIT_IBM_API_TOKEN: ${{ secrets.QISKIT_IBM_API_TOKEN }} - QISKIT_IBM_API_URL: ${{ secrets.QISKIT_IBM_API_URL }} - QISKIT_IBM_HGP: ${{ secrets.QISKIT_IBM_HGP }} - QISKIT_IBM_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_CLOUD_TOKEN }} - QISKIT_IBM_CLOUD_URL: ${{ secrets.QISKIT_IBM_CLOUD_URL }} - QISKIT_IBM_CLOUD_CRN: ${{ secrets.QISKIT_IBM_CLOUD_CRN }} - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_TESTS: run_slow - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test diff --git a/.github/workflows/cron-staging.yml b/.github/workflows/cron-staging.yml deleted file mode 100644 index 460f6136f..000000000 --- a/.github/workflows/cron-staging.yml +++ /dev/null @@ -1,110 +0,0 @@ -# This code is part of Qiskit. -# -# (C) Copyright IBM 2021. -# -# This code is licensed under the Apache License, Version 2.0. You may -# obtain a copy of this license in the LICENSE.txt file in the root directory -# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. -# -# Any modifications or derivative works of this code must retain this -# copyright notice, and modified files need to carry a notice indicating -# that they have been altered from the originals. - -name: Cron-staging -on: - schedule: - - cron: '0 5 * * *' -jobs: - test1: - name: tests1-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_STAGING_API_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_API_TOKEN }} - QISKIT_IBM_STAGING_API_URL: ${{ secrets.QISKIT_IBM_STAGING_API_URL }} - QISKIT_IBM_STAGING_HGP: ${{ secrets.QISKIT_IBM_STAGING_HGP }} - QISKIT_IBM_STAGING_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_TOKEN }} - QISKIT_IBM_STAGING_CLOUD_URL: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_URL }} - QISKIT_IBM_STAGING_CLOUD_CRN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_CRN }} - USE_STAGING_CREDENTIALS: True - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test1 - test2: - name: tests2-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_STAGING_API_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_API_TOKEN }} - QISKIT_IBM_STAGING_API_URL: ${{ secrets.QISKIT_IBM_STAGING_API_URL }} - QISKIT_IBM_STAGING_HGP: ${{ secrets.QISKIT_IBM_STAGING_HGP }} - QISKIT_IBM_STAGING_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_TOKEN }} - QISKIT_IBM_STAGING_CLOUD_URL: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_URL }} - QISKIT_IBM_STAGING_CLOUD_CRN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_CRN }} - QISKIT_IBM_USE_STAGING_CREDENTIALS: True - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test2 - test3: - name: tests3-python${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.7, 3.8, 3.9] - os: ["windows-latest", "ubuntu-latest"] - env: - QISKIT_IBM_STAGING_API_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_API_TOKEN }} - QISKIT_IBM_STAGING_API_URL: ${{ secrets.QISKIT_IBM_STAGING_API_URL }} - QISKIT_IBM_STAGING_HGP: ${{ secrets.QISKIT_IBM_STAGING_HGP }} - QISKIT_IBM_STAGING_CLOUD_TOKEN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_TOKEN }} - QISKIT_IBM_STAGING_CLOUD_URL: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_URL }} - QISKIT_IBM_STAGING_CLOUD_CRN: ${{ secrets.QISKIT_IBM_STAGING_CLOUD_CRN }} - QISKIT_IBM_USE_STAGING_CREDENTIALS: True - LOG_LEVEL: DEBUG - STREAM_LOG: True - QISKIT_IN_PARALLEL: True - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Deps - run: | - python -m pip install --upgrade pip - pip install -c constraints.txt -e . - pip install -U -c constraints.txt -r requirements-dev.txt - - name: Run Tests - run: make test3 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..44060bc2c --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,60 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2021. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +name: Integration Tests +on: + schedule: + - cron: '0 4 * * *' + # allow everyone with write access to the repository to trigger the workflow run manually + workflow_dispatch: + inputs: + environment: + description: 'Environment to trigger the test run for.' + required: true +jobs: + integration-tests: + name: Run integration tests - ${{ matrix.environment }} + runs-on: ${{ matrix.os }} + if: ${{ (github.event_name == 'schedule') || (matrix.environment == github.event.inputs.environment) }} + strategy: + matrix: + python-version: [ 3.9 ] + os: [ "ubuntu-latest" ] + environment: [ "legacy-production", "legacy-staging", "cloud-production" ] + environment: ${{ matrix.environment }} + env: + QISKIT_IBM_TOKEN: ${{ secrets.QISKIT_IBM_TOKEN }} + QISKIT_IBM_URL: ${{ secrets.QISKIT_IBM_URL }} + QISKIT_IBM_INSTANCE: ${{ secrets.QISKIT_IBM_INSTANCE }} + LOG_LEVEL: DEBUG + STREAM_LOG: True + QISKIT_IN_PARALLEL: True + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -c constraints.txt -e . + pip install -U -c constraints.txt -r requirements-dev.txt + - name: Run integration tests + run: make coverage + - name: Upload integration test coverage report + uses: actions/upload-artifact@v2 + with: + name: Integration test coverage report - ${{ matrix.environment }} + path: htmlcov \ No newline at end of file From 55465fa0a231d25c3ea37197100075999338da94 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 14:05:51 +0100 Subject: [PATCH 13/18] update scheduled job runs --- .github/workflows/integration-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 44060bc2c..70fdf8fdb 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -16,10 +16,6 @@ on: - cron: '0 4 * * *' # allow everyone with write access to the repository to trigger the workflow run manually workflow_dispatch: - inputs: - environment: - description: 'Environment to trigger the test run for.' - required: true jobs: integration-tests: name: Run integration tests - ${{ matrix.environment }} From e7478bf123c12463a6bedb98d09868d5fd45e1b2 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 14:15:12 +0100 Subject: [PATCH 14/18] fix integration test casess --- test/test_integration_backend.py | 44 +++++++++++++-------------- test/test_integration_job.py | 8 +++-- test/test_integration_retrieve_job.py | 7 +++-- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/test/test_integration_backend.py b/test/test_integration_backend.py index 025f099fd..1b76d95f9 100644 --- a/test/test_integration_backend.py +++ b/test/test_integration_backend.py @@ -52,43 +52,43 @@ def setUpClass(cls): super().setUpClass() if cls.dependencies.auth == "cloud": # TODO use real device when cloud supports it - cls.device = cls.dependencies.service.least_busy(min_num_qubits=5) + cls.backend = cls.dependencies.service.least_busy(min_num_qubits=5) if cls.dependencies.auth == "legacy": - cls.device = cls.dependencies.service.least_busy( + cls.backend = cls.dependencies.service.least_busy( simulator=False, min_num_qubits=5, instance=cls.dependencies.instance ) def test_backend_status(self): """Check the status of a real chip.""" - for backend in self.devices: - with self.subTest(backend=backend.name()): - self.assertTrue(backend.status().operational) + backend = self.backend + with self.subTest(backend=backend.name()): + self.assertTrue(backend.status().operational) def test_backend_properties(self): """Check the properties of calibration of a real chip.""" - for backend in self.devices: - with self.subTest(backend=backend.name()): - if backend.configuration().simulator: - raise SkipTest("Skip since simulator does not have properties.") - self.assertIsNotNone(backend.properties()) + backend = self.backend + with self.subTest(backend=backend.name()): + if backend.configuration().simulator: + raise SkipTest("Skip since simulator does not have properties.") + self.assertIsNotNone(backend.properties()) def test_backend_pulse_defaults(self): """Check the backend pulse defaults of each backend.""" - for backend in self.devices: - with self.subTest(backend=backend.name()): - if backend.configuration().simulator: - raise SkipTest("Skip since simulator does not have defaults.") - self.assertIsNotNone(backend.defaults()) + backend = self.backend + with self.subTest(backend=backend.name()): + if backend.configuration().simulator: + raise SkipTest("Skip since simulator does not have defaults.") + self.assertIsNotNone(backend.defaults()) def test_backend_configuration(self): """Check the backend configuration of each backend.""" - for backend in self.devices: - with self.subTest(backend=backend.name()): - self.assertIsNotNone(backend.configuration()) + backend = self.backend + with self.subTest(backend=backend.name()): + self.assertIsNotNone(backend.configuration()) def test_backend_run(self): """Check one cannot do backend.run""" - for backend in self.devices: - with self.subTest(backend=backend.name()): - with self.assertRaises(RuntimeError): - backend.run() + backend = self.backend + with self.subTest(backend=backend.name()): + with self.assertRaises(RuntimeError): + backend.run() diff --git a/test/test_integration_job.py b/test/test_integration_job.py index b970ba42d..376039c74 100644 --- a/test/test_integration_job.py +++ b/test/test_integration_job.py @@ -56,9 +56,13 @@ def test_run_program_real_device(self, service): self.assertEqual(JobStatus.DONE, job.status()) self.assertEqual("foo", result) - def test_run_program_cloud_no_backend(self): + @run_integration_test + def test_run_program_cloud_no_backend(self, service): """Test running a cloud program with no backend.""" - service = [serv for serv in self.services if serv.auth == "cloud"][0] + + if self.dependencies.auth == "legacy": + self.skipTest("Not supported on legacy") + job = self._run_program(service, backend="") self.assertTrue(job.backend, f"Job {job.job_id} has no backend.") diff --git a/test/test_integration_retrieve_job.py b/test/test_integration_retrieve_job.py index 26fdbe1d9..328f13d9e 100644 --- a/test/test_integration_retrieve_job.py +++ b/test/test_integration_retrieve_job.py @@ -127,9 +127,12 @@ def test_retrieve_jobs_by_program_id(self, service): self.assertEqual(program_id, rjobs[0].program_id) self.assertEqual(1, len(rjobs), f"Retrieved jobs: {[j.job_id for j in rjobs]}") - def test_jobs_filter_by_hgp(self): + @run_integration_test + def test_jobs_filter_by_hgp(self, service): """Test retrieving jobs by hgp.""" - service = [serv for serv in self.services if serv.auth == "legacy"][0] + if self.dependencies.auth == "cloud": + self.skipTest("Not supported on cloud") + default_hgp = list(service._hgps.keys())[0] program_id = self._upload_program(service) job = self._run_program(service, program_id=program_id) From 0bc55ff967c640918f534467ca11625735d131be Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 14:22:37 +0100 Subject: [PATCH 15/18] limit concurrent workflow runs per branch --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b60471d..028c5baf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ name: CI on: [ push, pull_request ] +# save resources: cancel redundant workflow runs on the same branch when new commits are pushed +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true jobs: code-quality: name: Run code quality checks From 9f7e79c25e1f40332a649648d8d902fa764ce4c3 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 14:27:54 +0100 Subject: [PATCH 16/18] remove broken if --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 028c5baf1..c76fb17e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,6 @@ jobs: - name: Run unit tests run: make coverage - name: Upload unit test coverage report - if: uses: actions/upload-artifact@v2 with: name: Unit test coverage report - python${{ matrix.python-version }}-${{ matrix.os }} From 99843a078ffa7cfed9789be0d002b37c3cda6662 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 14:31:04 +0100 Subject: [PATCH 17/18] [ci skip] update integration test yaml --- .github/workflows/integration-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 70fdf8fdb..11e78b968 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -20,7 +20,6 @@ jobs: integration-tests: name: Run integration tests - ${{ matrix.environment }} runs-on: ${{ matrix.os }} - if: ${{ (github.event_name == 'schedule') || (matrix.environment == github.event.inputs.environment) }} strategy: matrix: python-version: [ 3.9 ] From 45ff4e2e3bd014371e2aa26a6db055f9d300fcd0 Mon Sep 17 00:00:00 2001 From: Daniel Kaulen Date: Fri, 21 Jan 2022 15:15:26 +0100 Subject: [PATCH 18/18] [ci skip] update contribution guidelines --- CONTRIBUTING.md | 72 +++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f8f7b9922..c15007414 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,11 +7,11 @@ included in the qiskit documentation: https://qiskit.org/documentation/contributing_to_qiskit.html -Contributing to Qiskit IBM Provider +Contributing to Qiskit IBM Runtime --------------------------- In addition to the general guidelines there are specific details for -contributing to the Qiskit IBM Provider, these are documented below. +contributing to the Qiskit IBM Runtime, these are documented below. ### Pull request checklist @@ -199,7 +199,7 @@ section of the Qiskit documentation. ### Test New features often imply changes in the existent tests or new ones are -needed. Once they\'re updated/added run this be sure they keep passing. +needed. Once they're updated/added run this be sure they keep passing. For executing the tests, a `make test` target is available. @@ -209,7 +209,7 @@ command: Linux and Mac: ``` {.bash} -$ LOG_LEVEL=INFO python -m unittest test/test_something.py +$ python -m unittest test/test_something.py ``` Windows: @@ -218,43 +218,33 @@ Windows: C:\..\> python -m unittest test/test_something.py ``` -Note many of the tests will not be executed unless you have setup an -IBM Quantum account. To set this up please go to this -[page](https://quantum-computing.ibm.com/login) and -register an account. - -By default, and if there is no user credentials available, the tests -that require online access are run with recorded (mocked) information. -This is, the remote requests are replayed from a `test/cassettes` and -not real HTTP requests is generated. If user credentials are found, in -that cases it use them to make the network requests. - -How and which tests are executed is controlled by a environment variable -`QISKIT_TESTS`. The options are (where `uc_available = True` if the user -credentials are available, and `False` otherwise): - - ---------------------------------------------------------------------------------------------------- - Option Description Default If `True`, forces - --------------- --------------------------------------- -------------------- ----------------------- - `skip_online` Skips tests that require remote `False` `rec = False` - requests (also, no mocked information - is used). Does not require user - credentials. - - `mock_online` It runs the online tests using mocked `not uc_available` `skip_online = False` - information. Does not require user - credentials. - - `run_slow` It runs tests tagged as *slow*. `False` - - `rec` It records the remote requests. It `False` `skip_online = False` - requires user credentials. `run_slow = False` - ---------------------------------------------------------------------------------------------------- - -It is possible to provide more than one option separated with commas. -The order of precedence in the options is right to left. For example, -`QISKIT_TESTS=skip_online,rec` will set the options as -`skip_online == False` and `rec == True`. +The python tests either run locally ("unit tests") or against a remote API ("integration tests") +depending on the environment configuration. Check the [workflow definition](.github/workflows/ci.yml), +or the examples below for the set of supported environment variables. + +Run integration tests against IBM Quantum +```bash +QISKIT_IBM_TOKEN=... # IBM Quantum API token +QISKIT_IBM_URL=https://auth.quantum-computing.ibm.com/api # IBM Quantum API URL +QISKIT_IBM_INSTANCE=ibm-q/open/main # IBM Quantum provider to use (hub/group/project) +``` + +Run integration tests against IBM Cloud +```bash +QISKIT_IBM_TOKEN=... # IBM Cloud API key +QISKIT_IBM_URL=https://us-east.quantum-computing.cloud.ibm.com # Runtime URL +QISKIT_IBM_INSTANCE=crn:v1:bluemix:... # The CRN value of the Quantum service instance +``` + +Run unit tests +```bash +QISKIT_TESTS=skip_online # Option to skip all integration tests +``` + +To enable integration test cases to run in your private fork, make sure to set above values as +[encrypted environment secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-an-environment). +The names of the environments must match the ones that the [CI workflow](.github/workflows/ci.yml) relies +upon. ### Style guide