Make CI tests faster (qiskit-community#1246)

### Summary This PR does a few things to make tests run faster: - [x] Only test on the lowest and highest supported python versions - [x] Set MacOS build options to be the same as the other OSes (and remove coverage) - [x] Add `.stestr` to the cache as suggested by @mtreinish. This doesn't seem to improve the Ubuntu and Windows runtimes but significantly improves MacOS's. - [x] Group tests in stestr by class name. This might avoid large parallelized tests being run on multiple workers simultaneously and slowing each test down. - [x] Fail a test automatically if it takes longer than 60 seconds (hopefully this can be shortened in the future, but for now it mostly prevents a very long test from being added) - [x] Shorten long tests by decreasing shots and generating smaller circuits where the size isn't relevant (such as the roundtrip serialization tests) - [x] Also fixes a bug in the DRAG experiment where integer `beta` values caused a serialization error. Test are currently 20-40 minutes for Windows/Ubuntu and 50+ minutes for MacOS. With this PR, all tests go down to ~10 minutes. --------- Co-authored-by: Will Shanks <wshaos@posteo.net>
nkanazawa1989 · Jan 10, 2024 · 2df9391 · 2df9391
1 parent 13621dc
commit 2df9391
Show file tree

Hide file tree

Showing 24 changed files with 88 additions and 89 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.8, "3.11"]
         os: ["ubuntu-latest", "macOS-latest", "windows-latest"]
     steps:
       - name: Print Concurrency Group
@@ -39,25 +39,20 @@ jobs:
             ${{ runner.os }}-${{ matrix.python-version }}-pip-tests-
             ${{ runner.os }}-${{ matrix.python-version }}-pip-
             ${{ runner.os }}-${{ matrix.python-version }}
+      - name: Stestr cache
+        uses: actions/cache@v3
+        with:
+          path: .stestr
+          key: stestr-${{ runner.os }}-${{ matrix.python-version }}
+          restore-keys: |
+            stestr-${{ runner.os }}-
+            stestr-
       - name: Install Deps
-        run: python -m pip install -U "tox==3.27.1" setuptools virtualenv wheel
-      - name: Install and Run Tests (Windows and Linux)
+        run: python -m pip install -U tox setuptools virtualenv wheel stestr
+      - name: Install and Run Tests
         run: tox -e py
-        if: runner.os != 'macOS'
-      - name: Install and Run Tests (Macs only)
-        run: tox -e cover
-        if: runner.os == 'macOS'
-        env:
-          OMP_NUM_THREADS: 1
-      - name: Report coverage to coveralls.io (Macs only)
-        if: runner.os == 'macOS'
-        uses: coverallsapp/github-action@v2
-        env:
-          ACTIONS_RUNNER_DEBUG: 1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          flag-name: unit-tests_python${{ matrix.python-version }}-${{ matrix.os }}
-          path-to-lcov: coverage.lcov
+      - name: Clean up stestr cache
+        run: stestr history remove all
 
   lint:
     name: lint

diff --git a/.stestr.conf b/.stestr.conf
@@ -1,2 +1,3 @@
 [DEFAULT]
 test_path=./test
+parallel_class=True
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -121,6 +121,8 @@ To run a method:
 tox -- -n test.python.test_examples.TestPythonExamples.test_all_examples
 ```
 
+Note that tests will fail automatically if they do not finish execution within 60 seconds.
+
 #### STDOUT/STDERR and logging capture
 
 When running tests in parallel using `stestr` either via tox, the Makefile (`make

diff --git a/README.md b/README.md
@@ -3,7 +3,6 @@
 [![License](https://img.shields.io/github/license/Qiskit-Extensions/qiskit-experiments.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Release](https://img.shields.io/github/release/Qiskit-Extensions/qiskit-experiments.svg)](https://github.com/Qiskit-Extensions/qiskit-experiments/releases)
 ![Python](https://img.shields.io/pypi/pyversions/qiskit-experiments.svg)
-[![Coverage Status](https://coveralls.io/repos/github/Qiskit-Extensions/qiskit-experiments/badge.svg?branch=main)](https://coveralls.io/github/Qiskit-Extensions/qiskit-experiments?branch=main)
 [![DOI](https://joss.theoj.org/papers/10.21105/joss.05329/status.svg)](https://doi.org/10.21105/joss.05329)
 
 **Qiskit Experiments** is a repository that builds tools for building, running,

diff --git a/qiskit_experiments/library/characterization/drag.py b/qiskit_experiments/library/characterization/drag.py
@@ -166,7 +166,7 @@ def circuits(self) -> List[QuantumCircuit]:
             )
 
             for beta_val in self.experiment_options.betas:
-                beta_val = np.round(beta_val, decimals=6)
+                beta_val = float(np.round(beta_val, decimals=6))
 
                 assigned_circuit = circuit.assign_parameters({beta: beta_val}, inplace=False)
 

diff --git a/qiskit_experiments/library/tomography/mit_qpt_experiment.py b/qiskit_experiments/library/tomography/mit_qpt_experiment.py
@@ -78,7 +78,7 @@ def __init__(
             basis_indices: Optional, a list of basis indices for generating partial
                 tomography measurement data. Each item should be given as a pair of
                 lists of preparation and measurement basis configurations
-                ``([p[0], p[1], ..], m[0], m[1], ...])``, where ``p[i]`` is the
+                ``([p[0], p[1], ...], [m[0], m[1], ...])``, where ``p[i]`` is the
                 preparation basis index, and ``m[i]`` is the measurement basis index
                 for qubit-i. If not specified full tomography for all indices of the
                 preparation and measurement bases will be performed.

diff --git a/qiskit_experiments/library/tomography/qpt_experiment.py b/qiskit_experiments/library/tomography/qpt_experiment.py
@@ -81,7 +81,7 @@ def __init__(
             basis_indices: Optional, a list of basis indices for generating partial
                 tomography measurement data. Each item should be given as a pair of
                 lists of preparation and measurement basis configurations
-                ``([p[0], p[1], ..], m[0], m[1], ...])``, where ``p[i]`` is the
+                ``([p[0], p[1], ...], [m[0], m[1], ...])``, where ``p[i]`` is the
                 preparation basis index, and ``m[i]`` is the measurement basis index
                 for qubit-i. If not specified full tomography for all indices of the
                 preparation and measurement bases will be performed.

diff --git a/releasenotes/notes/rabi-and-qv-bugfix-34636baee6651af1.yaml b/releasenotes/notes/rabi-and-qv-bugfix-34636baee6651af1.yaml
@@ -5,4 +5,6 @@ fixes:
     `qiskit-ibm-provider` using custom amplitudes provided as a numpy array.
   - |
     Resolved an issue that caused QV experiments to fail when executed via `qiskit-ibm-provider` using
-    Qiskit Terra for calculating ideal probabilities, instead of Aer.
+    Qiskit Terra for calculating ideal probabilities, instead of Aer.
+  - |
+    Resolved a serialization issue that affected DRAG experiments with integral beta values specified.
diff --git a/test/base.py b/test/base.py
@@ -13,11 +13,13 @@
 Qiskit Experiments test case class
 """
 
+import os
 import json
 import pickle
 import warnings
 from typing import Any, Callable, Optional
 
+import fixtures
 import uncertainties
 from qiskit.test import QiskitTestCase
 from qiskit.utils.deprecation import deprecate_func
@@ -30,10 +32,17 @@
 from qiskit_experiments.framework.experiment_data import ExperimentStatus
 from .extended_equality import is_equivalent
 
+# Fail tests that take longer than this
+TEST_TIMEOUT = os.environ.get("TEST_TIMEOUT", 60)
+
 
 class QiskitExperimentsTestCase(QiskitTestCase):
     """Qiskit Experiments specific extra functionality for test cases."""
 
+    def setUp(self):
+        super().setUp()
+        self.useFixture(fixtures.Timeout(TEST_TIMEOUT, gentle=True))
+
     @classmethod
     def setUpClass(cls):
         """Set-up test class."""

diff --git a/test/library/calibration/test_drag.py b/test/library/calibration/test_drag.py
@@ -48,58 +48,42 @@ def setUp(self):
         self.x_plus = xp
         self.test_tol = 0.1
 
-    # pylint: disable=no-member
-    def test_end_to_end(self):
+    @data(
+        (None, None, None),
+        (0.0044, None, None),
+        (0.04, np.linspace(-4, 4, 31), {"beta": 1.8, "freq": 0.08}),
+    )
+    @unpack
+    def test_end_to_end(self, freq, betas, p0_opt):
         """Test the drag experiment end to end."""
 
         drag_experiment_helper = DragHelper(gate_name="Drag(xp)")
+        if freq:
+            drag_experiment_helper.frequency = freq
         backend = MockIQBackend(drag_experiment_helper)
 
         drag = RoughDrag([1], self.x_plus)
+        drag.set_run_options(shots=200)
+
+        if betas is not None:
+            drag.set_experiment_options(betas=betas)
+        if p0_opt:
+            drag.analysis.set_options(p0=p0_opt)
 
         expdata = drag.run(backend)
         self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
 
-        # pylint: disable=no-member
-        self.assertTrue(abs(result.value.n - backend.experiment_helper.ideal_beta) < self.test_tol)
-        self.assertEqual(result.quality, "good")
-
-        # Small leakage will make the curves very flat, in this case one should
-        # rather increase beta.
-        drag_experiment_helper.frequency = 0.0044
-
-        drag = RoughDrag([0], self.x_plus)
-        exp_data = drag.run(backend)
-        self.assertExperimentDone(exp_data)
-        result = exp_data.analysis_results(1)
-
-        # pylint: disable=no-member
-        self.assertTrue(abs(result.value.n - backend.experiment_helper.ideal_beta) < self.test_tol)
-        self.assertEqual(result.quality, "good")
-
-        # Large leakage will make the curves oscillate quickly.
-        drag_experiment_helper.frequency = 0.04
-        drag = RoughDrag([1], self.x_plus, betas=np.linspace(-4, 4, 31))
-        # pylint: disable=no-member
-        drag.set_run_options(shots=200)
-        drag.analysis.set_options(p0={"beta": 1.8, "freq": 0.08})
-        exp_data = drag.run(backend)
-        self.assertExperimentDone(exp_data)
-        result = exp_data.analysis_results(1)
-
-        meas_level = exp_data.metadata["meas_level"]
-
-        self.assertEqual(meas_level, MeasLevel.CLASSIFIED)
         self.assertTrue(abs(result.value.n - backend.experiment_helper.ideal_beta) < self.test_tol)
         self.assertEqual(result.quality, "good")
+        self.assertEqual(expdata.metadata["meas_level"], MeasLevel.CLASSIFIED)
 
     @data(
-        (0.0040, 1.0, 0.00, [1, 3, 5], None, 0.1),  # partial oscillation.
-        (0.0020, 0.5, 0.00, [1, 3, 5], None, 0.5),  # even slower oscillation with amp < 1
-        (0.0040, 0.8, 0.05, [3, 5, 7], None, 0.1),  # constant offset, i.e. lower SNR.
-        (0.0800, 0.9, 0.05, [1, 3, 5], np.linspace(-1, 1, 51), 0.1),  # Beta not in range
-        (0.2000, 0.5, 0.10, [1, 3, 5], np.linspace(-2.5, 2.5, 51), 0.1),  # Max closer to zero
+        (0.0040, 1.0, 0.00, [1, 3, 5], None, 0.2),  # partial oscillation.
+        (0.0020, 0.5, 0.00, [1, 3, 5], None, 1.0),  # even slower oscillation with amp < 1
+        (0.0040, 0.8, 0.05, [3, 5, 7], None, 0.2),  # constant offset, i.e. lower SNR.
+        (0.0800, 0.9, 0.05, [1, 3, 5], np.linspace(-1, 1, 51), 0.2),  # Beta not in range
+        (0.2000, 0.5, 0.10, [1, 3, 5], np.linspace(-2.5, 2.5, 51), 0.2),  # Max closer to zero
     )
     @unpack
     def test_nasty_data(self, freq, amp, offset, reps, betas, tol):
@@ -113,6 +97,7 @@ def test_nasty_data(self, freq, amp, offset, reps, betas, tol):
 
         drag = RoughDrag([0], self.x_plus, betas=betas)
         drag.set_experiment_options(reps=reps)
+        drag.set_run_options(shots=500)
 
         exp_data = drag.run(backend)
         self.assertExperimentDone(exp_data)
@@ -190,7 +175,7 @@ def test_default_circuits(self):
     def test_circuit_roundtrip_serializable(self):
         """Test circuit serializations for drag experiment."""
         drag = RoughDrag([0], self.x_plus)
-        drag.set_experiment_options(reps=[2, 4, 8])
+        drag.set_experiment_options(reps=[2, 4], betas=[-5, 5])
         drag.backend = FakeWashingtonV2()
         self.assertRoundTripSerializable(drag._transpiled_circuits())
 

diff --git a/test/library/calibration/test_rabi.py b/test/library/calibration/test_rabi.py
@@ -50,9 +50,10 @@ def setUp(self):
     def test_rabi_end_to_end(self):
         """Test the Rabi experiment end to end."""
 
-        test_tol = 0.015
+        test_tol = 0.15
 
         rabi = Rabi([self.qubit], self.sched, backend=self.backend)
+        rabi.set_run_options(shots=200)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.1, 0.1, 21))
         expdata = rabi.run()
         self.assertExperimentDone(expdata)

diff --git a/test/library/calibration/test_rough_amplitude.py b/test/library/calibration/test_rough_amplitude.py
@@ -84,7 +84,7 @@ def test_update(self):
 
     def test_circuit_roundtrip_serializable(self):
         """Test round trip JSON serialization"""
-        test_amps = [-0.5, 0, 0.5]
+        test_amps = [-0.5, 0]
         rabi = RoughXSXAmplitudeCal([0], self.cals, amplitudes=test_amps, backend=self.backend)
         self.assertRoundTripSerializable(rabi._transpiled_circuits())
 
@@ -151,14 +151,15 @@ def test_ef_circuits(self):
             self.assertEqual(circ.calibrations["Rabi"][((0,), (amp,))], expected_x12)
 
     def test_ef_update(self):
-        """Tes that we properly update the pulses on the 1<->2 transition."""
+        """Test that we properly update the pulses on the 1<->2 transition."""
 
-        tol = 0.01
+        tol = 0.05
         default_amp = 0.5 / self.backend.rabi_rate_12
 
         rabi_ef = EFRoughXSXAmplitudeCal(
             [0], self.cals, amplitudes=np.linspace(-0.1, 0.1, 11), backend=self.backend
         )
+        rabi_ef.set_run_options(shots=200)
         expdata = rabi_ef.run()
         self.assertExperimentDone(expdata)
 
@@ -172,6 +173,7 @@ def test_ef_update(self):
         self.cals.add_parameter_value(int(4 * 160 / 5), "duration", 0, "x12")
         self.cals.add_parameter_value(int(4 * 160 / 5), "duration", 0, "sx12")
         rabi_ef = EFRoughXSXAmplitudeCal([0], self.cals, amplitudes=np.linspace(-0.1, 0.1, 11))
+        rabi_ef.set_run_options(shots=200)
         expdata = rabi_ef.run(self.backend)
         self.assertExperimentDone(expdata)
 

diff --git a/test/library/characterization/test_cross_resonance_hamiltonian.py b/test/library/characterization/test_cross_resonance_hamiltonian.py
@@ -192,7 +192,7 @@ def test_integration(self, ix, iy, iz, zx, zy, zz):
         expr = cr_hamiltonian.CrossResonanceHamiltonian(
             physical_qubits=(0, 1),
             sigma=sigma,
-            # A hack to avoild local function in pickle, i.e. in transpile.
+            # A hack to avoid local function in pickle, i.e. in transpile.
             cr_gate=functools.partial(
                 SimulatableCRGate, hamiltonian=hamiltonian, sigma=sigma, dt=dt
             ),

diff --git a/test/library/characterization/test_qubit_spectroscopy.py b/test/library/characterization/test_qubit_spectroscopy.py
@@ -160,7 +160,7 @@ def test_experiment_config(self):
 
     def test_roundtrip_serializable(self):
         """Test round trip JSON serialization"""
-        exp = QubitSpectroscopy([1], np.linspace(int(100e6), int(150e6), int(20e6)))
+        exp = QubitSpectroscopy([1], np.linspace(int(100e6), int(150e6), 4))
         # Checking serialization of the experiment
         self.assertRoundTripSerializable(exp)
 
@@ -270,7 +270,9 @@ def test_parallel_experiment(self):
         par_experiment = ParallelExperiment(
             exp_list, flatten_results=False, backend=parallel_backend
         )
-        par_experiment.set_run_options(meas_level=MeasLevel.KERNELED, meas_return="single")
+        par_experiment.set_run_options(
+            meas_level=MeasLevel.KERNELED, meas_return="single", shots=20
+        )
 
         par_data = par_experiment.run()
         self.assertExperimentDone(par_data)
@@ -288,7 +290,7 @@ def test_circuit_roundtrip_serializable(self):
         backend = FakeWashingtonV2()
         qubit = 1
         freq01 = BackendData(backend).drive_freqs[qubit]
-        frequencies = np.linspace(freq01 - 10.0e6, freq01 + 10.0e6, 21)
+        frequencies = np.linspace(freq01 - 10.0e6, freq01 + 10.0e6, 3)
         exp = QubitSpectroscopy([1], frequencies, backend=backend)
         # Checking serialization of the experiment
         self.assertRoundTripSerializable(exp._transpiled_circuits())
diff --git a/test/library/characterization/test_readout_angle.py b/test/library/characterization/test_readout_angle.py
@@ -36,7 +36,7 @@ def test_readout_angle_end2end(self):
             MockIQReadoutAngleHelper(iq_cluster_centers=[((-3.0, 3.0), (5.0, 5.0))]),
         )
         exp = ReadoutAngle([0])
-        expdata = exp.run(backend, shots=100000)
+        expdata = exp.run(backend, shots=10000)
         self.assertExperimentDone(expdata)
         res = expdata.analysis_results(0)
         self.assertAlmostEqual(res.value % (2 * np.pi), np.pi / 2, places=2)
@@ -45,7 +45,7 @@ def test_readout_angle_end2end(self):
             MockIQReadoutAngleHelper(iq_cluster_centers=[((0, -3.0), (5.0, 5.0))]),
         )
         exp = ReadoutAngle([0])
-        expdata = exp.run(backend, shots=100000)
+        expdata = exp.run(backend, shots=10000)
         self.assertExperimentDone(expdata)
         res = expdata.analysis_results(0)
         self.assertAlmostEqual(res.value % (2 * np.pi), 15 * np.pi / 8, places=2)

diff --git a/test/library/characterization/test_resonator_spectroscopy.py b/test/library/characterization/test_resonator_spectroscopy.py
@@ -134,14 +134,14 @@ def test_end_to_end(self, freq_shift):
 
     def test_experiment_config(self):
         """Test converting to and from config works"""
-        exp = ResonatorSpectroscopy([1], frequencies=np.linspace(100, 150, 20) * 1e6)
+        exp = ResonatorSpectroscopy([1], frequencies=np.linspace(100, 150, 4) * 1e6)
         loaded_exp = ResonatorSpectroscopy.from_config(exp.config())
         self.assertNotEqual(exp, loaded_exp)
         self.assertEqualExtended(exp, loaded_exp)
 
     def test_roundtrip_serializable(self):
         """Test round trip JSON serialization"""
-        exp = ResonatorSpectroscopy([1], frequencies=np.linspace(int(100e6), int(150e6), int(20e6)))
+        exp = ResonatorSpectroscopy([1], frequencies=np.linspace(int(100e6), int(150e6), 4))
         self.assertRoundTripSerializable(exp)
 
     def test_circuit_roundtrip_serializable(self):
@@ -158,7 +158,7 @@ def test_circuit_roundtrip_serializable(self):
             ),
         )
         res_freq = BackendData(backend).meas_freqs[qubit]
-        frequencies = np.linspace(res_freq - 20e6, res_freq + 20e6, 51)
+        frequencies = np.linspace(res_freq - 20e6, res_freq + 20e6, 3)
         exp = ResonatorSpectroscopy([qubit], backend=backend, frequencies=frequencies)
         self.assertRoundTripSerializable(exp._transpiled_circuits())
 
@@ -177,7 +177,7 @@ def test_kerneled_expdata_serialization(self, freq_shift):
 
         res_freq = BackendData(backend).meas_freqs[qubit]
 
-        frequencies = np.linspace(res_freq - 20e6, res_freq + 20e6, 51)
+        frequencies = np.linspace(res_freq - 20e6, res_freq + 20e6, 11)
         exp = ResonatorSpectroscopy([qubit], backend=backend, frequencies=frequencies)
 
         expdata = exp.run(backend)
@@ -226,8 +226,8 @@ def test_parallel_experiment(self):
         res_freq1 = backend_data.meas_freqs[qubit1]
         res_freq2 = backend_data.meas_freqs[qubit2]
 
-        frequencies1 = np.linspace(res_freq1 - 20e6, res_freq1 + 20e6, 51)
-        frequencies2 = np.linspace(res_freq2 - 20e6, res_freq2 + 20e6, 53)
+        frequencies1 = np.linspace(res_freq1 - 20e6, res_freq1 + 20e6, 11)
+        frequencies2 = np.linspace(res_freq2 - 20e6, res_freq2 + 20e6, 13)
 
         res_spect1 = ResonatorSpectroscopy(
             [qubit1], backend=parallel_backend, frequencies=frequencies1

diff --git a/test/library/characterization/test_t1.py b/test/library/characterization/test_t1.py
@@ -311,7 +311,7 @@ def test_experiment_config(self):
 
     def test_roundtrip_serializable(self):
         """Test round trip JSON serialization"""
-        exp = T1([0], [1, 2, 3, 4, 5])
+        exp = T1([0], [1, 2])
         self.assertRoundTripSerializable(exp)
 
     def test_circuit_roundtrip_serializable(self):