diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d0f3ece..4787b254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,9 @@ v0.6 (June 11, 2018) -------------------- Breaking Changes: -- `operator_estimation.py` is entirely replaced. +- `operator_estimation.py` is entirely replaced. All changes from (gh-135) except where stated otherwise. -- `pyquil.operator_estimation` dependencies replaced with `forest.benchmarking.operator_estimation` +- `pyquil.operator_estimation` dependencies replaced with `forest.benchmarking.operator_estimation` (gh-129,132,133,134,135) - `operator_estimation.TomographyExperiment.out_op` -> `operator_estimation.ObservablesExperiment.out_observable` @@ -17,13 +17,13 @@ Breaking Changes: - `utils.all_pauli_terms` -> `utils.all_traceless_pauli_terms` -- `DFEData` and `DFEEstimate` dataclasses removed in favor of `ExperimentResult` and tuple of results respectively. +- `DFEData` and `DFEEstimate` dataclasses removed in favor of `ExperimentResult` and tuple of results respectively (gh-134). - plotting moved out of `qubit_spectroscopy`; instead, use `fit_*_results()` to get a `lmfit.model.ModelResult` and pass this into `analysis.fitting.make_figure()` -- `pandas.DataFrame` is no longer used in `randomized_benchmarking`, `qubit_spectroscopy`, and `robust_phase_estimation`. These now make use of `operator_estimation.ObservablesExperiment`, and as such the API has changed substantially. Please refer to example notebooks for new usage. +- `pandas.DataFrame` is no longer used in `randomized_benchmarking` (gh-133), `qubit_spectroscopy` (gh-129), and `robust_phase_estimation` (gh-135). These now make use of `operator_estimation.ObservablesExperiment`, and as such the API has changed substantially. Please refer to example notebooks for new usage. -- `pandas.DataFrame` methods removed from `quantum_volume`. See examples notebook for alternative usage. +- `pandas.DataFrame` methods removed from `quantum_volume`. See examples notebook for alternative usage (gh-136). - `utils.determine_simultaneous_grouping()` removed in favor of similar functionality in `operator_estimation.group_settings` diff --git a/examples/quantum_volume.ipynb b/examples/quantum_volume.ipynb index c42c0b3f..4f258a52 100644 --- a/examples/quantum_volume.ipynb +++ b/examples/quantum_volume.ipynb @@ -213,7 +213,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## For more fine-grained control, create and maintain a dataframe" + "## Run intermediate steps yourself" ] }, { @@ -222,19 +222,61 @@ "metadata": {}, "outputs": [], "source": [ - "from forest.benchmarking.quantum_volume import (generate_quantum_volume_experiments,\n", - " add_programs_to_dataframe,\n", - " acquire_quantum_volume_data,\n", - " acquire_heavy_hitters,\n", - " get_results_by_depth,\n", - " extract_quantum_volume_from_results)" + "from forest.benchmarking.quantum_volume import generate_abstract_qv_circuit, _naive_program_generator, collect_heavy_outputs\n", + "from pyquil.numpy_simulator import NumpyWavefunctionSimulator\n", + "from pyquil.gates import RESET\n", + "import time\n", + "\n", + "def generate_circuits(depths):\n", + " for d in depths:\n", + " yield generate_abstract_qv_circuit(d)\n", + "\n", + "def convert_ckts_to_programs(qc, circuits, qubits=None):\n", + " for idx, ckt in enumerate(circuits):\n", + " if qubits is None:\n", + " d_qubits = qc.qubits() # by default the program can act on any qubit in the computer\n", + " else:\n", + " d_qubits = qubits[idx]\n", + "\n", + " yield _naive_program_generator(qc, d_qubits, *ckt)\n", + "\n", + "\n", + "def acquire_quantum_volume_data(qc, programs, num_shots = 1000, use_active_reset = False):\n", + " for program in programs:\n", + " start = time.time()\n", + "\n", + " if use_active_reset:\n", + " reset_measure_program = Program(RESET())\n", + " program = reset_measure_program + program\n", + "\n", + " # run the program num_shots many times\n", + " program.wrap_in_numshots_loop(num_shots)\n", + " executable = qc.compiler.native_quil_to_executable(program)\n", + "\n", + " results = qc.run(executable)\n", + "\n", + " runtime = time.time() - start\n", + " yield results\n", + "\n", + "\n", + "def acquire_heavy_hitters(abstract_circuits):\n", + " for ckt in abstract_circuits:\n", + " perms, gates = ckt\n", + " depth = len(perms)\n", + " wfn_sim = NumpyWavefunctionSimulator(depth)\n", + "\n", + " start = time.time()\n", + " heavy_outputs = collect_heavy_outputs(wfn_sim, perms, gates)\n", + " runtime = time.time() - start\n", + "\n", + " yield heavy_outputs\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Get a dataframe with (depth x n_circuits) many \"Abstract Ckt\"s that describe each model circuit for each depth." + "Generate (len(unique_depths) x n_circuits) many \"Abstract Ckt\"s that describe each model circuit for each depth." ] }, { @@ -244,16 +286,17 @@ "outputs": [], "source": [ "n_circuits = 100\n", - "depths = [2,3]\n", - "df = generate_quantum_volume_experiments(depths, n_circuits)\n", - "df" + "unique_depths = [2,3]\n", + "depths = [d for d in unique_depths for _ in range(n_circuits)]\n", + "ckts = list(generate_circuits(depths))\n", + "print(ckts[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Use the default program_generator to synthesize native pyquil programs that implement each ckt natively on the qc." + "Use the _naive_program_generator to synthesize native pyquil programs that implement each ckt natively on the qc." ] }, { @@ -262,8 +305,8 @@ "metadata": {}, "outputs": [], "source": [ - "df = add_programs_to_dataframe(df, noisy_qc)\n", - "print(df[\"Program\"].values[0])" + "progs = list(convert_ckts_to_programs(noisy_qc, ckts))\n", + "print(progs[0])" ] }, { @@ -279,7 +322,8 @@ "metadata": {}, "outputs": [], "source": [ - "df = acquire_quantum_volume_data(df, noisy_qc, num_shots=10)" + "num_shots=10\n", + "results = list(acquire_quantum_volume_data(noisy_qc, progs, num_shots=num_shots))" ] }, { @@ -295,8 +339,24 @@ "metadata": {}, "outputs": [], "source": [ - "df = acquire_heavy_hitters(df)\n", - "print(df[\"Num HH Sampled\"].values[0:10])" + "ckt_hhs = acquire_heavy_hitters(ckts)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Count the number of heavy hitters that were sampled on the qc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from forest.benchmarking.quantum_volume import count_heavy_hitters_sampled\n", + "num_hh_sampled = count_heavy_hitters_sampled(results, ckt_hhs)" ] }, { @@ -312,7 +372,9 @@ "metadata": {}, "outputs": [], "source": [ - "results = get_results_by_depth(df)\n", + "from forest.benchmarking.quantum_volume import get_prob_sample_heavy_by_depth\n", + "\n", + "results = get_prob_sample_heavy_by_depth(depths, num_hh_sampled, [num_shots for _ in depths])\n", "results" ] }, @@ -329,6 +391,7 @@ "metadata": {}, "outputs": [], "source": [ + "from forest.benchmarking.quantum_volume import extract_quantum_volume_from_results\n", "qv = extract_quantum_volume_from_results(results)\n", "qv" ] diff --git a/forest/benchmarking/quantum_volume.py b/forest/benchmarking/quantum_volume.py index 7abf3646..cd09aa22 100644 --- a/forest/benchmarking/quantum_volume.py +++ b/forest/benchmarking/quantum_volume.py @@ -1,17 +1,13 @@ -from typing import List, Sequence, Tuple, Callable, Dict +from typing import List, Sequence, Tuple, Callable, Dict, Iterator import warnings from tqdm import tqdm import numpy as np from statistics import median -from collections import OrderedDict -from pandas import DataFrame, Series -import time from copy import copy from pyquil.api import QuantumComputer from pyquil.numpy_simulator import NumpyWavefunctionSimulator from pyquil.quil import DefGate, Program -from pyquil.gates import RESET from rpcq.messages import TargetDevice from rpcq._utils import RPCErrorError @@ -21,8 +17,8 @@ log = logging.getLogger(__name__) -def _naive_program_generator(qc: QuantumComputer, qubits: Sequence[int], permutations: np.ndarray, - gates: np.ndarray) -> Program: +def _naive_program_generator(qc: QuantumComputer, qubits: Sequence[int], + permutations: Sequence[np.ndarray], gates: np.ndarray) -> Program: """ Naively generates a native quil program to implement the circuit which is comprised of the given permutations and gates. @@ -157,8 +153,8 @@ def generate_abstract_qv_circuit(depth: int) -> Tuple[List[np.ndarray], np.ndarr def sample_rand_circuits_for_heavy_out(qc: QuantumComputer, qubits: Sequence[int], depth: int, program_generator: Callable[[QuantumComputer, Sequence[int], - np.ndarray, np.ndarray], - Program], + Sequence[np.ndarray], + np.ndarray], Program], num_circuits: int = 100, num_shots: int = 1000, show_progress_bar: bool = False) -> int: """ @@ -236,8 +232,9 @@ def calculate_prob_est_and_err(num_heavy: int, num_circuits: int, num_shots: int def measure_quantum_volume(qc: QuantumComputer, qubits: Sequence[int] = None, program_generator: Callable[[QuantumComputer, Sequence[int], - np.ndarray, np.ndarray], Program] = - _naive_program_generator, num_circuits: int = 100, num_shots: int = 1000, + Sequence[np.ndarray], np.ndarray], + Program] = _naive_program_generator, + num_circuits: int = 100, num_shots: int = 1000, depths: np.ndarray = None, achievable_threshold: float = 2/3, stop_when_fail: bool = True, show_progress_bar: bool = False) \ -> Dict[int, Tuple[float, float]]: @@ -265,7 +262,7 @@ def measure_quantum_volume(qc: QuantumComputer, qubits: Sequence[int] = None, :param qubits: available qubits on which to act during measurement. Default all qubits in qc. :param program_generator: a method which 1) takes in a quantum computer, the qubits on that - computer available for use, an array of sequences representing the qubit permutations + computer available for use, a series of sequences representing the qubit permutations in a model circuit, an array of matrices representing the 2q gates in the model circuit 2) outputs a native quil program that implements the circuit and measures the appropriate qubits in the order implicitly dictated by the model circuit representation created in @@ -319,225 +316,61 @@ def measure_quantum_volume(qc: QuantumComputer, qubits: Sequence[int] = None, return results -def generate_quantum_volume_experiments(depths: Sequence[int], num_circuits: int) -> DataFrame: +def count_heavy_hitters_sampled(qc_results: Iterator[np.ndarray], + heavy_hitters: Iterator[List[int]]) -> Iterator[int]: """ - Generate a dataframe with (depth * num_circuits) many rows each populated with an abstract - description of a model circuit of given depth=width necessary to measure quantum volume. - - See generate_abstract_qv_circuit and the reference [QVol] for more on the structure of each - circuit and the representation used here. - - :param num_circuits: The number of circuits to run for each depth. Should be > 100 - :param depths: The depths to measure. In order to properly lower bound the quantum volume of - a circuit, the depths should start at 2 and increase in increments of 1. Depths greater - than 4 will take several minutes for data collection. Further, the acquire_heavy_hitters - step involves a classical simulation that scales exponentially with depth. - :return: a dataframe with columns "Depth" and "Abstract Ckt" populated with the depth and an - abstract representation of a model circuit with that depth and width. - """ - def df_dict(): - for d in depths: - for _ in range(num_circuits): - yield OrderedDict({"Depth": d, - "Abstract Ckt": generate_abstract_qv_circuit(d)}) - return DataFrame(df_dict()) - - -def add_programs_to_dataframe(df: DataFrame, qc: QuantumComputer, - qubits_at_depth: Dict[int, Sequence[int]] = None, - program_generator: Callable[[QuantumComputer, Sequence[int], - np.ndarray, np.ndarray], Program] = - _naive_program_generator) -> DataFrame: - """ - Passes the abstract circuit description in each row of the dataframe df along to the supplied - program_generator which yields a program that can be run on the available - qubits_at_depth[depth] on the given qc resource. - - :param df: a dataframe populated with abstract descriptions of model circuits, i.e. a df - returned by a call to generate_quantum_volume_experiments. - :param qc: the quantum resource on which each output program will be run. - :param qubits_at_depth: the qubits of the qc available for use at each depth, default all - qubits in the qc for each depth. Any subset of these may actually be used by the program. - :param program_generator: a method which uses the given qc, its available qubits, and an - abstract description of the model circuit to produce a PyQuil program implementing the - circuit using only native gates and the given qubits. This program must respect the - topology of the qc induced by the given qubits. The default _naive_program_generator uses - the qc's compiler to achieve this result. - :return: a copy of df with a new "Program" column populated with native PyQuil programs that - implement the circuit in "Abstract Ckt" on the qc using a subset of the qubits specified - as available for the given depth. The used qubits are also recorded in a "Qubits" column. - Note that although the abstract circuit has depth=width, for the program width >= depth. - """ - new_df = df.copy() - - depths = new_df["Depth"].values - circuits = new_df["Abstract Ckt"].values - - if qubits_at_depth is None: - all_qubits = qc.qubits() # by default the program can act on any qubit in the computer - qubits = [all_qubits for _ in circuits] - else: - qubits = [qubits_at_depth[depth] for depth in depths] - - programs = [program_generator(qc, qbits, *ckt) for qbits, ckt in zip(qubits, circuits)] - new_df["Program"] = Series(programs) - - # these are the qubits actually used in the program, a subset of qubits_at_depth[depth] - new_df["Qubits"] = Series([program.get_qubits() for program in programs]) - - return new_df - - -def acquire_quantum_volume_data(df: DataFrame, qc: QuantumComputer, num_shots: int = 1000, - use_active_reset: bool = False) -> DataFrame: - """ - Runs each program in the dataframe df on the given qc and outputs a copy of df with results. - - :param df: a dataframe populated with PyQuil programs that can be run natively on the given qc, - i.e. a df returned by a call to add_programs_to_dataframe(df, qc, etc.) with identical qc. - :param qc: the quantum resource on which to run each program. - :param num_shots: the number of times to sample the output of each program. - :param use_active_reset: if true, speeds up the overall computation (only on a real qpu) by - actively resetting at the start of each program. - :return: a copy of df with a new "Results" column populated with num_shots many depth-bit arrays - that can be compared to the Heavy Hitters with a call to bit_array_to_int. There is also - a column "Run Time" which records the time taken to acquire the data for each program. - """ - new_df = df.copy() - - def run(q_comp, program, n_shots): - start = time.time() - - if use_active_reset: - reset_measure_program = Program(RESET()) - program = reset_measure_program + program - - # run the program num_shots many times - program.wrap_in_numshots_loop(n_shots) - executable = q_comp.compiler.native_quil_to_executable(program) - - res = q_comp.run(executable) - - end = time.time() - return res, end - start - - programs = new_df["Program"].values - data = [run(qc, program, num_shots) for program in programs] - - results = [datum[0] for datum in data] - times = [datum[1] for datum in data] - - new_df["Results"] = Series(results) - new_df["Run Time"] = Series(times) - - # supply the count of heavy hitters sampled if heavy hitters are known. - if "Heavy Hitters" in new_df.columns.values: - new_df = count_heavy_hitters_sampled(new_df) - - return new_df - - -def acquire_heavy_hitters(df: DataFrame) -> DataFrame: - """ - Runs a classical simulation of each circuit in the dataframe df and records which outputs - qualify as heavy hitters in a copied df with newly populated "Heavy Hitters" column. - - An output is a heavy hitter if the ideal probability of measuring that output from the - circuit is greater than the median probability among all possible bitstrings of the same size. + Simple helper to count the number of heavy hitters sampled given the sampled results for a + number of circuits along with the the actual heavy hitters for each circuit. - :param df: a dataframe populated with abstract descriptions of model circuits, i.e. a df - returned by a call to generate_quantum_volume_experiments. - :return: a copy of df with a new "Heavy Hitters" column. There is also a column "Sim Time" - which records the time taken to simulate and collect the heavy hitters for each circuit. + :param qc_results: results from running each circuit on a quantum computer. + :param heavy_hitters: the heavy hitters for each circuit (presumably calculated through + simulating the circuit classically) + :return: the number of samples which were heavy for each circuit. """ - new_df = df.copy() - - def run(depth, circuit): - wfn_sim = NumpyWavefunctionSimulator(depth) - - start = time.time() - heavy_outputs = collect_heavy_outputs(wfn_sim, *circuit) - end = time.time() - - return heavy_outputs, end - start - - circuits = new_df["Abstract Ckt"].values - depths = new_df["Depth"].values - - data = [run(d, ckt) for d, ckt in zip(depths, circuits)] - - heavy_hitters = [datum[0] for datum in data] - times = [datum[1] for datum in data] - - new_df["Heavy Hitters"] = Series(heavy_hitters) - new_df["Sim Time"] = Series(times) - - # supply the count of heavy hitters sampled if sampling results are known. - if "Results" in new_df.columns.values: - new_df = count_heavy_hitters_sampled(new_df) - - return new_df - - -def count_heavy_hitters_sampled(df: DataFrame) -> DataFrame: - """ - Given a df populated with both sampled results and the actual heavy hitters, copies the df - and populates a new column with the number of samples which are heavy hitters. - - :param df: a dataframe populated with sampled results and heavy hitters. - :return: a copy of df with a new "Num HH Sampled" column. - """ - new_df = df.copy() - - def count(hh, res): + for results, hh_list in zip(qc_results, heavy_hitters): num_heavy = 0 # determine if each result bitstring is a heavy output, as determined from simulation - for result in res: + for result in results: # convert result to int for comparison with heavy outputs. output = bit_array_to_int(result) - if output in hh: + if output in hh_list: num_heavy += 1 - return num_heavy - - exp_results = new_df["Results"].values - heavy_hitters = new_df["Heavy Hitters"].values + yield num_heavy - new_df["Num HH Sampled"] = Series([count(hh, exp_res) for hh, exp_res in zip(heavy_hitters, - exp_results)]) - return new_df - - -def get_results_by_depth(df: DataFrame) -> Dict[int, Tuple[float, float]]: +def get_prob_sample_heavy_by_depth(depths: Iterator[int], num_hh_sampled: Iterator[int], + num_shots: Iterator[int]) -> Dict[int, Tuple[float, float]]: """ - Analyzes a dataframe df to determine an estimate of the probability of outputting a heavy - hitter at each depth in the df, a lower bound on this estimate, and whether that depth was - achieved. + Analyzes the given information for each circuit to determine [an estimate of the probability of + outputting a heavy hitter at each depth, a lower bound on this estimate, and whether that + depth was achieved] The output of this method can be fed directly into extract_quantum_volume_from_results to obtain the quantum volume measured. - :param df: a dataframe populated with results, num hh sampled, and circuits for some number - of depths. + :param depths: the depth of each circuit + :param num_hh_sampled: the number of heavy hitters sampled from each circuit + :param num_shots: the number of shots / total number of samples from each circuit :return: for each depth key, provides a tuple of (estimate of probability of outputting hh for that depth=width, 2-sigma confidence interval (lower bound) on that estimate). The lower bound on the estimate is used to judge whether a depth is considered "achieved" in the context of the quantum volume. """ - depths = df["Depth"].values - - results = {} - for depth in depths: - single_depth = df.loc[df["Depth"] == depth] - num_shots = len(single_depth["Results"].values[0]) - num_heavy = sum(single_depth["Num HH Sampled"].values) - num_circuits = len(single_depth["Abstract Ckt"].values) - - prob_est, conf_intrvl = calculate_prob_est_and_err(num_heavy, num_circuits, num_shots) - - results[depth] = (prob_est, conf_intrvl) - - return results + nheavy_by_depth = {} + for depth, num_heavy, n_shots in zip(depths, num_hh_sampled, num_shots): + if depth not in nheavy_by_depth.keys(): + nheavy_by_depth[depth] = ([num_heavy], n_shots) + else: + nheavy_by_depth[depth][0].append(num_heavy) + assert n_shots == nheavy_by_depth[depth][1], 'The number of shots should be the same ' \ + 'for each circuit of a given depth.' + + results_by_depth = {} + for depth, (n_heavy, n_shots) in nheavy_by_depth.items(): + prob_est, conf_intrvl = calculate_prob_est_and_err(sum(n_heavy), len(n_heavy), n_shots) + results_by_depth[depth] = (prob_est, conf_intrvl) + + return results_by_depth def extract_quantum_volume_from_results(results: Dict[int, Tuple[float, float]]) -> int: diff --git a/forest/benchmarking/tests/test_quantum_volume.py b/forest/benchmarking/tests/test_quantum_volume.py index 1b0b2e38..d59446fe 100644 --- a/forest/benchmarking/tests/test_quantum_volume.py +++ b/forest/benchmarking/tests/test_quantum_volume.py @@ -1,6 +1,8 @@ import numpy as np import warnings +from pyquil.numpy_simulator import NumpyWavefunctionSimulator from forest.benchmarking.quantum_volume import * +from forest.benchmarking.quantum_volume import _naive_program_generator np.random.seed(1) @@ -25,63 +27,30 @@ def test_extraction(): assert extract_quantum_volume_from_results(outcomes) == 8 -def test_qv_df_generation(): - depths = [2, 3] - n_ckts = 100 - - df = generate_quantum_volume_experiments(depths, n_ckts) - df_depths = df["Depth"].values - ckts = df["Abstract Ckt"].values - - assert len(df_depths) == len(depths)*n_ckts - - assert all([len(ckt[0]) == depth for ckt, depth in zip(ckts, df_depths)]) - assert all([len(ckt[0][0]) == depth for ckt, depth in zip(ckts, df_depths)]) - - assert all([ckt[1].shape == (depth, depth//2, 4, 4) for ckt, depth in zip(ckts, df_depths)]) - - -def test_qv_data_acquisition(qvm): - depths = [2, 3] - n_ckts = 10 - n_shots = 5 - - df = generate_quantum_volume_experiments(depths, n_ckts) - df = add_programs_to_dataframe(df, qvm) - df = acquire_quantum_volume_data(df, qvm, n_shots) - - df_depths = df["Depth"].values - results = df["Results"].values - - assert all([res.shape == (n_shots, depth) for res, depth in zip(results, df_depths)]) - - -def test_qv_count_heavy_hitters(qvm): +def test_qv_get_results_by_depth(qvm): depths = [2, 3] n_ckts = 10 n_shots = 5 - df = generate_quantum_volume_experiments(depths, n_ckts) - df = add_programs_to_dataframe(df, qvm) - df = acquire_quantum_volume_data(df, qvm, n_shots) - df = acquire_heavy_hitters(df) - - num_hhs = df["Num HH Sampled"].values - - assert all([0 <= num_hh <= n_shots for num_hh in num_hhs]) + ckt_results = [] + ckt_hhs = [] + for depth in depths: + wfn_sim = NumpyWavefunctionSimulator(depth) + for _ in range(n_ckts): + permutations, gates = generate_abstract_qv_circuit(depth) + program = _naive_program_generator(qvm, qvm.qubits(), permutations, gates) + program.wrap_in_numshots_loop(n_shots) + executable = qvm.compiler.native_quil_to_executable(program) + results = qvm.run(executable) + ckt_results.append(results) -def test_qv_get_results_by_depth(qvm): - - depths = [2, 3] - n_ckts = 10 - n_shots = 5 + heavy_outputs = collect_heavy_outputs(wfn_sim, permutations, gates) + ckt_hhs.append(heavy_outputs) - df = generate_quantum_volume_experiments(depths, n_ckts) - df = add_programs_to_dataframe(df, qvm) - df = acquire_heavy_hitters(df) - df = acquire_quantum_volume_data(df, qvm, n_shots) - results = get_results_by_depth(df) + num_hh_sampled = count_heavy_hitters_sampled(ckt_results, ckt_hhs) + probs_by_depth = get_prob_sample_heavy_by_depth(depths, num_hh_sampled, + [n_shots for _ in depths]) - assert len(results.keys()) == len(depths) - assert [0 <= results[d][1] <= results[d][0] <= 1 for d in depths] + assert len(probs_by_depth.keys()) == len(depths) + assert [0 <= probs_by_depth[d][1] <= probs_by_depth[d][0] <= 1 for d in depths]