diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 23b080c93..b20855267 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1 +1,2 @@ 5c4a277fbb155529dc5cf06435f0d7419977d7dd +8e431b2f5827d3f9c69088bd68b432dc6d4d4769 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ead755daf..61093dfed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,7 +10,7 @@ repos: - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.11.0 hooks: - id: black args: [--safe, --line-length=100, --preview] @@ -24,7 +24,7 @@ repos: - id: docformatter - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.7.0 hooks: - id: mypy additional_dependencies: [ diff --git a/strax/config.py b/strax/config.py index 79cbbf2a7..a85d8a071 100644 --- a/strax/config.py +++ b/strax/config.py @@ -122,13 +122,11 @@ def __init__( # DeprecationWarning) if ( - sum( - [ - self.default is not OMITTED, - self.default_factory is not OMITTED, - self.default_by_run is not OMITTED, - ] - ) + sum([ + self.default is not OMITTED, + self.default_factory is not OMITTED, + self.default_by_run is not OMITTED, + ]) > 1 ): raise RuntimeError(f"Tried to specify more than one default for option {self.name}.") diff --git a/strax/context.py b/strax/context.py index 18d1153d6..43cfad2df 100644 --- a/strax/context.py +++ b/strax/context.py @@ -613,13 +613,11 @@ def _context_hash(self): """ base_hash_on_config = self.config.copy() # Also take into account the versions of the plugins registered - base_hash_on_config.update( - { - data_type: (plugin.__version__, plugin.compressor, plugin.input_timeout) - for data_type, plugin in self._plugin_class_registry.items() - if not data_type.startswith("_temp_") - } - ) + base_hash_on_config.update({ + data_type: (plugin.__version__, plugin.compressor, plugin.input_timeout) + for data_type, plugin in self._plugin_class_registry.items() + if not data_type.startswith("_temp_") + }) return strax.deterministic_hash(base_hash_on_config) def _plugins_are_cached( @@ -723,9 +721,9 @@ def _get_plugins( # Check all config options are taken by some registered plugin class # (helps spot typos) - all_opts = set().union( - *[pc.takes_config.keys() for pc in self._plugin_class_registry.values()] - ) + all_opts = set().union(*[ + pc.takes_config.keys() for pc in self._plugin_class_registry.values() + ]) for k in self.config: if not (k in all_opts or k in self.context_config["free_options"]): self.log.warning(f"Option {k} not taken by any registered plugin") @@ -2219,18 +2217,16 @@ def provided_dtypes(self, runid="0"): version """ - hashes = set( - [ - ( - data_type, - self.key_for(runid, data_type).lineage_hash, - self.get_save_when(data_type), - plugin.__version__, - ) - for plugin in self._plugin_class_registry.values() - for data_type in plugin.provides - ] - ) + hashes = set([ + ( + data_type, + self.key_for(runid, data_type).lineage_hash, + self.get_save_when(data_type), + plugin.__version__, + ) + for plugin in self._plugin_class_registry.values() + for data_type in plugin.provides + ]) return { data_type: dict(hash=_hash, save_when=save_when.name, version=version) diff --git a/strax/dtypes.py b/strax/dtypes.py index ce010507f..0f3dfe248 100644 --- a/strax/dtypes.py +++ b/strax/dtypes.py @@ -5,6 +5,7 @@ TODO: file numba issue. """ + from typing import Optional, Tuple import numpy as np diff --git a/strax/io.py b/strax/io.py index 8f2bd0e20..80d3c71bc 100644 --- a/strax/io.py +++ b/strax/io.py @@ -1,4 +1,5 @@ """Read/write numpy arrays to/from compressed files or file-like objects.""" + import os import bz2 diff --git a/strax/mailbox.py b/strax/mailbox.py index 173bda707..1e437f5a5 100644 --- a/strax/mailbox.py +++ b/strax/mailbox.py @@ -237,9 +237,9 @@ def _can_fetch(self): # If someone is still waiting for a message we already have # (so they just haven't woken up yet), don't fetch a new message. - if len(self._mailbox) and any( - [x is not None and x <= self._lowest_msg_number for x in self._subscriber_waiting_for] - ): + if len(self._mailbox) and any([ + x is not None and x <= self._lowest_msg_number for x in self._subscriber_waiting_for + ]): return False # Everyone is waiting for the new chunk or not at all. diff --git a/strax/plugins/merge_only_plugin.py b/strax/plugins/merge_only_plugin.py index 797ce713a..de798f71b 100644 --- a/strax/plugins/merge_only_plugin.py +++ b/strax/plugins/merge_only_plugin.py @@ -26,13 +26,11 @@ def infer_dtype(self): + str(deps_by_kind) ) - return strax.merged_dtype( - [ - self.deps[d].dtype_for(d) - # Sorting is needed here to match what strax.Chunk does in merging - for d in sorted(self.depends_on) - ] - ) + return strax.merged_dtype([ + self.deps[d].dtype_for(d) + # Sorting is needed here to match what strax.Chunk does in merging + for d in sorted(self.depends_on) + ]) def compute(self, **kwargs): return kwargs[list(kwargs.keys())[0]] diff --git a/strax/plugins/plugin.py b/strax/plugins/plugin.py index fe67da8ca..ee95d22c6 100644 --- a/strax/plugins/plugin.py +++ b/strax/plugins/plugin.py @@ -3,6 +3,7 @@ A 'plugin' is something that outputs an array and gets arrays from one or more other plugins. """ + from enum import IntEnum import inspect import itertools @@ -466,9 +467,9 @@ class IterDone(Exception): inputs[d], back_to_buffer = inputs[d].split( t=this_chunk_end, allow_early_split=True ) - self.input_buffer[d] = strax.Chunk.concatenate( - [back_to_buffer, self.input_buffer[d]] - ) + self.input_buffer[d] = strax.Chunk.concatenate([ + back_to_buffer, self.input_buffer[d] + ]) max_passes_left -= 1 else: raise RuntimeError( diff --git a/strax/processing/data_reduction.py b/strax/processing/data_reduction.py index bd3dba2b1..a48c38718 100644 --- a/strax/processing/data_reduction.py +++ b/strax/processing/data_reduction.py @@ -1,4 +1,5 @@ """Functions to perform in-place pulse-level data reduction.""" + import numpy as np import numba from enum import IntEnum diff --git a/strax/processing/pulse_processing.py b/strax/processing/pulse_processing.py index 2a905ee32..aa1cbb072 100644 --- a/strax/processing/pulse_processing.py +++ b/strax/processing/pulse_processing.py @@ -1,5 +1,6 @@ """Functions that perform processing on pulses (other than data reduction functions, which are in data_reduction.py)""" + import typing as ty import numpy as np diff --git a/strax/run_selection.py b/strax/run_selection.py index 30081571a..fb9ac1507 100644 --- a/strax/run_selection.py +++ b/strax/run_selection.py @@ -1,4 +1,5 @@ """Context methods dealing with run scanning and selection.""" + import fnmatch import re import typing as ty @@ -142,9 +143,9 @@ def scan_runs( if isinstance(doc["source"], list): doc["source"] = ",".join(doc["source"]) - doc["tags"] = ",".join( - [t["name"] if isinstance(t, dict) else t for t in doc.get("tags", [])] - ) + doc["tags"] = ",".join([ + t["name"] if isinstance(t, dict) else t for t in doc.get("tags", []) + ]) # Set a default livetime if we have start and stop if ( @@ -524,17 +525,13 @@ def _tags_match(dsets, patterns, pattern_type, ignore_underscore): patterns = [patterns] for i, tags in enumerate(dsets.tags): - result[i] = any( - [ - any( - [ - _tag_match(tag, pattern, pattern_type, ignore_underscore) - for tag in tags.split(",") - for pattern in patterns - ] - ) - ] - ) + result[i] = any([ + any([ + _tag_match(tag, pattern, pattern_type, ignore_underscore) + for tag in tags.split(",") + for pattern in patterns + ]) + ]) return result diff --git a/strax/storage/common.py b/strax/storage/common.py index 01c014013..3e3be50aa 100644 --- a/strax/storage/common.py +++ b/strax/storage/common.py @@ -3,6 +3,7 @@ Please see the developer documentation for more details on strax' storage hierarchy. """ + from ast import literal_eval from concurrent.futures import wait import logging diff --git a/strax/utils.py b/strax/utils.py index 00e1a5fc3..b2f2c52af 100644 --- a/strax/utils.py +++ b/strax/utils.py @@ -157,12 +157,10 @@ def unpack_dtype(dtype): @export def remove_titles_from_dtype(dtype): """Return np.dtype with titles removed from fields.""" - return np.dtype( - [ - (fieldname[-1] if isinstance(fieldname, tuple) else fieldname, dt) - for fieldname, dt in unpack_dtype(np.dtype(dtype)) - ] - ) + return np.dtype([ + (fieldname[-1] if isinstance(fieldname, tuple) else fieldname, dt) + for fieldname, dt in unpack_dtype(np.dtype(dtype)) + ]) @export diff --git a/tests/test_corrections.py b/tests/test_corrections.py index f05eedee9..478858d8c 100644 --- a/tests/test_corrections.py +++ b/tests/test_corrections.py @@ -36,14 +36,12 @@ def setUp(self): def make_dummy_df(): """Make a dummy pandas.dataframe()""" dates = [datetime(2017, 1, 1), datetime(2021, 1, 1), datetime(2021, 9, 23)] - df = pd.DataFrame( - { - "ONLINE": [10.0, 10.0, 8.0], - "v1": [12.0, 12.0, 14.0], - "v2": [13.0, 14.0, np.nan], - "time": dates, - } - ) + df = pd.DataFrame({ + "ONLINE": [10.0, 10.0, 8.0], + "v1": [12.0, 12.0, 14.0], + "v2": [13.0, 14.0, np.nan], + "time": dates, + }) df["time"] = pd.to_datetime(df["time"], utc=True) df = df.set_index("time") diff --git a/tests/test_general_processing.py b/tests/test_general_processing.py index 9bc6c15c9..5b0311b9a 100644 --- a/tests/test_general_processing.py +++ b/tests/test_general_processing.py @@ -282,9 +282,7 @@ def test_raw_to_records(r): @hypothesis.given( - hyp_numpy.arrays( - np.int64, 10**2, elements=hypothesis.strategies.integers(10**9, 5 * 10**9) - ), + hyp_numpy.arrays(np.int64, 10**2, elements=hypothesis.strategies.integers(10**9, 5 * 10**9)), hyp_numpy.arrays(np.int16, 10**2, elements=hypothesis.strategies.integers(-10, 10**3)), ) @hypothesis.settings(deadline=None) diff --git a/tests/test_mongo_frontend.py b/tests/test_mongo_frontend.py index 518f75cc7..06627c7fb 100644 --- a/tests/test_mongo_frontend.py +++ b/tests/test_mongo_frontend.py @@ -138,12 +138,10 @@ def test_interrupt_iterator(self): def test_allow_incomplete(self): """Test loading incomplete data.""" st_incomplete_allowed = self.st.new_context() - st_incomplete_allowed.set_context_config( - { - "allow_incomplete": True, - "forbid_creation_of": "*", - } - ) + st_incomplete_allowed.set_context_config({ + "allow_incomplete": True, + "forbid_creation_of": "*", + }) assert not self.is_stored_in_mongo self.st.config["n_chunks"] = 3 @@ -193,12 +191,10 @@ def test_allow_incomplete_during_md_creation(self): """ st_incomplete_allowed = self.st.new_context() - st_incomplete_allowed.set_context_config( - { - "allow_incomplete": True, - "forbid_creation_of": "*", - } - ) + st_incomplete_allowed.set_context_config({ + "allow_incomplete": True, + "forbid_creation_of": "*", + }) assert not self.is_stored_in_mongo self.st.config["n_chunks"] = 3 diff --git a/tests/test_multi_output.py b/tests/test_multi_output.py index a0da47532..2c99fab49 100644 --- a/tests/test_multi_output.py +++ b/tests/test_multi_output.py @@ -160,13 +160,11 @@ def test_save_when_per_provide_same_save_when(self): def test_save_when_per_provide(self): """Tests if save when works properly in case of different save when per provided data_type.""" - _save_when = immutabledict( - { - "even_recs": strax.SaveWhen.NEVER, - "odd_recs": strax.SaveWhen.TARGET, - "rec_count": strax.SaveWhen.ALWAYS, - } - ) + _save_when = immutabledict({ + "even_recs": strax.SaveWhen.NEVER, + "odd_recs": strax.SaveWhen.TARGET, + "rec_count": strax.SaveWhen.ALWAYS, + }) p = self.mystrax._plugin_class_registry["rec_count"] p.save_when = _save_when @@ -207,13 +205,11 @@ def test_save_when_per_provide(self): def test_save_per_provide_inlined(self): """Checks whether the plugin inlining works for different combinations.""" - _save_when = immutabledict( - { - "even_recs": strax.SaveWhen.NEVER, - "odd_recs": strax.SaveWhen.TARGET, - "rec_count": strax.SaveWhen.ALWAYS, - } - ) + _save_when = immutabledict({ + "even_recs": strax.SaveWhen.NEVER, + "odd_recs": strax.SaveWhen.TARGET, + "rec_count": strax.SaveWhen.ALWAYS, + }) p = self.mystrax._plugin_class_registry["rec_count"] p.save_when = _save_when @@ -255,17 +251,17 @@ def test_double_dependency_notlazy(self): def test_double_dependency_multiprocess(self): """Tests if double dependency of a plugin on another plugin leads to dead lock in processing.""" - self.mystrax.set_context_config( - {"timeout": 120, "allow_lazy": False, "allow_multiprocess": True} - ) + self.mystrax.set_context_config({ + "timeout": 120, "allow_lazy": False, "allow_multiprocess": True + }) self._test_double_dependency(max_workers=2) def test_double_dependency_inline_plugins(self): """Tests if double dependency of a plugin on another plugin leads to dead lock in processing.""" - self.mystrax.set_context_config( - {"timeout": 120, "allow_lazy": False, "allow_multiprocess": True} - ) + self.mystrax.set_context_config({ + "timeout": 120, "allow_lazy": False, "allow_multiprocess": True + }) self._test_double_dependency(max_workers=2, make_data_type="zipped_records_classified") self.mystrax.is_stored(run_id, "even_recs_classified") diff --git a/tests/test_superruns.py b/tests/test_superruns.py index b8ef96f1b..827a5aeb9 100644 --- a/tests/test_superruns.py +++ b/tests/test_superruns.py @@ -211,9 +211,9 @@ def test_rechnunking_and_loading(self): # three chunks next_subrun_id = int(self.subrun_ids[-1]) + 1 for run_id in range(next_subrun_id, next_subrun_id + 2): - self.context.set_config( - {"secret_time_offset": int(endtime + self.offset_between_subruns)} - ) + self.context.set_config({ + "secret_time_offset": int(endtime + self.offset_between_subruns) + }) rr = self.context.get_array(str(run_id), "records") self._write_run_doc( run_id, @@ -325,9 +325,9 @@ def _create_subruns(self, n_subruns=3): self.now + datetime.timedelta(0, int(endtime)), ) - self.context.set_config( - {"secret_time_offset": int(endtime + self.offset_between_subruns)} - ) + self.context.set_config({ + "secret_time_offset": int(endtime + self.offset_between_subruns) + }) assert self.context.is_stored(run_id, "records") def _write_run_doc(self, run_id, time, endtime):