update development

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.0
+current_version = 1.6.0
 files = setup.py straxen/__init__.py
 commit = True
 tag = True

.github/workflows/code_style.yml

@@ -7,7 +7,7 @@ jobs:
     name: Quality check
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@master
         with:
@@ -16,7 +16,7 @@ jobs:
         run: |
              bash .github/scripts/pre_pyflakes.sh
       - name: Wemake Python Stylguide
-        uses: wemake-services/wemake-python-styleguide@0.16.0
+        uses: wemake-services/wemake-python-styleguide@0.16.1
         continue-on-error: true
         with:
           reporter: 'github-pr-review'

.github/workflows/pypi_install.yml

@@ -18,7 +18,7 @@ jobs:
           python-version: '3.8'
 
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install dependencies
         run: pip install wheel

.github/workflows/pytest.yml

@@ -47,7 +47,7 @@ jobs:
     steps:
       # Setup and installation
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup python
         uses: actions/setup-python@v2.3.0

.github/workflows/test_install.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: pre-install requirements
         run: pip install -r requirements.txt
       - name: Install straxen

HISTORY.md

@@ -1,3 +1,28 @@
+1.6.0 / 2022-03-09
+------------------
+Minor:
+- Patch corrected areas (#964)
+- Patch in EventShadow (#967)
+- Bump version of merged S2s (#919)
+- Add Bayes peak probability (#954)
+- fix hit sorting, add raw-record-aqm overlap check (#961)
+
+Notes:
+- Lineage changed for event_shadow, merged_s2s, corrected_area and aqmon_hits.
+- Added new data_types peak_classification_bayes and event_w_bayes_class
+
+
+1.5.0 / 2022-03-08
+------------------
+- Update Shadow plugins and add Ambience plugins (#912)
+- Update online_monitor.py (#958)
+- fix ref to PR in docs (#957)
+
+Notes:
+ - Lineage changes for online_monitor_mv
+ - New plugins `peak_ambiance` and `event_ambiance` and changes in lineage of `event_shadow` and `peak_shadow`
+
+
 1.4.0 / 2022-03-02
 ------------------
 Minor:
@@ -11,7 +36,7 @@ Patch
 
 Notes:
 - Lineage changed for events_positions_nv, corrected_areas and event_pattern_fit due to option changes
-- Lineage chanes for events, event_basics, event_positions and peak_shadow due to change old config style to new URL style
+- Lineage changes for events, event_basics, event_positions and peak_shadow due to change old config style to new URL style
 
 
 1.3.0 / 2022-02-26

bin/bootstrax

@@ -1304,6 +1304,7 @@ def run_strax(run_id, input_dir, targets, readout_threads, compressor,
                         st.make(run_id,
                                 post_target,
                                 config=strax_config,
+                                progress_bar=True,
                                 max_workers=cores)
                     else:
                         log.info(f'Not making {post_target}, it is already stored')

bin/straxer

@@ -175,8 +175,8 @@ def setup_context(args):
             st.storage += [strax.DataDirectory(args.add_folder)]
 
     if st.is_stored(args.run_id, args.target):
-        logging.info("This data is already available.")
-        return 1
+        logging.warning("This data is already available. Straxer is done")
+        sys.exit(0)
     return st
 
 

docs/source/build_datastructure_doc.py

@@ -321,7 +321,7 @@ def get_plugin(pov):
     for data_kind in tree.keys():
         graph_tree.node(data_kind,
                         style='filled',
-                        href='#' + data_kind.replace('_', '-'),
+                        href='#' + data_kind.replace('_', '-') + '-data-kind',
                         fillcolor=kind_colors.get(data_kind, 'grey'),
                         shape='box3d',
                         )
@@ -344,15 +344,15 @@ def get_plugin(pov):
         graph_tree.attr(rankdir='LR')
         graph_tree.node(data_kind + '-data-kind',
                         style='filled',
-                        href='#' + data_kind.replace('_', '-'),
+                        href='#' + data_kind.replace('_', '-') + '-data-kind',
                         fillcolor=kind_colors.get(data_kind, 'grey'),
                         shape='box3d',
                         )
 
         for dtype in data_types:
             graph_tree.node(dtype,
                             style='filled',
-                            href='#' + data_kind.replace('_', '-'),
+                            href='#' + data_kind.replace('_', '-') + '-data-kind',
                             fillcolor=kind_colors.get(data_kind, 'grey'),
                             )
             graph_tree.edge(data_kind + '-data-kind', dtype)

docs/source/build_release_notes.py

@@ -29,7 +29,7 @@ def convert_release_notes():
                     pr_number = pr_number[:-1]
             if pr_number:
                 line = line.replace(f'#{pr_number}',
-                                    f'`#{pr_number} <https://github.com/AxFoundation/strax/pull/>`_'
+                                    f'`#{pr_number} <https://github.com/XENONnT/straxen/pull/{pr_number}>`_'
                                     )
         with_ref += line + '\n'
     target = os.path.join(this_dir, 'reference', 'release_notes.rst')

docs/source/online_monitor.rst

@@ -17,7 +17,7 @@ and start developing live-displays!
 .. code-block:: python
 
     import straxen
-    st = straxen.contexts.xenonnt_online(_add_online_monitor_frontend=True)
+    st = straxen.contexts.xenonnt_online(include_online_monitor=True)
 
     # Allow unfinished runs to be loaded, even before the DAQ has finished processing this run!
     st.set_context_config({'allow_incomplete': True})
@@ -65,6 +65,8 @@ At the time of writing, these were:
   - ``online_peak_monitor``
   - ``event_basics``
   - ``veto_regions``
+  - ``online_monitor_nv``
+  - ``online_monitor_mv``
 
 For the most up-to-date information, one can check the registration in the
 ``straxen.contexts.xenonnt_online`` context:
@@ -84,7 +86,7 @@ a few bigger files which makes it much faster to load next time.
 .. code-block:: python
 
     import straxen
-    st = straxen.contexts.xenonnt_online(_add_online_monitor_frontend=True)
+    st = straxen.contexts.xenonnt_online(include_online_monitor=True)
     st.copy_to_frontend(latest_run_id, 'event_basics', rechunk=True)
 
 One can look now where this run is stored:

extra_requirements/requirements-docs.txt

@@ -6,3 +6,4 @@ nbsphinx==0.8.8
 recommonmark==0.7.1
 sphinx==4.4.0
 sphinx_rtd_theme==1.0.0
+Jinja2==3.0.3

extra_requirements/requirements-tests.txt

@@ -14,6 +14,7 @@ holoviews==1.14.8
 hypothesis==6.38.0
 ipywidgets==7.6.5
 jupyter-client==7.1.2                           # for ipywidgets
+Jinja2==3.0.3                                   # Holoviews dependency
 keras==2.8.0                                    # Tensorflow dependency
 matplotlib==3.5.1
 multihist==0.6.5

setup.py

@@ -21,7 +21,7 @@ def open_requirements(path):
     history = file.read()
 
 setuptools.setup(name='straxen',
-                 version='1.4.0',
+                 version='1.6.0',
                  description='Streaming analysis for XENON',
                  author='Straxen contributors, the XENON collaboration',
                  url='https://github.com/XENONnT/straxen',

straxen/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.4.0'
+__version__ = '1.6.0'
 
 from utilix import uconfig
 from .common import *

straxen/contexts.py

@@ -81,10 +81,14 @@
                                            straxen.PeaksHighEnergy,
                                            straxen.PeakletsHighEnergy,
                                            straxen.PeakletClassificationHighEnergy,
+                                           straxen.BayesPeakClassification,
                                            straxen.MergedS2sHighEnergy,
                                            straxen.EventInfo,
                                            straxen.PeakShadow,
                                            straxen.EventShadow,
+                                           straxen.EventwBayesClass,
+                                           straxen.PeakAmbience,
+                                           straxen.EventAmbience,
                                            ],
     'register_all': common_opts['register_all'] + [straxen.nveto_recorder,
                                                    straxen.veto_pulse_processing,

straxen/corrections_services.py

@@ -15,11 +15,13 @@
 corrections_w_file = ['mlp_model', 'cnn_model', 'gcn_model',
                       's2_xy_map_mlp', 's2_xy_map_cnn', 's2_xy_map_gcn', 's2_xy_map',
                       's1_xyz_map_mlp', 's1_xyz_map_cnn', 's1_xyz_map_gcn',
-                      'fdc_map_mlp', 'fdc_map_cnn', 'fdc_map_gcn', 's1_aft_xyz_map']
+                      'fdc_map_mlp', 'fdc_map_cnn', 'fdc_map_gcn', 's1_aft_xyz_map',
+                      'bayes_model']
 
 single_value_corrections = ['elife_xenon1t', 'elife', 'baseline_samples_nv',
                             'electron_drift_velocity', 'electron_drift_time_gate',
-                            'se_gain', 'rel_extraction_eff', 'relative_light_yield']
+                            'se_gain', 'rel_extraction_eff', 'relative_light_yield',
+                            'avg_se_gain']
 
 arrays_corrections = ['hit_thresholds_tpc', 'hit_thresholds_he',
                       'hit_thresholds_nv', 'hit_thresholds_mv']

straxen/plugins/__init__.py

@@ -10,6 +10,9 @@
 from . import peaklet_processing
 from .peaklet_processing import *
 
+from . import bayes_peak_classification
+from .bayes_peak_classification import *
+
 from . import peak_processing
 from .peak_processing import *
 
@@ -28,6 +31,9 @@
 from . import event_processing
 from .event_processing import *
 
+from .event_w_bayes_class import *
+from . import event_w_bayes_class
+
 from . import afterpulse_processing
 from .afterpulse_processing import *
 

straxen/plugins/acqmon_processing.py

@@ -41,7 +41,7 @@ class AqmonHits(strax.Plugin):
     GPS SYNC analysis, etc.
     """
     save_when = strax.SaveWhen.TARGET
-    __version__ = '1.1.0'
+    __version__ = '1.1.1'
     hit_min_amplitude_aqmon = straxen.URLConfig(
         default=(
             # Analogue signals
@@ -70,6 +70,12 @@ class AqmonHits(strax.Plugin):
         track=True,
         help='Number of samples to use at the start of the pulse to determine the baseline'
     )
+    check_raw_record_aqmon_overlaps = straxen.URLConfig(
+        default=True,
+        track=False,
+        help='Crash if any of the pulses in raw_records_aqmon overlap with others '
+             'in the same channel'
+    )
 
     depends_on = 'raw_records_aqmon'
     provides = 'aqmon_hits'
@@ -83,11 +89,17 @@ def compute(self, raw_records_aqmon):
         if not_allowed_channels:
             raise ValueError(
                 f'Unknown channel {not_allowed_channels}. Only know {self.aqmon_channels}')
+
+        if self.check_raw_record_aqmon_overlaps:
+            straxen.check_overlaps(raw_records_aqmon,
+                                   n_channels = max(AqmonChannels).value + 1
+            )
+
         records = strax.raw_to_records(raw_records_aqmon)
-        strax.sort_by_time(records)
         strax.zero_out_of_bounds(records)
         strax.baseline(records, baseline_samples=self.baseline_samples_aqmon, flip=True)
         aqmon_hits = self.find_aqmon_hits_per_channel(records)
+        aqmon_hits = strax.sort_by_time(aqmon_hits)
         return aqmon_hits
 
     @property
@@ -109,7 +121,6 @@ def find_aqmon_hits_per_channel(self, records):
         if np.sum(is_artificial):
             aqmon_hits = np.concatenate([
                 aqmon_hits, self.get_deadtime_hits(records[is_artificial])])
-            strax.sort_by_time(aqmon_hits)
         return aqmon_hits
 
     def get_deadtime_hits(self, artificial_deadtime):
@@ -452,7 +463,7 @@ class DetectorSynchronization(strax.Plugin):
     Reference:
         * xenon:xenonnt:dsg:mveto:sync_monitor
     """
-    __version__ = '0.0.1'
+    __version__ = '0.0.2'
     depends_on = ('raw_records_aqmon',
                   'raw_records_aqmon_nv',
                   'raw_records_aux_mv')
@@ -580,6 +591,7 @@ def estimate_delay(self, hits_det0, hits_det1):
             else:
                 # Add err_value in case offset is not valid
                 offsets.append(err_value)
+                prev_time = hits_det0['time'][ind]
 
         return np.array(offsets)