From ae7ac1013702ca90789e35cb7094640630ca3998 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 9 Jul 2024 17:12:41 +0200 Subject: [PATCH 1/6] close #576 --- src/fabio/app/hdf2neggia.py | 141 ++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100755 src/fabio/app/hdf2neggia.py diff --git a/src/fabio/app/hdf2neggia.py b/src/fabio/app/hdf2neggia.py new file mode 100755 index 00000000..e673999a --- /dev/null +++ b/src/fabio/app/hdf2neggia.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +__date__ = "09/06/2024" +__author__ = "Jerome Kieffer" +__license__ = "MIT" + +import os +import sys +import argparse +import logging +logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) +logging.basicConfig() +import numpy +import fabio +from fabio.nexus import Nexus +import h5py +import hdf5plugin +try: + from pyFAI import load +except ImportError: + pyFAI = None + logger.error("Unable to import pyFAI, won't be able to parse PONI-file!") + + +def parse(argv=None): + if argv is None: + argv = [] + parser = argparse.ArgumentParser(prog='lima2neggia', + description='Convert any HDF5 file containing images to a file processable by XDS using the neggia plugin from Dectris.' + 'Do not forget to specify LIB=/path/to/plugin/dectris-neggia.so in XDS.INP', + epilog='Requires hdf5plugin and pyFAI to parse configuration file. Geometry can be calibrated with pyFAI-calib2') + parser.add_argument("input", nargs='+', help="Space separated list of input files (HDF5)") + parser.add_argument("--verbose", "-v", help="increase output verbosity", + action="count") + parser.add_argument("--force", "-f", help="force overwrite output file", + action="store_true") + parser.add_argument("--copy", "-c", help="copy dataset instead of using external links", + action="store_true") + parser.add_argument("--geometry", "-g", help="PONI-file containing the geometry (pyFAI format") + parser.add_argument("--output", "-o", help="output filename", default="master.h5") + return parser.parse_args() + +def process(options): + if options.verbose: + if option.verbose>1: + logger.setLevel(logging.debug) + else: + logger.setLevel(logging.info) + + if not options.geometry or not os.path.exists(options.geometry): + logger.error(f"Unable to parse PONI-file: {options.geometry}") + return 1 + try: + poni = load(options.geometry) + except Exception as err: + logger.error(f"Unable to parse PONI-file: {options.geometry}") + raise err + return 1 + f2d = poni.getFit2D() + frames = [fabio.open(i) for i in options.input] + if os.path.exists(options.output): + if options.force: + mode = "w" + else: + logger.error("Output file exist, not overwriting it. Aborting") + return 1 + else: + mode = "w" + + dest_dir = os.path.dirname(os.path.abspath(options.output)) + with Nexus(options.output, mode) as nxs: + entry = nxs.new_entry(entry="entry", program_name="lima2neggia", force_name=True) + instrument = nxs.new_instrument(entry=entry, instrument_name="instrument") + if poni.wavelength: + beam = nxs.new_class(instrument, "beam", "NXbeam") + beam.create_dataset("incident_wavelength", data=poni.wavelength*1e10).attrs["unit"] = "A" + detector = nxs.new_class(instrument, "detector", "NXdetector") + detector.create_dataset("x_pixel_size", data=float(poni.pixel2)).attrs["unit"] = "m" + detector.create_dataset("y_pixel_size", data=float(poni.pixel1)).attrs["unit"] = "m" + detector.create_dataset("beam_center_x", data=float(f2d["centerX"])).attrs["unit"] = "pixel" + detector.create_dataset("beam_center_y", data=float(f2d["centerY"])).attrs["unit"] = "pixel" + detectorSpecific = nxs.new_class(detector, "detectorSpecific", "NXcollection") + detectorSpecific.create_dataset("nimages", data=sum(i.nframes for i in frames)) + detectorSpecific.create_dataset("ntrigger", data=1) + mask = poni.detector.mask + if mask is None: + mask = numpy.zeros(poni.detector.shape, dtype="uint32") + else: + mask = mask.astype("uint32") + detectorSpecific.create_dataset("pixel_mask", data=mask) + data = nxs.new_class(entry, "data", "NXdata") + cnt = 0 + for fimg in frames: + if isinstance(fimg.dataset, h5py.Dataset): + cnt += 1 + if options.copy: + data.create_dataset(f"data_{cnt:06d}", data=fimg.dataset[()], + chunks=(1,)+fimg.shape, + **hdf5plugin.Bitshuffle(nelems=0, cname='lz4')) + else: + data[f"data_{cnt:06d}"] = h5py.ExternalLink(os.path.relpath(fimg.dataset.file.filename, dest_dir), fimg.dataset.name) + elif isinstance(fimg.dataset, numpy.ndarray): + cnt += 1 + if fimg.dataset.ndim < 3: + dataset = numpy.atleast_3d(fimg.dataset) + dataset.shape = (1,)*(3-fimg.dataset.ndim)+fimg.dataset.shape + else: + dataset = fimg.dataset + data.create_dataset(f"data_{cnt:06d}", data=dataset, + chunks=(1,)+fimg.shape, + **hdf5plugin.Bitshuffle(nelems=0, cname='lz4')) + else: # assume it is a list + for item in fimg.dataset: + # each item can be either a dataset or a numpy array + if isinstance(item, h5py.Dataset): + cnt += 1 + if options.copy: + data.create_dataset(f"data_{cnt:06d}", data=item[()], + chunks=(1,)+fimg.shape, + **hdf5plugin.Bitshuffle(nelems=0, cname='lz4')) + else: + data[f"data_{cnt:06d}"] = h5py.ExternalLink(os.path.relpath(item.file.filename, dest_dir), item.name) + + elif isinstance(fimg.dataset, numpy.ndarray): + cnt += 1 + if fimg.dataset.ndim < 3: + dataset = numpy.atleast_3d(item) + dataset.shape = (1,)*(3-item.ndim)+item.shape + else: + dataset = item + data.create_dataset(f"data_{cnt:06d}", data=dataset, + chunks=(1,)+fimg.shape, + **hdf5plugin.Bitshuffle(nelems=0, cname='lz4')) + else: + logger.warning("Don't know how to handle {item}, skipping") + return 0 + +if __name__=="__main__": + options = parse(sys.argv) + rc = process(options) + sys.exit(rc) From 855a929b1ae0fab4cc77d91b2aac70d1143e9862 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 9 Jul 2024 17:15:01 +0200 Subject: [PATCH 2/6] some more integration --- pyproject.toml | 1 + src/fabio/app/hdf2neggia.py | 21 +++++++++++++-------- src/fabio/app/meson.build | 1 + 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf090e8a..d55cddb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ densify_Bragg = 'fabio.app.densify:main' eiger2cbf = 'fabio.app.eiger2cbf:main' eiger2crysalis = 'fabio.app.eiger2crysalis:main' fabio-convert = 'fabio.app.convert:main' +hdf2neggia = 'fabio.app.hdf2neggia:main' [project.gui-scripts] fabio_viewer = 'fabio.app.viewer:main' diff --git a/src/fabio/app/hdf2neggia.py b/src/fabio/app/hdf2neggia.py index e673999a..8287103f 100755 --- a/src/fabio/app/hdf2neggia.py +++ b/src/fabio/app/hdf2neggia.py @@ -8,7 +8,8 @@ import sys import argparse import logging -logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) +application_name = os.path.splitext(os.path.basename(__file__))[0] +logger = logging.getLogger(application_name) logging.basicConfig() import numpy import fabio @@ -25,7 +26,7 @@ def parse(argv=None): if argv is None: argv = [] - parser = argparse.ArgumentParser(prog='lima2neggia', + parser = argparse.ArgumentParser(prog=application_name, description='Convert any HDF5 file containing images to a file processable by XDS using the neggia plugin from Dectris.' 'Do not forget to specify LIB=/path/to/plugin/dectris-neggia.so in XDS.INP', epilog='Requires hdf5plugin and pyFAI to parse configuration file. Geometry can be calibrated with pyFAI-calib2') @@ -36,9 +37,9 @@ def parse(argv=None): action="store_true") parser.add_argument("--copy", "-c", help="copy dataset instead of using external links", action="store_true") - parser.add_argument("--geometry", "-g", help="PONI-file containing the geometry (pyFAI format") + parser.add_argument("--geometry", "-g", help="PONI-file containing the geometry (pyFAI format)") parser.add_argument("--output", "-o", help="output filename", default="master.h5") - return parser.parse_args() + return parser.parse_args(argv) def process(options): if options.verbose: @@ -69,7 +70,7 @@ def process(options): dest_dir = os.path.dirname(os.path.abspath(options.output)) with Nexus(options.output, mode) as nxs: - entry = nxs.new_entry(entry="entry", program_name="lima2neggia", force_name=True) + entry = nxs.new_entry(entry="entry", program_name=application_name, force_name=True) instrument = nxs.new_instrument(entry=entry, instrument_name="instrument") if poni.wavelength: beam = nxs.new_class(instrument, "beam", "NXbeam") @@ -135,7 +136,11 @@ def process(options): logger.warning("Don't know how to handle {item}, skipping") return 0 -if __name__=="__main__": + +def main(): options = parse(sys.argv) - rc = process(options) - sys.exit(rc) + return process(options) + + +if __name__=="__main__": + sys.exit(main()) diff --git a/src/fabio/app/meson.build b/src/fabio/app/meson.build index d7b5429d..a4ad94fd 100644 --- a/src/fabio/app/meson.build +++ b/src/fabio/app/meson.build @@ -8,6 +8,7 @@ py.install_sources( 'viewer.py', 'densify.py', 'eiger2crysalis.py', + 'hdf2neggia.py' ] , pure: false, # Will be installed next to binaries From d2f116795ab95dfefc0087ba7e51144c195756f9 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 9 Jul 2024 17:49:44 +0200 Subject: [PATCH 3/6] use lasy formating --- src/fabio/app/hdf2neggia.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fabio/app/hdf2neggia.py b/src/fabio/app/hdf2neggia.py index 8287103f..4d16f431 100755 --- a/src/fabio/app/hdf2neggia.py +++ b/src/fabio/app/hdf2neggia.py @@ -43,18 +43,18 @@ def parse(argv=None): def process(options): if options.verbose: - if option.verbose>1: + if options.verbose>1: logger.setLevel(logging.debug) else: logger.setLevel(logging.info) if not options.geometry or not os.path.exists(options.geometry): - logger.error(f"Unable to parse PONI-file: {options.geometry}") + logger.error("Unable to parse PONI-file: %s", options.geometry) return 1 try: poni = load(options.geometry) except Exception as err: - logger.error(f"Unable to parse PONI-file: {options.geometry}") + logger.error("Unable to parse PONI-file: %s", options.geometry) raise err return 1 f2d = poni.getFit2D() @@ -133,7 +133,7 @@ def process(options): chunks=(1,)+fimg.shape, **hdf5plugin.Bitshuffle(nelems=0, cname='lz4')) else: - logger.warning("Don't know how to handle {item}, skipping") + logger.warning("Don't know how to handle %s, skipping", item) return 0 From d2edd465f2af45cc688fc4d663be77731fd82443 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 9 Jul 2024 18:04:04 +0200 Subject: [PATCH 4/6] install hdf5plugin as well --- ci/requirements_gh.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements_gh.txt b/ci/requirements_gh.txt index f8c33636..55f93a01 100644 --- a/ci/requirements_gh.txt +++ b/ci/requirements_gh.txt @@ -14,3 +14,4 @@ Pillow lxml>=4.6.3 pyqt5 matplotlib +hdf5plugin From 6881866034405b760e674984add76d4d7057498d Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 10 Jul 2024 09:16:02 +0200 Subject: [PATCH 5/6] typo --- src/fabio/app/hdf2neggia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabio/app/hdf2neggia.py b/src/fabio/app/hdf2neggia.py index 4d16f431..a0276174 100755 --- a/src/fabio/app/hdf2neggia.py +++ b/src/fabio/app/hdf2neggia.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -__date__ = "09/06/2024" +__date__ = "10/07/2024" __author__ = "Jerome Kieffer" __license__ = "MIT" @@ -138,7 +138,7 @@ def process(options): def main(): - options = parse(sys.argv) + options = parse(sys.argv[1:]) return process(options) From 8258e206229dbe0a5a439eaca0f44db36dbfd39b Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 10 Jul 2024 14:31:39 +0200 Subject: [PATCH 6/6] provide also the distance --- src/fabio/app/hdf2neggia.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fabio/app/hdf2neggia.py b/src/fabio/app/hdf2neggia.py index a0276174..366f52b0 100755 --- a/src/fabio/app/hdf2neggia.py +++ b/src/fabio/app/hdf2neggia.py @@ -37,7 +37,7 @@ def parse(argv=None): action="store_true") parser.add_argument("--copy", "-c", help="copy dataset instead of using external links", action="store_true") - parser.add_argument("--geometry", "-g", help="PONI-file containing the geometry (pyFAI format)") + parser.add_argument("--geometry", "-g", help="PONI-file containing the geometry (pyFAI format, MANDATORY)") parser.add_argument("--output", "-o", help="output filename", default="master.h5") return parser.parse_args(argv) @@ -58,6 +58,10 @@ def process(options): raise err return 1 f2d = poni.getFit2D() + + if len(options.input)==0: + logger.error("No input HDF5 file provided. Aborting") + return 1 frames = [fabio.open(i) for i in options.input] if os.path.exists(options.output): if options.force: @@ -80,6 +84,7 @@ def process(options): detector.create_dataset("y_pixel_size", data=float(poni.pixel1)).attrs["unit"] = "m" detector.create_dataset("beam_center_x", data=float(f2d["centerX"])).attrs["unit"] = "pixel" detector.create_dataset("beam_center_y", data=float(f2d["centerY"])).attrs["unit"] = "pixel" + detector.create_dataset("detector_distance", data=f2d["directDist"]*1e-3).attrs["unit"] = "m" detectorSpecific = nxs.new_class(detector, "detectorSpecific", "NXcollection") detectorSpecific.create_dataset("nimages", data=sum(i.nframes for i in frames)) detectorSpecific.create_dataset("ntrigger", data=1)