diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index aab1eb7..8da9f74 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,13 +13,9 @@ on: - cron: 0 0 * * * # Scheduled run every day at midnight jobs: build: - - runs-on: ${{ matrix.platform }} - strategy: - fail-fast: false - matrix: - platform: [ubuntu-latest] - python-version: [3.9] + runs-on: ubuntu-latest + container: + image: public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4df21dd..f7bf4ff 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -35,12 +35,55 @@ jobs: pip install spacepy --no-build-isolation pip install -e .[test] if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }} - - name: Run tests + + - name: Install CDF library on Ubuntu + run: | + wget https://sdc-aws-support.s3.amazonaws.com/cdf-binaries/latest.zip + unzip latest.zip + echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV + if: ${{(matrix.platform == 'ubuntu-latest')}} + + - name: Install CDF library on MacOS + run: | + wget https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/macosx/CDF3_9_0-binary-signed.pkg + sudo installer -pkg CDF3_9_0-binary-signed.pkg -target / + ls -l + echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV + if: ${{(matrix.platform == 'macos-latest')}} + + # Set up CDF and run tests on Windows + - name: Install CDF on Windows + shell: cmd + run: | + # Download and unzip CDF + curl -L "https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/latest/windows/cdf3.9.0_64bit_WinZip_Installer.zip" --output cdf.zip + mkdir cdf_lib + tar -xf cdf.zip -C cdf_lib + + # Set environment variables for CDF + set mydir=%cd% + set CDF_BASE=%mydir%\cdf_lib + set CDF_INC=%mydir%\cdf_lib\include + set CDF_LIB=%mydir%\cdf_lib\lib + set CDF_HELP=%mydir%\cdf_lib\help + set CDF_LEAPSECONDSTABLE=%mydir%\cdf_lib\CDFLeapSeconds.txt + set CLASSPATH=%mydir%\cdf_lib\CDFToolsDriver.jar;%mydir%\cdf_lib\lib\cdfjava.jar;%mydir%\cdf_lib\lib\cdfml.jar;%mydir%\cdf_lib\lib\cdfjson.jar;%mydir%\cdf_lib\lib\cdfj.jar;%mydir%\cdf_lib\lib\gson-2.8.6.jar;%mydir%\cdf_lib\lib\javax.json-1.0.4.jar;. + set PATH=%mydir%\cdf_lib;%mydir%\cdf_lib\bin;%PATH% + set TERMINFO=%mydir%\cdf_lib\lib\terminfo + set JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8" + + # Run tests + pytest --pyargs hermes_eea --cov hermes_eea + # Skip Windows Python 3.11 tests until SpacePy is updated + if: ${{(matrix.platform == 'windows-latest' && matrix.python-version != '3.11')}} + + - name: Run tests on Ubuntu and MacOS run: pytest --pyargs hermes_eea --cov hermes_eea env: PLATFORM: ${{ matrix.platform }} # Skip Windows Python 3.11 tests until SpacePy is updated - if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }} + if: ${{ !(matrix.platform == 'windows-latest') }} + - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 # Skip Windows Python 3.11 tests until SpacePy is updated diff --git a/.readthedocs.yml b/.readthedocs.yml index ccfcd98..f26b2be 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,6 @@ # Read the Docs configuration file for Sphinx projects # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details -# Required version: 2 # Set the OS, Python version and other tools you might need @@ -21,4 +20,4 @@ build: # Optionally build your docs in additional formats such as PDF and ePub formats: - pdf - - epub \ No newline at end of file + - epub diff --git a/docs/conf.py b/docs/conf.py index 1017a8e..8f98f8a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,6 +8,10 @@ import os import sys +# -- Environmental Variables ------------------------------------------------ +# Set CDF Library Path +os.environ["CDF_LIB"] = "../cdf/lib" + sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- diff --git a/docs/whatsnew/changelog.rst b/docs/whatsnew/changelog.rst index a3678c4..9e8a37b 100644 --- a/docs/whatsnew/changelog.rst +++ b/docs/whatsnew/changelog.rst @@ -4,7 +4,5 @@ Full Changelog ************** -.. changelog:: - :towncrier: ../../ - :towncrier-skip-if-empty: - :changelog_file: ../../CHANGELOG.rst +.. include:: ../../CHANGELOG.rst + diff --git a/hermes_eea/SkymapFactory.py b/hermes_eea/SkymapFactory.py new file mode 100644 index 0000000..2fc512a --- /dev/null +++ b/hermes_eea/SkymapFactory.py @@ -0,0 +1,122 @@ +import numpy as np +from hermes_core import log +from hermes_eea.io import EEA +from hermes_eea.util.time import ccsds_to_cdf_time +from hermes_eea import energies as voltages + +N_ENERGIES = 41 +N_DEFLECTIONS = 4 +N_AZIMUTH = 32 + + +# This may eventually be handled in a python multiprocessor module instance: +def SkymapFactory(l0_cdf, energies, deflections, myEEA): + # ['Epoch', 'Epoch_plus_var', 'Epoch_minus_var', 'hermes_eea_step_counter', + # 'hermes_eea_counter1', 'hermes_eea_counter2', 'hermes_eea_accumulations', + # 'hermes_eea_sector_index', 'hermes_eea_sector_label']) + + # science_data: + start_of_good_data = np.where(l0_cdf["SHEID"][:] == 1)[0][0] + integrates_at_end = np.where( + l0_cdf["SHEID"][start_of_good_data:] == 0 + ) # has 63 values + # We are expecting integrates to be only at the beginning + + # The Science Data: + stepper_table_packets = (np.where(l0_cdf["SHEID"][:] > 0))[0] + return_package = {} + beginning_packets = ( + np.where((l0_cdf["STEP"][stepper_table_packets[0] :]) == 0)[0] + + stepper_table_packets[0] + ) + package = [] + + epochs = ccsds_to_cdf_time.helpConvertEEA(l0_cdf) + try: + for ptr in range(0, len(beginning_packets)): + package.append( + ( + l0_cdf["STEP"][beginning_packets[ptr] : beginning_packets[ptr + 1]], + l0_cdf["ACCUM"][ + beginning_packets[ptr] : beginning_packets[ptr + 1] + ], + l0_cdf["COUNTER1"][ + beginning_packets[ptr] : beginning_packets[ptr + 1] + ], + l0_cdf["COUNTER2"][ + beginning_packets[ptr] : beginning_packets[ptr + 1] + ], + epochs[beginning_packets[ptr] : beginning_packets[ptr + 1]], + energies, + deflections, + ptr, + ) + ) + except IndexError: + log.info("Finished last interval") + + result = [] + for pckt in package: + packet_contents = do_eea_packet(*pckt) + if packet_contents is not None: + result.append(packet_contents) + myEEA.populate(myEEA, result) + + +def do_eea_packet( + stepperTableCounter, counts, cnt1, cnt2, epoch, energies, deflections, ith_FSmap +): + """ + This function populates each sweep, or pass through + all of the energies and deflections designated by the stepper table + + Parameters + ---------- + stepperTableCounter - n_deflections * n_energies + counts - the structured arrays returned by CCSDSPY + cnt1 - the sum of this sweep's accum + cnt2 - same as above but +1...not clear yet + epoch - CDF Formatted time for every single measurement, [0] is the time for the sweep/packet + energies - so far we only have one stepper table with 41 energies + deflections - ...and 4 deflections + ith_FSmap - sweep counter + + Returns + ------- + + """ + return_package = {} + rows = len(stepperTableCounter) + # skymap is already full of zeros, why do it again? + # skymap = np.zeros((beginning_packets[ptr+1]-beginning_packets[ptr],N_AZIMUTH)) + skymaps = [] + pulse_a = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16) + pulse_b = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16) + counter1 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16) + counter2 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16) + usec = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16) + + skymap = np.zeros((N_ENERGIES, N_DEFLECTIONS, N_AZIMUTH), dtype=np.uint16) + + for row in stepperTableCounter: + dim0 = energies[row] + dim1 = deflections[row] + skymap[dim0, dim1, :] = counts[row, 0:N_AZIMUTH] + pulse_a[dim0, dim1] = counts[row][N_AZIMUTH] + pulse_b[dim0, dim1] = counts[row][N_AZIMUTH + 1] + counter1[dim0, dim1] = cnt1[row] + counter2[dim0, dim1] = cnt2[row] + usec[dim0, dim1] = epoch[row] + + return_package["pulse_a"] = pulse_a + return_package["pulse_b"] = list(pulse_b) + return_package["counts"] = skymap + return_package["usec"] = usec + return_package["Epoch"] = epoch[0] + return_package["stats"] = np.sum(skymap) + return_package["energies"] = voltages + return_package["sun_angles"] = deflections + return_package["counter1"] = counter1 + return_package["counter2"] = counter2 + + return return_package diff --git a/hermes_eea/__init__.py b/hermes_eea/__init__.py index 700d8c0..21b6e18 100644 --- a/hermes_eea/__init__.py +++ b/hermes_eea/__init__.py @@ -1,6 +1,8 @@ # Licensed under Apache License v2 - see LICENSE.rst import os.path +import sys +sys.path.append(os.getcwd()) from hermes_core import log from hermes_eea.io.file_tools import read_file @@ -21,5 +23,54 @@ _package_directory = os.path.dirname(os.path.abspath(__file__)) _data_directory = os.path.abspath(os.path.join(_package_directory, "data")) +_calibration_directory = os.path.abspath(os.path.join(_data_directory, "calibration")) + log.info(f"hermes_eea version: {__version__}") + +stepper_table = "flight_stepper.txt" + + +energies = [ + 2.18000000e00, + 2.63177330e00, + 3.17717004e00, + 3.83559233e00, + 4.63046306e00, + 5.59005918e00, + 6.74851766e00, + 8.14704980e00, + 9.83540739e00, + 1.18736525e01, + 1.43342944e01, + 1.73048684e01, + 2.08910507e01, + 2.52204172e01, + 3.04469818e01, + 3.67566761e01, + 4.43739626e01, + 5.35698211e01, + 6.46713874e01, + 7.80735920e01, + 9.42532085e01, + 1.13785815e02, + 1.37366271e02, + 1.65833433e02, + 2.00200000e02, + 2.39800000e02, + 3.17794829e02, + 4.21157437e02, + 5.58138682e02, + 7.39673007e02, + 9.80251281e02, + 1.29907752e03, + 1.72160182e03, + 2.28155195e03, + 3.02362557e03, + 4.00705827e03, + 5.31035195e03, + 7.03754125e03, + 9.32649800e03, + 1.23599368e04, + 1.63800000e04, +] diff --git a/hermes_eea/calibration/build_spectra.py b/hermes_eea/calibration/build_spectra.py new file mode 100644 index 0000000..816666d --- /dev/null +++ b/hermes_eea/calibration/build_spectra.py @@ -0,0 +1,108 @@ +from hermes_core.timedata import HermesData +import astropy.units as astropy_units +from astropy.timeseries import TimeSeries +from astropy.time import Time +from hermes_core.timedata import HermesData +from astropy.nddata import NDData +from ndcube import NDCube, NDCollection +import numpy as np +from astropy.wcs import WCS +from spacepy.pycdf import lib + + +class Hermes_EEA_Data_Processor: + """ + This class plays the role of that the Write* classes provide in FPI + It not only handles the populating and writing of the cdf but attibutes are also added here. + """ + + def __init__(self, myEEA): + self.EEA = myEEA + self.raw_counts = astropy_units.def_unit("raw instrument counts") + + def build_HermesData(self): + iso_times = Time([lib.tt2000_to_datetime(e) for e in self.EEA.Epoch[:]]) + ts_1d_uQ = TimeSeries( + time=iso_times, + data={ + "hermes_eea_stats": astropy_units.Quantity( + self.EEA.stats, "gauss", dtype=np.uint16 + ) + }, + ) # this works + self._hermes_eea_spectra() + bare_attrs = HermesData.global_attribute_template("eea", "l1", "1.0.0") + ts_justTime = TimeSeries(time=iso_times) + + self.hermes_eea_data = HermesData( + timeseries=ts_1d_uQ, spectra=self.multiple_spectra, meta=bare_attrs + ) + self.hermes_eea_data.timeseries["hermes_eea_stats"].meta.update( + {"CATDESC": "Sum of skymap for each sweep"} + ) + + def _hermes_eea_spectra(self): + self.multiple_spectra = NDCollection( + [ + ( + "hermes_eea_settle_step_times", + NDCube( + data=np.array(self.EEA.usec), + wcs=WCS(naxis=2), + meta={"CATDESC": "Settle for Each Step"}, + unit="s", + ), + ), + ( + "hermes_eea_energy_profile", + NDCube( + data=np.array(self.EEA.EnergyLabels), + wcs=WCS(naxis=2), + meta={"CATDESC": "Energy Profile"}, + unit="eV", + ), + ), + ( + "hermes_eea_accum", + NDCube( + data=np.array(self.EEA.ACCUM), + wcs=WCS(naxis=3), + meta={"CATDESC": "EEA raw skymap"}, + unit="count", + ), + ), + ( + "hermes_eea_counter1", + NDCube( + data=np.array(self.EEA.Counter1), + wcs=WCS(naxis=2), + meta={ + "CATDESC": "Estimate 1 of the number of counts in this accumulation" + }, + unit=astropy_units.dimensionless_unscaled, + ), + ), + ( + "hermes_eea_counter1", + NDCube( + data=np.array(self.EEA.Counter1), + wcs=WCS(naxis=2), + meta={ + "CATDESC": "Estimate 1 of the number of counts in this accumulation" + }, + unit=astropy_units.dimensionless_unscaled, + ), + ), + ( + "hermes_eea_counter2", + NDCube( + data=np.array(self.EEA.Counter1), + wcs=WCS(naxis=2), + meta={ + "CATDESC": "Estimate 1 of the number of counts in this accumulation" + }, + unit=astropy_units.dimensionless_unscaled, + ), + ), + ] + ) diff --git a/hermes_eea/calibration/calibration.py b/hermes_eea/calibration/calibration.py index 37765dd..b625efd 100644 --- a/hermes_eea/calibration/calibration.py +++ b/hermes_eea/calibration/calibration.py @@ -1,17 +1,24 @@ """ A module for all things calibration. """ +from datetime import datetime, timezone, timedelta import random import os.path from pathlib import Path - +import sys import ccsdspy +import numpy as np from hermes_core import log from hermes_core.util.util import create_science_filename, parse_science_filename - import hermes_eea from hermes_eea.io import read_file +import hermes_eea.calibration as calib +from hermes_eea.io.EEA import EEA +from hermes_eea.SkymapFactory import SkymapFactory +from spacepy import pycdf + +from hermes_eea.calibration.build_spectra import Hermes_EEA_Data_Processor __all__ = [ "process_file", @@ -40,17 +47,17 @@ def process_file(data_filename: Path) -> list: """ log.info(f"Processing file {data_filename}.") output_files = [] - - calibrated_file = calibrate_file(data_filename) - output_files.append(calibrated_file) - # data_plot_files = plot_file(data_filename) - # calib_plot_files = plot_file(calibrated_file) + for filename in data_filename: + calibrated_file = calibrate_file(filename) + output_files.append(calibrated_file) + # data_plot_files = plot_file(data_filename) + # calib_plot_files = plot_file(calibrated_file) # add other tasks below return output_files -def calibrate_file(data_filename: Path) -> Path: +def calibrate_file(data_filename: Path, destination_dir) -> Path: """ Given an input data file, raise it to the next level (e.g. level 0 to level 1, level 1 to quicklook) it and return a new file. @@ -71,6 +78,11 @@ def calibrate_file(data_filename: Path) -> Path: >>> level1_file = calibrate_file('hermes_EEA_l0_2022239-000000_v0.bin') # doctest: +SKIP """ log.info(f"Calibrating file:{data_filename}.") + if not destination_dir.is_dir(): + raise OSError( + "Output directory: " + str(destination_dir) + ". Please create first." + ) + return output_filename = ( data_filename # TODO: for testing, the output filename MUST NOT same as input ) @@ -81,12 +93,9 @@ def calibrate_file(data_filename: Path) -> Path: file_metadata["instrument"] == hermes_eea.INST_NAME and file_metadata["level"] == "l0" ): - # data = parse_l0_sci_packets(data_filename) - data = {} - # test opening the file - with open(data_filename, "r") as fp: - pass - level1_filename = l0_sci_data_to_cdf(data, data_filename) + # because of error handling, no test of data is necessary here. + data = parse_l0_sci_packets(data_filename) + level1_filename = l0_sci_data_to_cdf(data, data_filename, destination_dir) output_filename = level1_filename elif ( file_metadata["instrument"] == hermes_eea.INST_NAME @@ -118,10 +127,8 @@ def calibrate_file(data_filename: Path) -> Path: # create an empty file for testing purposes with open(data_filename.parent / ql_filename, "w"): pass - - # example log messages - log.info(f"Despiking removing {random.randint(0, 10)} spikes") - log.warning(f"Despiking could not remove {random.randint(1, 5)}") + # here + data = parse_l0_sci_packets(data_filename) output_filename = ql_filename else: raise ValueError(f"The file {data_filename} is not recognized.") @@ -152,13 +159,15 @@ def parse_l0_sci_packets(data_filename: Path) -> dict: log.info(f"Parsing packets from file:{data_filename}.") pkt = ccsdspy.FixedLength.from_file( - os.path.join(hermes_eea._data_directory, "EEA_sci_packet_def.csv") + os.path.join(hermes_eea._data_directory, "hermes_EEA_sci_packet_def.csv") ) data = pkt.load(data_filename) return data -def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path: +def l0_sci_data_to_cdf( + data: dict, original_filename: Path, destination_dir: Path +) -> Path: """ Write level 0 eea science data to a level 1 cdf file. @@ -184,26 +193,60 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path: >>> data_packets = calib.parse_l0_sci_packets(data_filename) # doctest: +SKIP >>> cdf_filename = calib.l0_sci_data_to_cdf(data_packets, data_filename) # doctest: +SKIP """ + + # this is transferring name.bin to name.cdf file_metadata = parse_science_filename(original_filename.name) + # coarse = data["SHCOARSE"][idx] + # fine = data["SHFINE"][idx] + # time = convert_packet_time_to_datetime(coarse, fine) cdf_filename = original_filename.parent / create_science_filename( file_metadata["instrument"], file_metadata["time"], "l1", f'1.0.{file_metadata["version"]}', ) + if not cdf_filename.is_file(): + try: + cdf = pycdf.CDF( + str(cdf_filename), + os.path.join( + hermes_eea._data_directory, + "masterSkeletons/hermes_eea_l1_00000000000000_v0.0.0.cdf", + ), + ) + cdf.close() + except FileNotFoundError: + pass + if data: + # cdf = pycdf.CDF(str(cdf_filename)) + # cdf.readonly(False) + + calibration_file = get_calibration_file(hermes_eea.stepper_table) + read_calibration_file(calibration_file) - # create an empty file for testing purposes - with open(cdf_filename, "w"): - pass + myEEA = EEA(file_metadata) + # This populates so doesn't have to return much + SkymapFactory(data, calib.energies, calib.deflections, myEEA) + most_active = np.where(np.array(myEEA.stats) > 150) - return cdf_filename + hermes_eea_factory = Hermes_EEA_Data_Processor(myEEA) + hermes_eea_factory.build_HermesData() + + try: + cdf_path = hermes_eea_factory.hermes_eea_data.save( + str(destination_dir), True + ) + except Exception as e: + log.error(e) + sys.exit(2) + + return cdf_path def get_calibration_file(data_filename: Path, time=None) -> Path: """ Given a time, return the appropriate calibration file. - Parameters ---------- data_filename: str @@ -218,7 +261,7 @@ def get_calibration_file(data_filename: Path, time=None) -> Path: Examples -------- """ - return None + return os.path.join(hermes_eea._calibration_directory, data_filename) def read_calibration_file(calib_filename: Path): @@ -238,4 +281,41 @@ def read_calibration_file(calib_filename: Path): Examples -------- """ - return None + lines = read_file(os.path.join(calib_filename)) + calib.energies = [] + calib.deflections = [] + for line in lines: + calib.energies.append(int(line[8:10], 16)) + calib.deflections.append(int(line[10:12], 16)) + + +def retrieve_canned_attributes(): + input_attrs = { + "DOI": "https://doi.org//", + "Data_level": "L1>Level 1", # NOT AN ISTP ATTR + "Data_version": "0.0.1", + "Descriptor": "EEA>Electron Electrostatic Analyzer", + "Data_product_descriptor": "odpd", + "HTTP_LINK": [ + "https://spdf.gsfc.nasa.gov/istp_guide/istp_guide.html", + "https://spdf.gsfc.nasa.gov/istp_guide/gattributes.html", + "https://spdf.gsfc.nasa.gov/istp_guide/vattributes.html", + ], + "Instrument_mode": "default", # NOT AN ISTP ATTR + "Instrument_type": "Electric Fields (space)", + "LINK_TEXT": ["ISTP Guide", "Global Attrs", "Variable Attrs"], + "LINK_TITLE": ["ISTP Guide", "Global Attrs", "Variable Attrs"], + "MODS": [ + "v0.0.0 - Original version.", + "v1.0.0 - Include trajectory vectors and optics state.", + "v1.1.0 - Update metadata: counts -> flux.", + "v1.2.0 - Added flux error.", + "v1.3.0 - Trajectory vector errors are now deltas.", + ], + "PI_affiliation": "HERMES", + "PI_name": "HERMES SOC", + "TEXT": "Valid Test Case", + "VATTRS": ["stats", "energies"], + } + + return input_attrs diff --git a/hermes_eea/data/calibration/flight_stepper.txt b/hermes_eea/data/calibration/flight_stepper.txt new file mode 100755 index 0000000..8471d1e --- /dev/null +++ b/hermes_eea/data/calibration/flight_stepper.txt @@ -0,0 +1,164 @@ +1014902200001770 +101d902a01001770 +1029903502001770 +1037904103001770 +1047905004001770 +105c906205001770 +1074907806001770 +1092909307001770 +10b590b208001770 +10e190d909001770 +111591070a001770 +115391400b001770 +119f91830c001770 +11fb91d50d001770 +126992380e001770 +12ef92af0f001770 +1390933f10001770 +145293ed11001770 +153d94bf12001770 +165895bc13001770 +17af96ee14001770 +194c985f15001770 +1b3e9a1c16001770 +1d989c3617001770 +002c9ec018001770 +0035803219001770 +004880431a001770 +006180591b001770 +008280761c001770 +00ad809d1d001770 +00e780d11e001770 +013481151f001770 +0199817020001770 +022081e821001770 +02d2828722001770 +03bf835a23001770 +04f8847124001770 +069785e425001770 +08bd87ce26001770 +0b968a5927001770 +0f5c8db728001770 +0f5c848d28012710 +0b96836f27011770 +08bd829726011770 +069781f425011770 +04f8817924011770 +03bf811c23011770 +02d280d622011770 +022080a121011770 +0199807920011770 +0134805b1f011770 +00e780451e011770 +00ad80331d011770 +00829da81c011770 +00619a4c1b011770 +004897c41a011770 +003595da19011770 +002c94e118011770 +1d98940a17011770 +1b3e935716011770 +194c92c315011770 +17af924814011770 +165891e313011770 +153d918f12011770 +1452914911011770 +1390910f10011770 +12ef90df0f011770 +126990b80e011770 +11fb90970d011770 +119f907c0c011770 +115390650b011770 +111590530a011770 +10e1904309011770 +10b5903608011770 +1092902c07011770 +1074902306011770 +105c901c05011770 +1047901604011770 +1037901103011770 +1029900d02011770 +101d900901011770 +1014900600011770 +1014103300022710 +101d103501021770 +1029103802021770 +1037103b03021770 +1047104004021770 +105c104505021770 +1074104b06021770 +1092105307021770 +10b5105c08021770 +10e1106609021770 +111510740a021770 +115310840b021770 +119f10970c021770 +11fb10ae0d021770 +126910ca0e021770 +12ef10ec0f021770 +1390111410021770 +1452114511021770 +153d118112021770 +165811c913021770 +17af121f14021770 +194c128815021770 +1b3e130616021770 +1d98139e17021770 +002c145618021770 +0035152a19021770 +004816cb1a021770 +006118f41b021770 +00821bd01c021770 +00ad1f9b1d021770 +00e700391e021770 +0134004c1f021770 +0199006620021770 +0220008821021770 +02d200b522021770 +03bf00f123021770 +04f8014024021770 +069701aa25021770 +08bd023526021770 +0b9602ee27021770 +0f5c03e328021770 +0f5c0d8328032710 +0b960a3127031770 +08bd07b026031770 +069705cc25031770 +04f8045f24031770 +03bf034c23031770 +02d2027c22031770 +022001df21031770 +0199016920031770 +013401101f031770 +00e700cc1e031770 +00ad00991d031770 +008200731c031770 +006100561b031770 +004800401a031770 +0035003019031770 +002c1eab18031770 +1d981c2d17031770 +1b3e1a1d16031770 +194c186715031770 +17af16fc14031770 +165815d013031770 +153d14d712031770 +1452140911031770 +1390135f10031770 +12ef12d10f031770 +1269125c0e031770 +11fb11fb0d031770 +119f11ab0c031770 +115311680b031770 +111511310a031770 +10e1110309031770 +10b510de08031770 +109210be07031770 +107410a406031770 +105c108f05031770 +1047107d04031770 +1037106e03031770 +1029106202031770 +101d105801031770 +1014104f00032710 \ No newline at end of file diff --git a/hermes_eea/data/hermes_EEA_l0_2023042-000000_v0.bin b/hermes_eea/data/hermes_EEA_l0_2023042-000000_v0.bin new file mode 100644 index 0000000..a7cf7d8 Binary files /dev/null and b/hermes_eea/data/hermes_EEA_l0_2023042-000000_v0.bin differ diff --git a/hermes_eea/data/hermes_EEA_sci_packet_def.csv b/hermes_eea/data/hermes_EEA_sci_packet_def.csv new file mode 100644 index 0000000..dc11ce5 --- /dev/null +++ b/hermes_eea/data/hermes_EEA_sci_packet_def.csv @@ -0,0 +1,10 @@ +name, data_type, bit_length +SHCOARSE, uint, 32 +SHFINE, uint, 20 +SHFILL, fill, 4 +SHEID, uint, 8 +STEP, uint, 8 +COUNTER1, uint, 16 +COUNTER2, uint, 16 +ACCUM, "uint(34)", 12 +CKSUM16, uint, 16 \ No newline at end of file diff --git a/hermes_eea/data/hermes_eea_l0_2023042T000000_v0.bin b/hermes_eea/data/hermes_eea_l0_2023042T000000_v0.bin new file mode 100644 index 0000000..a7cf7d8 Binary files /dev/null and b/hermes_eea/data/hermes_eea_l0_2023042T000000_v0.bin differ diff --git a/hermes_eea/io/EEA.py b/hermes_eea/io/EEA.py new file mode 100644 index 0000000..3d4c8a1 --- /dev/null +++ b/hermes_eea/io/EEA.py @@ -0,0 +1,41 @@ +class EEA: + def __init__(self, conf): + self.Epoch = [] # the first of each sweep, the first of each of the 164 times, + # when hermes_eea_intgr_or_stepper = 1 and hermes_eea_step_counter = 0 + self.Generation_date = None + self.Logical_file_id = None + self.Data_version = None + # ener, defl + self.usec = ( + [] + ) # [ 41, 4 ] each of the 164 times, whenever hermes_eea_intgr_or_stepper == 1 + self.PulseA = [] # [41, 4] overflow[0],accum[33] + self.PulseB = [] # [41, 4] overflow[1], accum[34] + self.Counter1 = [] + self.Counter2 = [] + self.ACCUM = [] # [41, 4, 32]. [ene, defl, accums] + self.SunAngles = [] # [4,32] really just metadata + self.EnergyLabels = [] # [41] really just metadata + self.stats = [] # [41] really just metadata + + def append(self, attrname, record): + try: + return record[attrname] + except KeyError: + # occasionally no value is returned see: compressionLoss is only in moms brst + if self.name_align(attrname) in self.default_obj: + return self.default_obj[self.name_align(attrname)] + + def populate(self, myEEA, skymap): + packet = 0 + for record in skymap: + myEEA.usec.append(record["usec"]) + myEEA.Epoch.append(record["Epoch"]) + myEEA.ACCUM.append(record["counts"]) + myEEA.PulseA.append(record["pulse_a"]) + myEEA.PulseB.append(record["pulse_b"]) + myEEA.SunAngles.append(record["sun_angles"]) + myEEA.EnergyLabels.append(record["energies"]) + myEEA.Counter1.append(record["counter1"]) + myEEA.Counter2.append(record["counter2"]) + myEEA.stats.append(record["stats"]) diff --git a/hermes_eea/io/file_tools.py b/hermes_eea/io/file_tools.py index 3f2f943..f14d2ee 100644 --- a/hermes_eea/io/file_tools.py +++ b/hermes_eea/io/file_tools.py @@ -1,3 +1,7 @@ +from ccsdspy import FixedLength +import numpy as np +import os + """ This module provides a generic file reader. """ @@ -21,4 +25,28 @@ def read_file(data_filename): Examples -------- """ - return None + try: + with open(data_filename) as fh: + return fh.readlines() + except Exception: + raise Exception("Could not find: " + os.path.abspath(data_filename)) + + +def read_ccsds(filename: str, pkt_def: FixedLength): + """ + Read a ccsds packet file. + + Parameters + ---------- + filename: str + A file to read. + + pkt_def: `ccsdspy.FixedLength` + CCSDS packet definition + + Returns + ------- + `OrderedDict` mapping field names to NumPy arrays. + """ + result = pkt_def.load(filename) + return result diff --git a/hermes_eea/tests/test_calibration.py b/hermes_eea/tests/test_calibration.py index 0bd7fcc..93d02a6 100644 --- a/hermes_eea/tests/test_calibration.py +++ b/hermes_eea/tests/test_calibration.py @@ -1,87 +1,25 @@ import pytest import os.path from pathlib import Path - +import tempfile +import ccsdspy import hermes_eea.calibration as calib +from hermes_eea import _data_directory, stepper_table from hermes_core.util.util import create_science_filename, parse_science_filename - -level0_filename = "hermes_EEA_l0_2022339-000000_v0.bin" -level1_filename = "hermes_eea_l1_20221205T000000_v1.0.0.cdf" -ql_filename = "hermes_eea_ql_20221205T000000_v1.0.0.cdf" - - -@pytest.fixture(scope="session") -def level0_file(tmp_path_factory): - fn = tmp_path_factory.mktemp("data") / level0_filename - with open(fn, "w"): - pass - return fn +import sys -@pytest.fixture(scope="session") -def level1_file(tmp_path_factory): - fn = tmp_path_factory.mktemp("data") / level1_filename - with open(fn, "w"): - pass +@pytest.fixture(scope="session") # this is a pytest fixture +def small_level0_file(tmp_path_factory): + fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023042-000000_v0.bin")) return fn -def test_l0_sci_data_to_cdf(level0_file): - """Test that the output filenames are correct and that a file was actually created.""" - data = {} - output_file = calib.l0_sci_data_to_cdf(data, level0_file) - assert output_file.name == level1_filename - assert output_file.is_file() - - -def test_calibrate_file_nofile_error(): - """Test that if file does not exist it produces the correct error. The file needs to be in the correct format.""" - with pytest.raises(FileNotFoundError): - calib.calibrate_file(Path("hermes_EEA_l0_2032339-000000_v0.bin")) - - -def test_process_file_nofile_error(): - """Test that if file does not exist it produces the correct error. The file needs to be in the correct format.""" - with pytest.raises(FileNotFoundError): - calib.process_file(Path("hermes_EEA_l0_2032339-000000_v0.bin")) - - -def test_calibrate_file(level0_file, level1_file): +def test_calibrate_file(small_level0_file): """Test that the output filenames are correct and that a file was actually created.""" - output_file = calib.calibrate_file(level0_file) - assert output_file.name == level1_filename - assert output_file.is_file() - output_file = calib.calibrate_file(level1_file) - assert output_file.name == ql_filename - assert output_file.is_file() - - # with pytest.raises(ValueError) as excinfo: - # calib.calibrate_file("datafile_with_no_calib.cdf") - # assert ( - # str(excinfo.value) - # == "Calibration file for datafile_with_no_calib.cdf not found." - # ) - - -def test_process_file_level0(level0_file): - """Test that the output filenames are correct and that a file was actually created.""" - file_output = calib.process_file(level0_file) - assert len(file_output) == 1 - assert file_output[0].name == level1_filename - assert file_output[0].is_file() - - -def test_process_file_level1(level1_file): - """Test that the output filenames are correct and that a file was actually created.""" - file_output = calib.process_file(level1_file) - assert len(file_output) == 1 - assert file_output[0].name == ql_filename - assert file_output[0].is_file() - - -def test_get_calibration_file(): - assert calib.get_calibration_file("") is None - - -def test_read_calibration_file(): - assert calib.read_calibration_file("calib_file") is None + with tempfile.TemporaryDirectory() as tmpdirname: + output_file = calib.calibrate_file(small_level0_file, Path(tmpdirname)) + assert ( + os.path.basename(output_file) == "hermes_eea_l1_20000101T170901_v1.0.0.cdf" + ) + assert os.path.getsize(output_file) > 200000 diff --git a/hermes_eea/tests/test_file_tools.py b/hermes_eea/tests/test_file_tools.py index b9f814b..37ad58e 100644 --- a/hermes_eea/tests/test_file_tools.py +++ b/hermes_eea/tests/test_file_tools.py @@ -2,4 +2,4 @@ def test_read_file(): - assert read_file("test_file.cdf") is None + assert read_file("./hermes_eea/data/calibration/flight_stepper.txt") is not None diff --git a/hermes_eea/util/time/ccsds_to_cdf_time.py b/hermes_eea/util/time/ccsds_to_cdf_time.py new file mode 100644 index 0000000..fe8e609 --- /dev/null +++ b/hermes_eea/util/time/ccsds_to_cdf_time.py @@ -0,0 +1,95 @@ +"""Utils for the CCSDSPy package.""" + +__author__ = "Richard Strub " + +import numpy as np + + +def helpConvert(decoded): + coarse = np.uint(decoded["START_CORSTIME"]) + fine = np.uint(decoded["START_FINETIME"]) + epoch = converting_ccsds_times_to_cdf(coarse, fine) + return epoch + + +def helpConvertEEA(decoded): + coarse = np.uint(decoded["SHCOARSE"]) + fine = np.uint(decoded["SHFINE"]) + epoch = converting_ccsds_times_to_cdf(coarse, fine) + return epoch + + +def helpConvertMagFld(decoded): + coarse = np.uint(decoded["MAGMSGCOARSETM"]) + fine = np.uint(decoded["MAGMSGFINETIME"]) + epoch = converting_ccsds_times_to_cdf(coarse, fine) + return epoch + + +def converting_ccsds_times_to_cdf(coarse, fine): + epoch = np.zeros(coarse.shape[0], dtype=np.uint) + p1 = np.zeros(coarse.shape[0], dtype=np.uint) + p2 = np.zeros(coarse.shape[0], dtype=np.uint) + + tai_time = {} + # FPI: + # tai_time["taiEpoch_tt2000"] = 1325419167816000000 + # EEA: + tai_time["taiEpoch_tt2000"] = -64184000000 + tai_time["nanosPerMicro"] = 1000 + tai_time["MicrosPerSec"] = 1000000 + tai_time["nanosPerSec"] = 1000000000 + + example = coarse[0] * tai_time["nanosPerSec"] + p1 = np.int64(coarse) * np.int64(tai_time["nanosPerSec"]) + p2 = np.int64(fine) * np.int64(tai_time["nanosPerMicro"]) + epoch = p1 + p2 + result = np.uint64(epoch - tai_time["taiEpoch_tt2000"]) + return result + + +def reverse_cnv_cdf_times_to_ccsds(epoch): + tai_time = {} + # FPI: + # tai_time["taiEpoch_tt2000"] = 1325419167816000000 + # EEA: + tai_time["taiEpoch_tt2000"] = -64184000000 + tai_time["nanosPerMicro"] = 1000 + tai_time["MicrosPerSec"] = 1000000 + tai_time["nanosPerSec"] = 1000000000 + TAI_us = int((epoch + tai_time["taiEpoch_tt2000"]) / tai_time["nanosPerMicro"]) + + coarse = int(TAI_us / tai_time["MicrosPerSec"]) # ; CCSDS sec + fine = TAI_us % tai_time["MicrosPerSec"] # ; CCSDS us + return (coarse, fine) + + +def calc_Epoch_for_Trigger(raw_data): + """ + I'm not at all sure what this is all about...How is this result different from the usual + PODA to CDF conversion? + :param raw_data: + :return: + """ + Timeus = {} + Timeus["288"] = 30000 + Timeus["296"] = 150000 + thisTimeus = Timeus[str(raw_data["PREPENDED_APID"][0])] + + usPerSec = 1000000 + elePerSamp = (raw_data["CMPTRIGGERTERM"]).shape[1] # 288 = 150 , 296 = 30 + nDes = len(raw_data["PREPENDED_APID"]) + corsTime = np.zeros(nDes * elePerSamp, "u4") + fineTime = np.zeros(nDes * elePerSamp, "u4") + for i in range(0, nDes): + startusec = ( + raw_data["START_CORSTIME"][i] * usPerSec + raw_data["START_FINETIME"][i] + ) + stop = (elePerSamp * thisTimeus) + startusec + usecs = np.arange(startusec, stop, thisTimeus) + offset = i * elePerSamp + corsTime[offset : offset + elePerSamp] = usecs / usPerSec + fineTime[offset : offset + elePerSamp] = usecs % usPerSec + + new_Epoch = converting_ccsds_times_to_cdf(corsTime, fineTime) + return new_Epoch diff --git a/pyproject.toml b/pyproject.toml index 52c7e84..00a9b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,8 @@ classifiers = [ "Topic :: Scientific/Engineering :: Physics", ] dependencies = [ - 'astropy>=4.1.0', - 'numpy>=1.16.0', 'hermes_core @ git+https://github.com/HERMES-SOC/hermes_core/', - 'ccsdspy @ git+https://github.com/ddasilva/ccsdspy.git' + 'ccsdspy @ git+https://github.com/ddasilva/ccsdspy.git', ] [project.optional-dependencies] diff --git a/setup.cfg b/setup.cfg index 6e4692d..ca18004 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,4 @@ [flake8] +extend-ignore = E203,E402 select = E101,E11,E111,E112,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E133,E20,E211,E231,E241,E242,E251,E252,E26,E265,E266,E27,E301,E302,E303,E304,E305,E306,E401,E402,E502,E701,E711,E712,E713,E714,E722,E731,E901,E902,F822,F823,W191,W291,W292,W293,W391,W601,W602,W603,W604,W605,W690 exclude = .txt, .rst, .md \ No newline at end of file