Draft: Single test refactor

.github/workflows/docs.yml

@@ -13,13 +13,9 @@ on:
     - cron: 0 0 * * * # Scheduled run every day at midnight
 jobs:
   build:
-
-    runs-on: ${{ matrix.platform }}
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [ubuntu-latest]
-        python-version: [3.9]
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/testing.yml

@@ -35,12 +35,55 @@ jobs:
         pip install spacepy --no-build-isolation
         pip install -e .[test]
       if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}
-    - name: Run tests
+
+    - name: Install CDF library on Ubuntu
+      run: |
+        wget https://sdc-aws-support.s3.amazonaws.com/cdf-binaries/latest.zip
+        unzip latest.zip
+        echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+      if: ${{(matrix.platform == 'ubuntu-latest')}}
+
+    - name: Install CDF library on MacOS
+      run: |
+        wget https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/macosx/CDF3_9_0-binary-signed.pkg
+        sudo installer -pkg CDF3_9_0-binary-signed.pkg -target /
+        ls -l
+        echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+      if: ${{(matrix.platform == 'macos-latest')}}
+
+    # Set up CDF and run tests on Windows
+    - name: Install CDF on Windows
+      shell: cmd
+      run: |
+        # Download and unzip CDF
+        curl -L "https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/latest/windows/cdf3.9.0_64bit_WinZip_Installer.zip" --output cdf.zip
+        mkdir cdf_lib
+        tar -xf cdf.zip -C cdf_lib
+
+        # Set environment variables for CDF
+        set mydir=%cd%
+        set CDF_BASE=%mydir%\cdf_lib
+        set CDF_INC=%mydir%\cdf_lib\include
+        set CDF_LIB=%mydir%\cdf_lib\lib
+        set CDF_HELP=%mydir%\cdf_lib\help
+        set CDF_LEAPSECONDSTABLE=%mydir%\cdf_lib\CDFLeapSeconds.txt
+        set CLASSPATH=%mydir%\cdf_lib\CDFToolsDriver.jar;%mydir%\cdf_lib\lib\cdfjava.jar;%mydir%\cdf_lib\lib\cdfml.jar;%mydir%\cdf_lib\lib\cdfjson.jar;%mydir%\cdf_lib\lib\cdfj.jar;%mydir%\cdf_lib\lib\gson-2.8.6.jar;%mydir%\cdf_lib\lib\javax.json-1.0.4.jar;.
+        set PATH=%mydir%\cdf_lib;%mydir%\cdf_lib\bin;%PATH%
+        set TERMINFO=%mydir%\cdf_lib\lib\terminfo
+        set JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
+
+        # Run tests
+        pytest --pyargs hermes_eea --cov hermes_eea
+      # Skip Windows Python 3.11 tests until SpacePy is updated
+      if: ${{(matrix.platform == 'windows-latest' && matrix.python-version != '3.11')}}
+
+    - name: Run tests on Ubuntu and MacOS
       run: pytest --pyargs hermes_eea --cov hermes_eea
       env:
         PLATFORM: ${{ matrix.platform }}
       # Skip Windows Python 3.11 tests until SpacePy is updated
-      if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}
+      if: ${{ !(matrix.platform == 'windows-latest') }}
+
     - name: Upload coverage reports to Codecov with GitHub Action
       uses: codecov/codecov-action@v3
       # Skip Windows Python 3.11 tests until SpacePy is updated

.readthedocs.yml

@@ -1,7 +1,6 @@
 # Read the Docs configuration file for Sphinx projects
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
-# Required
 version: 2
 
 # Set the OS, Python version and other tools you might need
@@ -21,4 +20,4 @@ build:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats:
   - pdf
-  - epub
+  - epub

docs/conf.py

@@ -8,6 +8,10 @@
 import os
 import sys
 
+# -- Environmental Variables ------------------------------------------------
+# Set CDF Library Path
+os.environ["CDF_LIB"] = "../cdf/lib"
+
 sys.path.insert(0, os.path.abspath(".."))
 # -- Project information -----------------------------------------------------
 

docs/whatsnew/changelog.rst

@@ -4,7 +4,5 @@
 Full Changelog
 **************
 
-.. changelog::
-   :towncrier: ../../
-   :towncrier-skip-if-empty:
-   :changelog_file: ../../CHANGELOG.rst
+.. include:: ../../CHANGELOG.rst
+

hermes_eea/SkymapFactory.py

@@ -0,0 +1,122 @@
+import numpy as np
+from hermes_core import log
+from hermes_eea.io import EEA
+from hermes_eea.util.time import ccsds_to_cdf_time
+from hermes_eea import energies as voltages
+
+N_ENERGIES = 41
+N_DEFLECTIONS = 4
+N_AZIMUTH = 32
+
+
+# This may eventually be handled in a python multiprocessor module instance:
+def SkymapFactory(l0_cdf, energies, deflections, myEEA):
+    # ['Epoch', 'Epoch_plus_var', 'Epoch_minus_var', 'hermes_eea_step_counter',
+    # 'hermes_eea_counter1', 'hermes_eea_counter2', 'hermes_eea_accumulations',
+    # 'hermes_eea_sector_index', 'hermes_eea_sector_label'])
+
+    # science_data:
+    start_of_good_data = np.where(l0_cdf["SHEID"][:] == 1)[0][0]
+    integrates_at_end = np.where(
+        l0_cdf["SHEID"][start_of_good_data:] == 0
+    )  # has 63 values
+    # We are expecting integrates to be only at the beginning
+
+    # The Science Data:
+    stepper_table_packets = (np.where(l0_cdf["SHEID"][:] > 0))[0]
+    return_package = {}
+    beginning_packets = (
+        np.where((l0_cdf["STEP"][stepper_table_packets[0] :]) == 0)[0]
+        + stepper_table_packets[0]
+    )
+    package = []
+
+    epochs = ccsds_to_cdf_time.helpConvertEEA(l0_cdf)
+    try:
+        for ptr in range(0, len(beginning_packets)):
+            package.append(
+                (
+                    l0_cdf["STEP"][beginning_packets[ptr] : beginning_packets[ptr + 1]],
+                    l0_cdf["ACCUM"][
+                        beginning_packets[ptr] : beginning_packets[ptr + 1]
+                    ],
+                    l0_cdf["COUNTER1"][
+                        beginning_packets[ptr] : beginning_packets[ptr + 1]
+                    ],
+                    l0_cdf["COUNTER2"][
+                        beginning_packets[ptr] : beginning_packets[ptr + 1]
+                    ],
+                    epochs[beginning_packets[ptr] : beginning_packets[ptr + 1]],
+                    energies,
+                    deflections,
+                    ptr,
+                )
+            )
+    except IndexError:
+        log.info("Finished last interval")
+
+    result = []
+    for pckt in package:
+        packet_contents = do_eea_packet(*pckt)
+        if packet_contents is not None:
+            result.append(packet_contents)
+    myEEA.populate(myEEA, result)
+
+
+def do_eea_packet(
+    stepperTableCounter, counts, cnt1, cnt2, epoch, energies, deflections, ith_FSmap
+):
+    """
+    This function populates each sweep, or pass through
+    all of the energies and deflections designated by the stepper table
+
+    Parameters
+    ----------
+    stepperTableCounter - n_deflections * n_energies
+    counts              - the structured arrays returned by CCSDSPY
+    cnt1                - the sum of this sweep's accum
+    cnt2                - same as above but +1...not clear yet
+    epoch               - CDF Formatted time for every single measurement, [0] is the time for the sweep/packet
+    energies            - so far we only have one stepper table with 41 energies
+    deflections         - ...and 4 deflections
+    ith_FSmap           - sweep counter
+
+    Returns
+    -------
+
+    """
+    return_package = {}
+    rows = len(stepperTableCounter)
+    # skymap is already full of zeros, why do it again?
+    # skymap = np.zeros((beginning_packets[ptr+1]-beginning_packets[ptr],N_AZIMUTH))
+    skymaps = []
+    pulse_a = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+    pulse_b = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+    counter1 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+    counter2 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+    usec = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+
+    skymap = np.zeros((N_ENERGIES, N_DEFLECTIONS, N_AZIMUTH), dtype=np.uint16)
+
+    for row in stepperTableCounter:
+        dim0 = energies[row]
+        dim1 = deflections[row]
+        skymap[dim0, dim1, :] = counts[row, 0:N_AZIMUTH]
+        pulse_a[dim0, dim1] = counts[row][N_AZIMUTH]
+        pulse_b[dim0, dim1] = counts[row][N_AZIMUTH + 1]
+        counter1[dim0, dim1] = cnt1[row]
+        counter2[dim0, dim1] = cnt2[row]
+        usec[dim0, dim1] = epoch[row]
+
+    return_package["pulse_a"] = pulse_a
+    return_package["pulse_b"] = list(pulse_b)
+    return_package["counts"] = skymap
+    return_package["usec"] = usec
+    return_package["Epoch"] = epoch[0]
+    return_package["stats"] = np.sum(skymap)
+    return_package["energies"] = voltages
+    return_package["sun_angles"] = deflections
+    return_package["counter1"] = counter1
+    return_package["counter2"] = counter2
+
+    return return_package

hermes_eea/__init__.py

@@ -1,6 +1,8 @@
 # Licensed under Apache License v2 - see LICENSE.rst
 import os.path
+import sys
 
+sys.path.append(os.getcwd())
 from hermes_core import log
 from hermes_eea.io.file_tools import read_file
 
@@ -21,5 +23,54 @@
 
 _package_directory = os.path.dirname(os.path.abspath(__file__))
 _data_directory = os.path.abspath(os.path.join(_package_directory, "data"))
+_calibration_directory = os.path.abspath(os.path.join(_data_directory, "calibration"))
+
 
 log.info(f"hermes_eea version: {__version__}")
+
+stepper_table = "flight_stepper.txt"
+
+
+energies = [
+    2.18000000e00,
+    2.63177330e00,
+    3.17717004e00,
+    3.83559233e00,
+    4.63046306e00,
+    5.59005918e00,
+    6.74851766e00,
+    8.14704980e00,
+    9.83540739e00,
+    1.18736525e01,
+    1.43342944e01,
+    1.73048684e01,
+    2.08910507e01,
+    2.52204172e01,
+    3.04469818e01,
+    3.67566761e01,
+    4.43739626e01,
+    5.35698211e01,
+    6.46713874e01,
+    7.80735920e01,
+    9.42532085e01,
+    1.13785815e02,
+    1.37366271e02,
+    1.65833433e02,
+    2.00200000e02,
+    2.39800000e02,
+    3.17794829e02,
+    4.21157437e02,
+    5.58138682e02,
+    7.39673007e02,
+    9.80251281e02,
+    1.29907752e03,
+    1.72160182e03,
+    2.28155195e03,
+    3.02362557e03,
+    4.00705827e03,
+    5.31035195e03,
+    7.03754125e03,
+    9.32649800e03,
+    1.23599368e04,
+    1.63800000e04,
+]