This does everything but write to cdf. I think I am supposed to use H…

…ermesData container

hermes_eea/Global_Attrs_Cnts.py

@@ -0,0 +1,133 @@
+import os
+from datetime import datetime
+import math
+
+class Global_Attrs:
+
+
+    def __init__(self, version,file_id,lo_ext):
+
+        # this is linux date format:
+        self.Generation_date = datetime.now().strftime('%a %b %d %H:%M:%S %Y')
+        self.Logical_file_id = os.path.basename(file_id)
+        self.Data_version = version
+        self.Spacecraft_attitude_filenames = ""
+        self.lo_ext = lo_ext
+
+
+    def setCorrectionTableValues(DES_glbattr, corrTable,conf,usec_searchfile):
+        """saved in commit 2374012bbab301a23075b29c79ba9ab89285eb08
+        these and others aren't needed by DES_L1A
+        """
+        sf_des = float(corrTable['sf_' + conf.mode])
+        #DES_glbattr.Correction_table_rev                = getfilerev(corrTable["dis_p2_file_path"])
+        DES_glbattr.Correction_table_name               = os.path.basename(conf.corr_file)
+        DES_glbattr.Correction_table_scaling_factor =  "{:0.5f}".format(sf_des)
+        DES_glbattr.Energy_table_name               =  conf.energyfile
+        #DES_glbattr.Lower_energy_integration_limit = "{:02f}".format(corrTable[inp['mode'] + 'lowIntegLimit'])
+        #DES_glbattr.Upper_energy_integration_limit = "{:02f}".format(corrTable[inp['mode'] + 'highIntegLimit'])
+        DES_glbattr.Dead_time_correction = str(math.ceil(conf.effective_deadtime / 1e-9)) + ' ns'
+        #DES_glbattr.Magnetic_field_filenames = srvy_fnames
+        DES_glbattr.skymap_avgN = corrTable['skymap_avgN']
+        if usec_searchfile != None and os.path.exists(usec_searchfile):
+            DES_glbattr.Microsecond_offset_filename = os.path.basename(usec_searchfile)
+
+    '''
+    glb attrs needed for L1A:
+    Data_version: 3.4.0, 0.0.0
+    Logical_file_id: mms1_fpi_fast_l1a_des-cnts_20171221020000_v3.4.0, mms1_fpi_fast_l1a_des-cnts_00000000000000_v0.0.0
+    Generation_date: Mon Oct 17 15:45:17 2022, 20151102
+     '''
+    def populate_global_attrs(self, myDES_L1A):
+
+        myDES_L1A.Generation_date =                self.Generation_date
+        myDES_L1A.Logical_file_id =                self.Logical_file_id.replace(".cdf","")
+        myDES_L1A.Data_version =                   self.Data_version
+
+    '''
+    POPULATE GLOBAL ATTRIBUTES
+    '''
+
+    def slow_survey_DES(DES_gblattr,corrTable, conf, MG, MGB,
+                        sc_pot, photo_model,PKT_SECHDREXID):
+        luts = corrTable['photomodel_luts']
+        cfile = "_".join([conf.sc_id , conf.mode, 'f1ct', corrTable["tag_des"] ,
+                          'p' + corrTable["etag_des1"] + '.txt'])
+
+        # BURST uses both brst magnetic field files and fast srvy magnetic field files
+        magfield_filenames = ((str(MG.srvy_basenames).replace("[","")[0:-1]).replace("'","")).replace(",","")
+        if MGB != None:
+            brst_magfield_filenames = ((str(MGB.srvy_basenames).replace("[", "")[0:-1]).replace("'", "")).replace(",", "")
+            magfield_filenames = " ".join([magfield_filenames, brst_magfield_filenames])
+
+
+        energy_e0 = 100
+        scp_energy_frac = 1.0
+        DES_gblattr.Correction_table_name               = cfile
+        DES_gblattr.Correction_table_scaling_factor    =  "{:0.5f}".format(float(luts[5]))
+        #DES_gblattr.Correction_table_rev                = getfilerev(os.path.join(corrTable['pathdir'], conf.sc_id, cfile))
+        correction_table_scaling_factor                 = "{:0.5f}".format(float(corrTable['sf_des']))
+        DES_gblattr.Energy_table_name                   = conf.energyfile
+        DES_gblattr.Dead_time_correction                = conf.effective_deadtime
+        DES_gblattr.Magnetic_field_filenames            = magfield_filenames
+        DES_gblattr.Spacecraft_potential_filenames      = ((str(sc_pot.scp_basenames).replace("[","")[0:-1]).replace("'","")).replace(",","")
+        DES_gblattr.Photoelectron_model_filenames       =  photo_model.fname
+        DES_gblattr.Photoelectron_model_scaling_factor  =  "{:0.7f}".format(float(luts[5]))
+        if conf.use_pfilter:
+            DES_gblattr.Photoelectron_filter                = "On"
+        else:
+            DES_gblattr.Photoelectron_filter                = "Off"
+        DES_gblattr.Lower_energy_integration_limit      = "{:0.2f}eV".format(float(corrTable["deslowIntegLimit"]))
+        DES_gblattr.Upper_energy_integration_limit      = "{:0.2f}eV".format(float(corrTable["deshighIntegLimit"]))
+        DES_gblattr.Energy_e0                           =  "{:0.2f}eV".format(energy_e0)  # from orbit_constants
+        DES_gblattr.Scp_energy_fraction                 = "{:0.2f}eV".format(scp_energy_frac)
+        DES_gblattr.skymap_avgN                         =  corrTable['skymap_avgN']
+        DES_gblattr.Quadrant                            = PKT_SECHDREXID & 3
+        DES_gblattr.High_energy_extrapolation           = 'Enabled' # as this is always true
+        try:
+            DES_gblattr.Low_energy_extrapolation = (['Disabled', 'Enabled'])[DES_gblattr.lo_ext]
+        except TypeError:
+            pass
+
+    # These files are obtained only during processing...
+    def build_attitude(self,defatt_filenames_obtained_from_dbcs):
+        # This file changes every time a day boundary is crossed and a new transform is needed (every 10 min)
+        for file in defatt_filenames_obtained_from_dbcs:
+            if file != None:
+                if self.Spacecraft_attitude_filenames.find(os.path.basename(file)) < 0:
+                      self.Spacecraft_attitude_filenames = self.Spacecraft_attitude_filenames + " " +  \
+                      os.path.basename(file)
+
+    def slow_survey_DIS(DIS_gblattr,corrTable, conf,
+                        MG, MGB, sc_pot, photo_model, PKT_SECHDREXID):
+        cfile =  "_".join([conf.sc_id, conf.mode,'f1ct',  corrTable["tag_dis"], 'p' + corrTable["etag_dis1"]]) + '.txt'
+        luts = corrTable['photomodel_luts']
+        energy_e0 = 100
+        scp_energy_frac = 1.0
+        DIS_gblattr.Correction_table_name = cfile
+        DIS_gblattr.Correction_table_scaling_factor = "{:0.5f}".format(float(corrTable['sf_dis']))
+        #DIS_gblattr.Correction_table_rev = getfilerev(os.path.join(corrTable['pathdir'], conf.sc_id, cfile))
+        correction_table_scaling_factor = "{:0.5f}".format(float(corrTable['sf_dis']))
+        DIS_gblattr.Energy_table_name = conf.energyfile
+        DIS_gblattr.Dead_time_correction = conf.effective_deadtime
+        DIS_gblattr.Magnetic_field_filenames = (
+            (str(MG.srvy_basenames).replace("[", "")[0:-1]).replace("'", "")).replace(",", "")
+        DIS_gblattr.Spacecraft_potential_filenames = (
+            (str(sc_pot.scp_basenames).replace("[", "")[0:-1]).replace("'", "")).replace(",", "")
+        DIS_gblattr.Photoelectron_model_filenames = photo_model.fname
+        DIS_gblattr.Photoelectron_model_scaling_factor =  float(luts[5])
+        DIS_gblattr.Photoelectron_filter  =  conf.use_pfilter
+        DIS_gblattr.Lower_energy_integration_limit = "{:0.2f}eV".format(float(corrTable["dislowIntegLimit"]))
+        DIS_gblattr.Upper_energy_integration_limit = "{:0.2f}eV".format(float(corrTable["dishighIntegLimit"]))
+
+        DIS_gblattr.Energy_e0 =  "{:0.2f}eV".format(energy_e0) # from orbit_constants
+        DIS_gblattr.Scp_energy_fraction = "{:0.2f}eV".format(scp_energy_frac)
+        DIS_gblattr.skymap_avgN = corrTable['skymap_avgN']
+        DIS_gblattr.Quadrant = PKT_SECHDREXID & 3
+        DIS_gblattr.High_energy_extrapolation = 'Enabled'
+        DIS_gblattr.Spacecraft_attitude_filenames = ''  # updated in map loop
+        DIS_gblattr.High_energy_extrapolation           = 'Enabled' # as this is always true
+        try:
+            DIS_gblattr.Low_energy_extrapolation = (['Disabled', 'Enabled'])[DIS_gblattr.lo_ext]
+        except TypeError:
+            pass

hermes_eea/SkymapFactory.py

@@ -0,0 +1,104 @@
+
+import numpy as np
+from hermes_core import log
+from hermes_eea.io import EEA
+from hermes_eea.util.time import ccsds_to_cdf_time
+
+
+# This may eventually be put inside a multiprocessor:
+def SkymapFactory(l0_cdf,energies,deflections,myEEA):
+    #['Epoch', 'Epoch_plus_var', 'Epoch_minus_var', 'hermes_eea_step_counter',
+    #'hermes_eea_counter1', 'hermes_eea_counter2', 'hermes_eea_accumulations',
+    #'hermes_eea_sector_index', 'hermes_eea_sector_label'])
+
+    # science_data:
+    start_of_good_data = np.where(l0_cdf['SHEID'][:] == 1)[0][0]
+    integrates_at_end = np.where(l0_cdf['SHEID'][start_of_good_data:] == 0) # has 63 values
+    # We are expecting integrates to be only at the beginning
+
+    # The Science Data:
+    stepper_table_packets = (np.where(l0_cdf['SHEID'][:] > 0))[0]
+    return_package = {}
+    beginning_packets = np.where((l0_cdf['STEP'][stepper_table_packets[0]:]) == 0 )[0] + stepper_table_packets[0]
+    package = []
+
+    epochs = ccsds_to_cdf_time.helpConvertEEA(l0_cdf)
+    try:
+        #for ptr in range(0,len(beginning_packets)):
+        for ptr in range(0, 7000):
+            #skymap = np.zeros((beginning_packets[ptr+1]-beginning_packets[ptr],32))
+            package.append((
+                l0_cdf['STEP'][beginning_packets[ptr]:beginning_packets[ptr+1]],
+                l0_cdf['ACCUM'][beginning_packets[ptr]:beginning_packets[ptr+1]],
+                l0_cdf['COUNTER1'][beginning_packets[ptr]:beginning_packets[ptr+1]],
+                l0_cdf['COUNTER2'][beginning_packets[ptr]:beginning_packets[ptr+1]],
+                epochs[beginning_packets[ptr]:beginning_packets[ptr+1]],
+
+                energies, deflections,ptr
+            ))
+    except IndexError:
+        log.info("Finished last interval")
+
+    result = []
+    for pckt in package:
+                packet_contents = do_eea_packet(*pckt)
+                if packet_contents != None:
+                   result.append(packet_contents)
+    myEEA.populate(myEEA, result)
+
+    #epoch = ccsds_to_cdf_time.helpConvert_eea(l0_cdf)
+    #zero_values_past_first = np.where(l0_cdf['hermes_eea_intgr_or_stepper'][135:] == 0)[0]
+    #l0_cdf['hermes_eea_step_counter'][zero_values_past_first]
+    #first_packages = np.where(l0_cdf['SHEID'] > 0)
+
+# This does an entire sweep of, nominally, 164 thingies
+def do_eea_packet( stepperTableCounter,
+                   counts,
+                   cnt1,cnt2,
+                   epoch,
+                   energies,
+                   deflections,ith_FSmap):
+
+     return_package = {}
+     rows = len(stepperTableCounter)
+     # skymap is already full of zeros, why do it again?
+     # skymap = np.zeros((beginning_packets[ptr+1]-beginning_packets[ptr],32))
+     skymaps = []
+     pulse_a = np.zeros((41,4))
+     pulse_b = np.zeros((41,4))
+     counter1 = np.zeros((41,4))
+     counter2 = np.zeros((41,4))
+     µepoch   = np.zeros((41,4))
+
+     skymap = np.zeros((41, 4, 32))
+
+     for row in stepperTableCounter:
+         dim0 = energies[row]
+         dim1 = deflections[row]
+         if cnt1[row] > 3:
+             pass
+         skymap[dim0, dim1, :]     = counts[row,0:32]
+         pulse_a[dim0, dim1] = counts[row][32]
+         pulse_b[dim0, dim1] = counts[row][33]
+         counter1[dim0, dim1] = cnt1[row]
+         counter2[dim0, dim1] = cnt2[row]
+         µepoch[dim0, dim1] = epoch[row]
+
+
+# if len(stepperTableCounter) != 64:
+    #    log.info(str(ith_FSmap) + ": stepperTable rows:" + str(len(stepperTableCounter)))
+    #    return None
+     return_package['pulse_a'] = pulse_a
+     return_package['pulse_b'] = list(pulse_b)
+     return_package['counts']  = skymap
+     return_package['µEpoch']  = µepoch
+     return_package['Epoch']  =  epoch[0]
+     return_package['stats']  =  np.sum(skymap)
+     return_package['energies']  =  energies
+     return_package['sun_angles']  = deflections
+     return_package['counter1'] = counter1
+     return_package['counter2'] = counter2
+
+     return return_package
+
+

hermes_eea/__init__.py

@@ -20,6 +20,11 @@
 INST_TO_TARGETNAME = {INST_NAME: INST_TARGETNAME}
 
 _package_directory = os.path.dirname(os.path.abspath(__file__))
+
 _data_directory = os.path.abspath(os.path.join(_package_directory, "data"))
 
+
 log.info(f"hermes_eea version: {__version__}")
+
+skeleton = str( os.path.join(_data_directory, "masterSkeletons", "hermes_eea_l0_00000000000000_v0.0.0.cdf") )
+stepper_table = "flight_stepper.txt"

hermes_eea/calibration/calibration.py

@@ -14,6 +14,11 @@
 from hermes_core.util.util import create_science_filename, parse_science_filename
 import hermes_eea
 from hermes_eea.io import read_file
+import hermes_eea.calibration as calib
+from hermes_eea.io.EEA import EEA
+from hermes_eea.SkymapFactory import SkymapFactory
+from hermes_eea.Global_Attrs_Cnts import Global_Attrs
+from hermes_eea.util.time.iso_epoch import epoch_to_iso_obj, epoch_to_iso
 
 __all__ = [
     "process_file",
@@ -222,22 +227,29 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path:
         cdf = pycdf.CDF(str(cdf_filename))
         cdf.readonly(False)
 
-        # writes the data to the blank cdfd
-        cdf["Epoch"] = converting_ccsds_times_to_cdf(data["SHCOARSE"], data["SHFINE"])
-        cdf["hermes_eea_accumulations"] = data["ACCUM"]
-        cdf["hermes_eea_counter1"] = data["COUNTER1"]
-        cdf["hermes_eea_counter2"] = data["COUNTER2"]
-        cdf["hermes_eea_step_counter"] = data["STEP"]
-
-        cdf.close()
+        calibration_file = get_calibration_file(hermes_eea.stepper_table)
+        read_calibration_file(calibration_file)
+
+        #eea_cdf = WriteEEACDF(file_metadata, data_filename, hermes_eea.skeleton)
+        glblattr = Global_Attrs(file_metadata['version'],
+                                                  cdf_filename.name, lo_ext=False)
+        myEEA = EEA(file_metadata)
+        # This populates so doesn't have to return much
+        SkymapFactory(data, calib.energies, calib.deflections, myEEA)
+        example_start_times = epoch_to_iso_obj(myEEA.Epoch[0:10])
+        range = [myEEA.Epoch[0], myEEA.Epoch[-1]]
+        range_of_packet = epoch_to_iso(range)
+        log.info("Range of file:" + range_of_packet[0] + " to " + range_of_packet[1])
+        n_packets = len(myEEA.Epoch)
+        #outputFile = eea_cdf.writeCDF(glblattr, myEEA, range, n_packets, srvy='normal')
+        #log.warning("Wrote CDF:" + outputFile)
 
     return cdf_filename
 
 
 def get_calibration_file(data_filename: Path, time=None) -> Path:
     """
     Given a time, return the appropriate calibration file.
-
     Parameters
     ----------
     data_filename: str
@@ -252,7 +264,7 @@ def get_calibration_file(data_filename: Path, time=None) -> Path:
     Examples
     --------
     """
-    return None
+    return os.path.join(hermes_eea._data_directory, data_filename)
 
 
 def read_calibration_file(calib_filename: Path):
@@ -272,4 +284,9 @@ def read_calibration_file(calib_filename: Path):
     Examples
     --------
     """
-    return None
+    lines = read_file(os.path.join(calib_filename))
+    calib.energies = []
+    calib.deflections = []
+    for line in lines:
+        calib.energies.append(int(line[8:10], 16))
+        calib.deflections.append(int(line[10:12], 16))