Merge pull request #234 from pysat/rc_0_0_6

RC 0.0.6

.github/workflows/docs.yml

@@ -16,9 +16,9 @@ jobs:
 
     name: Documentation tests
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/main.yml

@@ -11,25 +11,25 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.11", "3.12"]
         numpy_ver: ["latest"]
         test_config: ["latest"]
         include:
-          - python-version: "3.9"
-            numpy_ver: "1.21"
+          - python-version: "3.10"
+            numpy_ver: "1.24"
             os: ubuntu-latest
             test_config: "NEP29"
-          - python-version: "3.6.8"
-            numpy_ver: "1.19.5"
+          - python-version: "3.9"
+            numpy_ver: "1.23.5"
             os: "ubuntu-20.04"
             test_config: "Ops"
 
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }}
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -41,10 +41,7 @@ jobs:
       if: ${{ matrix.test_config == 'Ops'}}
       run: |
         pip install --no-cache-dir numpy==${{ matrix.numpy_ver }}
-        pip install "cdflib<1.0"
-        pip install -r requirements.txt
-        pip install -r test_requirements.txt
-        pip install .
+        pip install --upgrade-strategy only-if-needed .[test]
 
     - name: Install NEP29 dependencies
       if: ${{ matrix.test_config == 'NEP29'}}
@@ -71,7 +68,20 @@ jobs:
     - name: Test with pytest
       run: pytest
 
-    - name: Publish results to coveralls
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: coveralls --rcfile=pyproject.toml --service=github
+    - name: Coveralls Parallel
+      uses: coverallsapp/github-action@v2
+      with:
+        flag-name: run-${{ join(matrix.*, '-') }}
+        parallel: true
+
+  finish:
+    name: Finish Coverage Analysis
+    needs: build
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@v2
+      with:
+        parallel-finished: true
+

.github/workflows/pip_rc_install.yml

@@ -17,9 +17,9 @@ jobs:
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/pysat_rc.yml

@@ -18,9 +18,9 @@ jobs:
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -31,20 +31,31 @@ jobs:
     - name: Install pysat RC
       run: pip install --no-deps --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ pysat
 
-    - name: Install standard dependencies
-      run: |
-        pip install -r requirements.txt
-        pip install -r test_requirements.txt
+    - name: Install with standard dependencies
+      run: pip install .[test]
 
     - name: Set up pysat
       run: |
         mkdir pysatData
         python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'"
 
     - name: Test with pytest
-      run: pytest -vs --cov=pysatNASA/
+      run: pytest
+
+    - name: Coveralls Parallel
+      uses: coverallsapp/github-action@v2
+      with:
+        flag-name: run-${{ join(matrix.*, '-') }}
+        parallel: true
+
+  finish:
+    name: Finish Coverage Analysis
+    needs: build
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@v2
+      with:
+        parallel-finished: true
 
-    - name: Publish results to coveralls
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: coveralls --rcfile=setup.cfg --service=github

.gitignore

@@ -56,6 +56,7 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
+docs/.build/
 
 # PyBuilder
 target/

.readthedocs.yml

@@ -19,4 +19,7 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 python:
    install:
-   - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc

.zenodo.json

@@ -1,30 +1,54 @@
 {
+  "keywords": [
+    "pysat",
+    "ionosphere",
+    "atmosphere",
+    "thermosphere",
+    "magnetosphere",
+    "mars",
+    "heliosphere",
+    "observations",
+    "models",
+    "space",
+    "satellites",
+    "analysis"
+    ],
     "creators": [
       {
         "affiliation": "Goddard Space Flight Center",
         "name": "Klenzing, Jeff",
         "orcid": "0000-0001-8321-6074"
       },
-      {
-        "affiliation": "The University of Texas at Dallas",
-        "name": "Stoneback, Russell",
-        "orcid": "0000-0001-7216-4336"
-      },
       {
         "affiliation": "U.S. Naval Research Laboratory",
         "name": "Burrell, Angeline G.",
         "orcid": "0000-0001-8875-9326"
       },
+      {
+        "affiliation": "Cosmic Studio",
+        "name": "Stoneback, Russell",
+        "orcid": "0000-0001-7216-4336"
+      },
       {
         "affiliation": "Catholic University of America, Goddard Space Flight Center",
         "name": "Smith, Jonathon",
         "orcid": "0000-0002-8191-4765"
       },
       {
-        "affilitation":"University of Colorado at Boulder",
+        "affiliation":"University of Colorado at Boulder",
         "name": "Navarro, Luis",
         "orcid": "0000-0002-6362-6575"
       },
+      {
+        "affiliation": "NASA Postdoctoral Program, Goddard Space Flight Center",
+        "name": "Esman, Teresa",
+        "orcid": "0000-0003-0382-6281"
+      },
+      {
+        "affiliation": "Universities Space Research Association, Goddard Space Flight Center",
+        "name": "Govada, Aadarsh",
+        "orcid": "0009-0004-7873-5899"
+      },
       {
         "affiliation": "Predictive Science",
         "name": "Pembroke, Asher"

ACKNOWLEDGEMENTS.md

@@ -0,0 +1,30 @@
+Funding
+=======
+The following institutions, missions, and programs have provided funding
+for pysatNASA development.
+
+Institutions
+-----------
+- The Catholic University of America (CUA)
+- Cosmic Studio
+- Defense Advanced Research Projects Agency (DARPA) Defense Sciences Office
+- National Aeronautics and Space Administration (NASA)
+- National Oceanic and Atmospheric Administration (NOAA)
+- National Science Foundation (NSF)
+- Office of Naval Research (ONR)
+
+Missions
+--------
+- NASA Ionospheric Connections Explorer (ICON)
+- NASA Scintillation Observations and Response of the Ionosphere to
+  Electrodynamics (SORTIE)
+- NASA Scintillation Prediction Observations Research Task (SPORT)
+- NOAA Constellation Observing System for Meteorology Ionosphere and Climate (COSMIC-2)
+
+Disclaimers
+===========
+Any opinions or actions taken by the listed funding institutions are those of 
+the institutions and do not necessarily reflect the views of the pysat
+development team or individual authors. Any opinions, findings, and conclusions
+or recommendations expressed in this material are those of the author(s) and do
+not necessarily reflect the views of the funding agencies.

CHANGELOG.md

@@ -2,6 +2,51 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+## [0.0.6] - 2024-10-03
+* New Instruments
+  * DE2 VEFIMAGB - electric and magnetic field on the same cadence
+  * MAVEN MAG
+  * MAVEN SEP
+  * MAVEN in situ key parameters
+  * REACH Dosimeter
+  * DMSP SSUSI SDR-disk and SDR2-disk data
+* New Features
+  * Allow files to be unzipped after download
+  * Added custom `concat_data` method to JHUAPL methods, for TIMED-GUVI and
+    DMSP-SSUSI data
+  * Added time-dependent, file format function for DMSP SSUSI to DMSP methods
+  * Added cleaning to TIMED-GUVI SDR imaging data
+* Bug Fixes
+  * Fix general clean routine to skip transformation matrices
+  * New window needs to be integer for calculate_imf_steadiness
+  * Fixed version import
+  * Fixed a bug when data fails to load for CDF pandas objects
+  * Allow graceful failure with no files in jhuapl load functions
+  * New window needs to be integer for calculate_imf_steadiness
+  * Fixed a bug where cdas_download may drop the requested end date file
+  * Reverted the coveralls integration to the GitHub service for MacOS runs
+  * Fixed a bug where cdas_list_remote_files errored without remote data
+* Documentation
+  * Added example of how to export data for archival
+  * Updated documentation refs
+  * Add keywords to zenodo
+  * Fixed broken links
+* Deprecations
+  * Deprecated '' tag for de2_vefi module, support moved to de2_vefimagb
+* Maintenance
+  * Implemented unit tests for cleaning warnings
+  * Use pip install for readthedocs
+  * Moved references and acknowledgements to methods files
+  * Added tests for OMNI HRO routines
+  * Use standard clean routine for C/NOFS VEFI mag data
+  * Added version cap for sphinx_rtd_theme
+  * Include standard tests for ICON IVM-B
+  * Update NEP29 standards for Jun 2024
+  * Updated standards for pandas, numpy, and pysat
+  * Updated versions in GitHub Actions
+  * Implement coveralls app in GitHub Actions
+  * Cycled Operational Environment testing
+
 ## [0.0.5] - 2023-06-27
 * New Instruments
   * ACE EPAM

README.md

@@ -30,6 +30,7 @@ Python 3.6+.
 | numpy            |                   |                  |
 | pandas           |                   |                  |
 | requests         |                   |                  |
+| scipy>=1.4.0     |                   |                  |
 | xarray           |                   |                  |
 
 ## PyPi Installation

docs/acknowledgements.rst

@@ -0,0 +1 @@
+.. mdinclude:: ../ACKNOWLEDGEMENTS.md

docs/archival.rst

@@ -0,0 +1,74 @@
+Building data files for archival at NASA SPDF
+=============================================
+
+The codes and routines at :py:mod:`pysatNASA` are designed for end-users of
+NASA data products. However, pysat in general has also been used to build
+operational instruments for generating archival data to be uploaded to the
+Space Physics Data Facility (SPDF) at NASA.
+
+In general, such instruments should include separate naming conventions. An
+example of this is the REACH data, where netCDF4 files are generated for
+archival purposes as part of the :py:mod:`ops_reach` package, but can be
+accessed by the end user through :py:mod:`pysatNASA`.
+
+In general, a :py:class:`pysat.Instrument` object can be constructed for any
+dataset. Full instructions and conventions can be found
+`at ReadTheDocs <https://pysat.readthedocs.io/en/latest/new_instrument.html>`_.
+In the case of the REACH data, the operational code reads in a series of CSV
+files and updates the metadata according to user specifications. Once the file
+is loaded, it can be exported to a netCDF4 file via pysat. In the simplest case,
+this is:
+
+::
+
+  reach = pysat.Instrument(inst_module=aero_reach, tag='l1b', inst_id=inst_id)
+  pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch')
+
+
+However, there are additional options when translating pysat metadata to SPDF
+preferred formats.  An example of this is:
+
+::
+
+  # Use meta translation table to include SPDF preferred format.
+  # Note that multiple names are output for compliance with pysat.
+  # Using the most generalized form for labels for future compatibility.
+  meta_dict = {reach.meta.labels.min_val: ['VALIDMIN'],
+               reach.meta.labels.max_val: ['VALIDMAX'],
+               reach.meta.labels.units: ['UNITS'],
+               reach.meta.labels.name: ['CATDESC', 'LABLAXIS', 'FIELDNAM'],
+               reach.meta.labels.notes: ['VAR_NOTES'],
+               reach.meta.labels.fill_val: ['_FillValue'],
+               'Depend_0': ['DEPEND_0'],
+               'Format': ['FORMAT'],
+               'Monoton': ['MONOTON'],
+               'Var_Type': ['VAR_TYPE']}
+
+  pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch',
+                                meta_translation=meta_dict,
+                                export_pysat_info=False)
+
+
+In this case, note that the pysat 'name' label is output to three different
+metadata values required by the ITSP standards. Additionally, the
+:py:attr:`export_pysat_info` option is set to false here. This drops several
+internal :py:mod:`pysat` metadata values before writing to file. Note that
+this includes the default acknowledgements and references objects. These
+are set manually to avoid conflicts between the original dataset and the
+new dataset, as well as keeping in line with requirements with potentially
+different data servers. An example can be found in the [REACH Operational Software](https://github.com/jklenzing/ops_reach/blob/main/ops_reach/instruments/methods/reach.py).
+
+A full guide to SPDF metadata standards can be found 
+`at SPDF <https://spdf.gsfc.nasa.gov/istp_guide/istp_guide.html>`_.
+
+Other best practices for archival include adding the operational software
+version to the metadata header before writing. The pysat version will be
+automatically written to the metadata.
+
+::
+
+  reach.meta.header.Software_version = ops_reach.__version__
+
+
+A full example script to generate output files can be found at
+https://github.com/jklenzing/ops_reach/blob/main/scripts/netcdf_gen.py