Main inv ci

.github/workflows/testing.yml

@@ -8,6 +8,9 @@ on:
   pull_request:
     branches:
       - main
+  workflow_dispatch: # For on demand runs
+  schedule:
+    - cron: 0 0 * * * # Scheduled run every day at midnight
 jobs:
   build:
 
@@ -16,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.9, '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v2
@@ -26,12 +29,61 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install -e .[test]
-    - name: Run tests
+        pip install pip setuptools wheel --upgrade
+        # Install spacepy without build isolation to avoid issues with numpy
+        pip install numpy
+        pip install spacepy --no-build-isolation
+        pip install -e .[test]
+      if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}
+
+    - name: Install CDF library on Ubuntu
+      run: |
+        wget https://sdc-aws-support.s3.amazonaws.com/cdf-binaries/latest.zip
+        unzip latest.zip
+        echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+      if: ${{(matrix.platform == 'ubuntu-latest')}}
+
+    - name: Install CDF library on MacOS
+      run: |
+        wget https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/macosx/CDF3_9_0-binary-signed.pkg
+        sudo installer -pkg CDF3_9_0-binary-signed.pkg -target /
+        ls -l
+        echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+      if: ${{(matrix.platform == 'macos-latest')}}
+
+    # Set up CDF and run tests on Windows
+    - name: Install CDF on Windows
+      shell: cmd
+      run: |
+        # Download and unzip CDF
+        curl -L "https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/latest/windows/cdf3.9.0_64bit_WinZip_Installer.zip" --output cdf.zip
+        mkdir cdf_lib
+        tar -xf cdf.zip -C cdf_lib
+
+        # Set environment variables for CDF
+        set mydir=%cd%
+        set CDF_BASE=%mydir%\cdf_lib
+        set CDF_INC=%mydir%\cdf_lib\include
+        set CDF_LIB=%mydir%\cdf_lib\lib
+        set CDF_HELP=%mydir%\cdf_lib\help
+        set CDF_LEAPSECONDSTABLE=%mydir%\cdf_lib\CDFLeapSeconds.txt
+        set CLASSPATH=%mydir%\cdf_lib\CDFToolsDriver.jar;%mydir%\cdf_lib\lib\cdfjava.jar;%mydir%\cdf_lib\lib\cdfml.jar;%mydir%\cdf_lib\lib\cdfjson.jar;%mydir%\cdf_lib\lib\cdfj.jar;%mydir%\cdf_lib\lib\gson-2.8.6.jar;%mydir%\cdf_lib\lib\javax.json-1.0.4.jar;.
+        set PATH=%mydir%\cdf_lib;%mydir%\cdf_lib\bin;%PATH%
+        set TERMINFO=%mydir%\cdf_lib\lib\terminfo
+        set JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
+
+        # Run tests
+        pytest --pyargs hermes_eea --cov hermes_eea
+      # Skip Windows Python 3.11 tests until SpacePy is updated
+      if: ${{(matrix.platform == 'windows-latest' && matrix.python-version != '3.11')}}
+
+    - name: Run tests on Ubuntu and MacOS
       run: pytest --pyargs hermes_eea --cov hermes_eea
       env:
         PLATFORM: ${{ matrix.platform }}
+      if: ${{ !(matrix.platform == 'windows-latest') }}
+
     - name: Upload coverage reports to Codecov with GitHub Action
       uses: codecov/codecov-action@v3
-
+      # Skip Windows Python 3.11 tests until SpacePy is updated
+      if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}

hermes_eea/calibration/calibration.py

@@ -12,7 +12,6 @@
 
 from hermes_core import log
 from hermes_core.util.util import create_science_filename, parse_science_filename
-
 import hermes_eea
 from hermes_eea.io import read_file
 
@@ -79,8 +78,8 @@ def calibrate_file(data_filename: Path) -> Path:
 
     # check if level 0 binary file, if so call appropriate functions
     if file_metadata["instrument"] == hermes_eea.INST_NAME and file_metadata["level"] == "l0":
+        # because of error handling, no test of data is necessary here.
         data = parse_l0_sci_packets(data_filename)
-
         level1_filename = l0_sci_data_to_cdf(data, data_filename)
         output_filename = level1_filename
     elif file_metadata["instrument"] == hermes_eea.INST_NAME and file_metadata["level"] == "l1":
@@ -110,7 +109,9 @@ def calibrate_file(data_filename: Path) -> Path:
         # create an empty file for testing purposes
         with open(data_filename.parent / ql_filename, "w"):
             pass
-
+        # here
+        data = parse_l0_sci_packets(data_filename)
+        level1_filename = l0_sci_data_to_cdf(data, data_filename)
         # example log messages
         log.info(f"Despiking removing {random.randint(0, 10)} spikes")
         log.warning(f"Despiking could not remove {random.randint(1, 5)}")
@@ -195,6 +196,8 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path:
     >>> data_packets = calib.parse_l0_sci_packets(data_filename)  # doctest: +SKIP
     >>> cdf_filename = calib.l0_sci_data_to_cdf(data_packets, data_filename)  # doctest: +SKIP
     """
+
+    # this is transferring name.bin to name.cdf
     file_metadata = parse_science_filename(original_filename.name)
 
     # coarse = data["SHCOARSE"][idx]
@@ -214,6 +217,12 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path:
                 "masterSkeletons/hermes_eea_l1_00000000000000_v0.0.0.cdf",
             ),
         )
+        cdf.close()
+    if data:
+        cdf = pycdf.CDF(str(cdf_filename))
+        cdf.readonly(False)
+
+        # writes the data to the blank cdfd
         cdf["Epoch"] = converting_ccsds_times_to_cdf(data["SHCOARSE"], data["SHFINE"])
         cdf["hermes_eea_accumulations"] = data["ACCUM"]
         cdf["hermes_eea_counter1"] = data["COUNTER1"]

hermes_eea/tests/test_calibration.py

@@ -1,7 +1,7 @@
 import pytest
 import os.path
 from pathlib import Path
-
+import ccsdspy
 import hermes_eea.calibration as calib
 from hermes_eea import _data_directory
 from hermes_core.util.util import create_science_filename, parse_science_filename
@@ -10,9 +10,10 @@
 ql_filename = "hermes_eea_ql_20221205_000000_v1.0.0.cdf"
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="session") # this is a pytest fixture
 def level0_file(tmp_path_factory):
-    fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023038-000000_v0.bin"))
+    #fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023038-000000_v0.bin"))
+    fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023041-000000_v0.bin"))
     return fn
 
 
@@ -23,7 +24,7 @@ def level1_file(tmp_path_factory):
         pass
     return fn
 
-
+# this creates a blank cdf with the proper name -- not too interesting
 def test_l0_sci_data_to_cdf(level0_file):
     """Test that the output filenames are correct and that a file was actually created."""
     data = {}
@@ -32,18 +33,20 @@ def test_l0_sci_data_to_cdf(level0_file):
     assert output_file.is_file()
 
 
+# This drops all the way down to ccsdspy but seems to work
 def test_calibrate_file_nofile_error():
     """Test that if file does not exist it produces the correct error. The file needs to be in the correct format."""
     with pytest.raises(FileNotFoundError):
         calib.calibrate_file(Path("hermes_EEA_l0_2032339-000000_v0.bin"))
 
-
+# This one is less clear as yet...
 def test_process_file_nofile_error():
     """Test that if file does not exist it produces the correct error. The file needs to be in the correct format."""
     with pytest.raises(FileNotFoundError):
         calib.process_file(Path("hermes_EEA_l0_2032339-000000_v0.bin"))
 
 
+# this fills the blank cdf with data
 def test_calibrate_file(level0_file, level1_file):
     """Test that the output filenames are correct and that a file was actually created."""
     output_file = calib.calibrate_file(level0_file)
@@ -60,15 +63,15 @@ def test_calibrate_file(level0_file, level1_file):
     #    == "Calibration file for datafile_with_no_calib.cdf not found."
     # )
 
-
+# this also populates the file with data
 def test_process_file_level0(level0_file):
     """Test that the output filenames are correct and that a file was actually created."""
     file_output = calib.process_file(level0_file)
     assert len(file_output) == 1
     # assert file_output[0].name == level1_filename
     assert file_output[0].is_file()
 
-
+# this populates a level 1, a different file but doesn't really, now it is just a stub
 def test_process_file_level1(level1_file):
     """Test that the output filenames are correct and that a file was actually created."""
     file_output = calib.process_file(level1_file)

pyproject.toml

@@ -28,8 +28,6 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Physics",
 ]
 dependencies = [
-  'astropy>=4.1.0',
-  'numpy>=1.16.0',
   'hermes_core @ git+https://github.com/HERMES-SOC/hermes_core/',
   'ccsdspy @ git+https://github.com/ddasilva/ccsdspy.git'
 ]
@@ -78,7 +76,7 @@ testpaths = [
 ]
 doctest_plus = "enabled"
 text_file_format = "rst"
-addopts = "--doctest-rst"
+addopts = '--doctest-glob="*.rst"'
 
 [tool.coverage.run]
 omit = [