diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 81347ba..4d45b71 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,6 +8,9 @@ on: pull_request: branches: - main + workflow_dispatch: # For on demand runs + schedule: + - cron: 0 0 * * * # Scheduled run every day at midnight jobs: build: @@ -16,7 +19,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 @@ -26,12 +29,61 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install -e .[test] - - name: Run tests + pip install pip setuptools wheel --upgrade + # Install spacepy without build isolation to avoid issues with numpy + pip install numpy + pip install spacepy --no-build-isolation + pip install -e .[test] + if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }} + + - name: Install CDF library on Ubuntu + run: | + wget https://sdc-aws-support.s3.amazonaws.com/cdf-binaries/latest.zip + unzip latest.zip + echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV + if: ${{(matrix.platform == 'ubuntu-latest')}} + + - name: Install CDF library on MacOS + run: | + wget https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/macosx/CDF3_9_0-binary-signed.pkg + sudo installer -pkg CDF3_9_0-binary-signed.pkg -target / + ls -l + echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV + if: ${{(matrix.platform == 'macos-latest')}} + + # Set up CDF and run tests on Windows + - name: Install CDF on Windows + shell: cmd + run: | + # Download and unzip CDF + curl -L "https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/latest/windows/cdf3.9.0_64bit_WinZip_Installer.zip" --output cdf.zip + mkdir cdf_lib + tar -xf cdf.zip -C cdf_lib + + # Set environment variables for CDF + set mydir=%cd% + set CDF_BASE=%mydir%\cdf_lib + set CDF_INC=%mydir%\cdf_lib\include + set CDF_LIB=%mydir%\cdf_lib\lib + set CDF_HELP=%mydir%\cdf_lib\help + set CDF_LEAPSECONDSTABLE=%mydir%\cdf_lib\CDFLeapSeconds.txt + set CLASSPATH=%mydir%\cdf_lib\CDFToolsDriver.jar;%mydir%\cdf_lib\lib\cdfjava.jar;%mydir%\cdf_lib\lib\cdfml.jar;%mydir%\cdf_lib\lib\cdfjson.jar;%mydir%\cdf_lib\lib\cdfj.jar;%mydir%\cdf_lib\lib\gson-2.8.6.jar;%mydir%\cdf_lib\lib\javax.json-1.0.4.jar;. + set PATH=%mydir%\cdf_lib;%mydir%\cdf_lib\bin;%PATH% + set TERMINFO=%mydir%\cdf_lib\lib\terminfo + set JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8" + + # Run tests + pytest --pyargs hermes_eea --cov hermes_eea + # Skip Windows Python 3.11 tests until SpacePy is updated + if: ${{(matrix.platform == 'windows-latest' && matrix.python-version != '3.11')}} + + - name: Run tests on Ubuntu and MacOS run: pytest --pyargs hermes_eea --cov hermes_eea env: PLATFORM: ${{ matrix.platform }} + if: ${{ !(matrix.platform == 'windows-latest') }} + - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 - + # Skip Windows Python 3.11 tests until SpacePy is updated + if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }} \ No newline at end of file diff --git a/hermes_eea/calibration/calibration.py b/hermes_eea/calibration/calibration.py index e1fbb7b..609dcd6 100644 --- a/hermes_eea/calibration/calibration.py +++ b/hermes_eea/calibration/calibration.py @@ -12,7 +12,6 @@ from hermes_core import log from hermes_core.util.util import create_science_filename, parse_science_filename - import hermes_eea from hermes_eea.io import read_file @@ -79,8 +78,8 @@ def calibrate_file(data_filename: Path) -> Path: # check if level 0 binary file, if so call appropriate functions if file_metadata["instrument"] == hermes_eea.INST_NAME and file_metadata["level"] == "l0": + # because of error handling, no test of data is necessary here. data = parse_l0_sci_packets(data_filename) - level1_filename = l0_sci_data_to_cdf(data, data_filename) output_filename = level1_filename elif file_metadata["instrument"] == hermes_eea.INST_NAME and file_metadata["level"] == "l1": @@ -110,7 +109,9 @@ def calibrate_file(data_filename: Path) -> Path: # create an empty file for testing purposes with open(data_filename.parent / ql_filename, "w"): pass - + # here + data = parse_l0_sci_packets(data_filename) + level1_filename = l0_sci_data_to_cdf(data, data_filename) # example log messages log.info(f"Despiking removing {random.randint(0, 10)} spikes") log.warning(f"Despiking could not remove {random.randint(1, 5)}") @@ -195,6 +196,8 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path: >>> data_packets = calib.parse_l0_sci_packets(data_filename) # doctest: +SKIP >>> cdf_filename = calib.l0_sci_data_to_cdf(data_packets, data_filename) # doctest: +SKIP """ + + # this is transferring name.bin to name.cdf file_metadata = parse_science_filename(original_filename.name) # coarse = data["SHCOARSE"][idx] @@ -214,6 +217,12 @@ def l0_sci_data_to_cdf(data: dict, original_filename: Path) -> Path: "masterSkeletons/hermes_eea_l1_00000000000000_v0.0.0.cdf", ), ) + cdf.close() + if data: + cdf = pycdf.CDF(str(cdf_filename)) + cdf.readonly(False) + + # writes the data to the blank cdfd cdf["Epoch"] = converting_ccsds_times_to_cdf(data["SHCOARSE"], data["SHFINE"]) cdf["hermes_eea_accumulations"] = data["ACCUM"] cdf["hermes_eea_counter1"] = data["COUNTER1"] diff --git a/hermes_eea/tests/test_calibration.py b/hermes_eea/tests/test_calibration.py index c06bbb9..6742745 100644 --- a/hermes_eea/tests/test_calibration.py +++ b/hermes_eea/tests/test_calibration.py @@ -1,7 +1,7 @@ import pytest import os.path from pathlib import Path - +import ccsdspy import hermes_eea.calibration as calib from hermes_eea import _data_directory from hermes_core.util.util import create_science_filename, parse_science_filename @@ -10,9 +10,10 @@ ql_filename = "hermes_eea_ql_20221205_000000_v1.0.0.cdf" -@pytest.fixture(scope="session") +@pytest.fixture(scope="session") # this is a pytest fixture def level0_file(tmp_path_factory): - fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023038-000000_v0.bin")) + #fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023038-000000_v0.bin")) + fn = Path(os.path.join(_data_directory, "hermes_EEA_l0_2023041-000000_v0.bin")) return fn @@ -23,7 +24,7 @@ def level1_file(tmp_path_factory): pass return fn - +# this creates a blank cdf with the proper name -- not too interesting def test_l0_sci_data_to_cdf(level0_file): """Test that the output filenames are correct and that a file was actually created.""" data = {} @@ -32,18 +33,20 @@ def test_l0_sci_data_to_cdf(level0_file): assert output_file.is_file() +# This drops all the way down to ccsdspy but seems to work def test_calibrate_file_nofile_error(): """Test that if file does not exist it produces the correct error. The file needs to be in the correct format.""" with pytest.raises(FileNotFoundError): calib.calibrate_file(Path("hermes_EEA_l0_2032339-000000_v0.bin")) - +# This one is less clear as yet... def test_process_file_nofile_error(): """Test that if file does not exist it produces the correct error. The file needs to be in the correct format.""" with pytest.raises(FileNotFoundError): calib.process_file(Path("hermes_EEA_l0_2032339-000000_v0.bin")) +# this fills the blank cdf with data def test_calibrate_file(level0_file, level1_file): """Test that the output filenames are correct and that a file was actually created.""" output_file = calib.calibrate_file(level0_file) @@ -60,7 +63,7 @@ def test_calibrate_file(level0_file, level1_file): # == "Calibration file for datafile_with_no_calib.cdf not found." # ) - +# this also populates the file with data def test_process_file_level0(level0_file): """Test that the output filenames are correct and that a file was actually created.""" file_output = calib.process_file(level0_file) @@ -68,7 +71,7 @@ def test_process_file_level0(level0_file): # assert file_output[0].name == level1_filename assert file_output[0].is_file() - +# this populates a level 1, a different file but doesn't really, now it is just a stub def test_process_file_level1(level1_file): """Test that the output filenames are correct and that a file was actually created.""" file_output = calib.process_file(level1_file) diff --git a/pyproject.toml b/pyproject.toml index 1aabd95..90f4c71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,8 +28,6 @@ classifiers = [ "Topic :: Scientific/Engineering :: Physics", ] dependencies = [ - 'astropy>=4.1.0', - 'numpy>=1.16.0', 'hermes_core @ git+https://github.com/HERMES-SOC/hermes_core/', 'ccsdspy @ git+https://github.com/ddasilva/ccsdspy.git' ] @@ -78,7 +76,7 @@ testpaths = [ ] doctest_plus = "enabled" text_file_format = "rst" -addopts = "--doctest-rst" +addopts = '--doctest-glob="*.rst"' [tool.coverage.run] omit = [