Skip to content

Improved external test data file handling #1828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bourque merged 8 commits into from
Jun 24, 2025
Merged

Improved external test data file handling #1828

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
db239b1
Moved list of external test files to separate file
bourque Jun 13, 2025
ad20bb2
fixed import
bourque Jun 13, 2025
7346929
Removed unneeded call to _download_external_data
bourque Jun 13, 2025
e73113c
Improved readability
bourque Jun 13, 2025
090d241
Added CoDICE L1a validation files"
bourque Jun 13, 2025
bccf325
Removed CoDICE L1a validation files which are now stored in S3
bourque Jun 13, 2025
4d07f74
PR review suggestions from our robot overlords
bourque Jun 13, 2025
ec49769
PR review suggestions
bourque Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed ...s/codice/data/validation/imap_codice_l1a_hi-counters-aggregated_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ests/codice/data/validation/imap_codice_l1a_hi-counters-singles_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ocessing/tests/codice/data/validation/imap_codice_l1a_hi-ialirt_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...processing/tests/codice/data/validation/imap_codice_l1a_hi-omni_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...sing/tests/codice/data/validation/imap_codice_l1a_hi-priorities_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...essing/tests/codice/data/validation/imap_codice_l1a_hi-sectored_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed imap_processing/tests/codice/data/validation/imap_codice_l1a_hskp_20241110193622_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...s/codice/data/validation/imap_codice_l1a_lo-counters-aggregated_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ests/codice/data/validation/imap_codice_l1a_lo-counters-singles_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ocessing/tests/codice/data/validation/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ing/tests/codice/data/validation/imap_codice_l1a_lo-nsw-angular_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ng/tests/codice/data/validation/imap_codice_l1a_lo-nsw-priority_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ing/tests/codice/data/validation/imap_codice_l1a_lo-nsw-species_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ..._processing/tests/codice/data/validation/imap_codice_l1a_lo-pha_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...sing/tests/codice/data/validation/imap_codice_l1a_lo-sw-angular_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...ing/tests/codice/data/validation/imap_codice_l1a_lo-sw-priority_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
Binary file removed ...sing/tests/codice/data/validation/imap_codice_l1a_lo-sw-species_20241110193700_v0.0.0.cdf
View file
Open in desktop
Binary file not shown.
7 changes: 1 addition & 6 deletions imap_processing/tests/codice/test_codice_l1a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from imap_processing.cdf.utils import load_cdf, write_cdf
from imap_processing.codice import constants
from imap_processing.codice.codice_l1a import process_codice_l1a
from imap_processing.tests.conftest import _download_external_data, _test_data_paths

from .conftest import TEST_L0_FILE, VALIDATION_DATA

Expand Down Expand Up @@ -119,18 +118,14 @@


@pytest.fixture(scope="session")
def test_l1a_data() -> xr.Dataset:
def test_l1a_data() -> list[xr.Dataset]:
"""Return a ``xarray`` dataset containing test data.

Returns
-------
processed_datasets : list[xarray.Dataset]
A list of ``xarray`` datasets containing the test data
"""
# Make sure we have the data available here. This test collection gets
# skipped at the module level if the mark isn't present. We can't decorate
# a fixture, so add the needed call directly here instead.
_download_external_data(_test_data_paths())
processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE)

return processed_datasets
Expand Down
298 changes: 5 additions & 293 deletions imap_processing/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from imap_processing.cdf.utils import load_cdf
from imap_processing.spice import config as spice_config
from imap_processing.spice.time import TTJ2000_EPOCH, met_to_ttj2000ns
from imap_processing.tests.external_test_data_config import EXTERNAL_TEST_DATA


@pytest.fixture(autouse=True)
Expand Down Expand Up @@ -103,10 +104,10 @@ def _download_external_kernels(spice_test_data_path):

@pytest.fixture(scope="session")
def _download_test_data():
_download_external_data(_test_data_paths())
_download_external_data()


def _download_external_data(test_data_path_list):
def _download_external_data():
"""This fixture downloads externally-located test data files into a specific
location. The list of files and their storage locations are specified in
the `test_data_paths` parameter, which is a list of tuples; the zeroth
Expand All @@ -116,7 +117,7 @@ def _download_external_data(test_data_path_list):
logger = logging.getLogger(__name__)

api_path = "https://api.dev.imap-mission.com/download/test_data/"
for test_data_path in test_data_path_list:
for test_data_path in EXTERNAL_TEST_DATA:
source = api_path + test_data_path[0]
destination = test_data_path[1]
Comment thread
bourque marked this conversation as resolved.
Outdated

Expand All @@ -134,295 +135,6 @@ def _download_external_data(test_data_path_list):
logger.info(f"File already exists: {destination}")


def _test_data_paths():
"""Defines a list of test data files to download from the AWS S3 bucket
and the corresponding location in which to store the downloaded file"""
test_data_path_list = [
(
"apid_478.bin",
imap_module_directory / "tests" / "ialirt" / "data" / "l0" / "apid_478.bin",
),
(
"imap_codice_l0_raw_20241110_v001.pkts",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l0_raw_20241110_v001.pkts",
),
(
"imap_codice_l1a_hi-pha_20241110193700_v0.0.0.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "validation"
/ "imap_codice_l1a_hi-pha_20241110193700_v0.0.0.cdf",
),
(
"imap_codice_l1a_hi-counters-aggregated_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-counters-aggregated_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-counters-singles_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-counters-singles_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-ialirt_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-ialirt_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-omni_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-omni_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-pha_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-pha_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-priority_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-priority_20241110_v999.cdf",
),
(
"imap_codice_l1a_hi-sectored_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hi-sectored_20241110_v999.cdf",
),
(
"imap_codice_l1a_hskp_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_hskp_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-counters-aggregated_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-counters-aggregated_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-counters-singles_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-counters-singles_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-ialirt_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-ialirt_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-nsw-angular_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-nsw-angular_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-nsw-priority_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-nsw-priority_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-nsw-species_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-nsw-species_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-pha_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-pha_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-sw-angular_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-sw-angular_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-sw-priority_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-sw-priority_20241110_v999.cdf",
),
(
"imap_codice_l1a_lo-sw-species_20241110_v999.cdf",
imap_module_directory
/ "tests"
/ "codice"
/ "data"
/ "imap_codice_l1a_lo-sw-species_20241110_v999.cdf",
),
(
"imap_hi_l1a_45sensor-de_20250415_v999.cdf",
imap_module_directory
/ "tests"
/ "hi"
/ "data"
/ "l1"
/ "imap_hi_l1a_45sensor-de_20250415_v999.cdf",
),
(
"imap_hi_l1b_45sensor-de_20250415_v999.cdf",
imap_module_directory
/ "tests"
/ "hi"
/ "data"
/ "l1"
/ "imap_hi_l1b_45sensor-de_20250415_v999.cdf",
),
(
"imap_hi_l1c_45sensor-pset_20250415_v999.cdf",
imap_module_directory
/ "tests"
/ "hi"
/ "data"
/ "l1"
/ "imap_hi_l1c_45sensor-pset_20250415_v999.cdf",
),
(
"idex_l1a_validation_file.h5",
imap_module_directory
/ "tests"
/ "idex"
/ "test_data"
/ "idex_l1a_validation_file.h5",
),
(
"idex_l1b_validation_file.h5",
imap_module_directory
/ "tests"
/ "idex"
/ "test_data"
/ "idex_l1b_validation_file.h5",
),
(
"IMAP-Ultra45_r1_L1_V0_shortened.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "IMAP-Ultra45_r1_L1_V0_shortened.csv",
),
(
"imap_ultra_l0_raw_20260924_v001.pkts",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l0"
/ "imap_ultra_l0_raw_20260924_v001.pkts",
),
(
"imap_ultra_l1b_45sensor-de_20240207_v999.cdf",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "imap_ultra_l1b_45sensor-de_20240207_v999.cdf",
),
(
"ultra-90_raw_event_data_shortened.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "ultra-90_raw_event_data_shortened.csv",
),
(
"imap_ultra_l1c-90sensor-efficiencies_20250101_v000.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "imap_ultra_l1c-90sensor-efficiencies_20250101_v000.csv",
),
(
"imap_ultra_l1c-90sensor-gf_20250101_v000.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "imap_ultra_l1c-90sensor-gf_20250101_v000.csv",
),
(
"imap_ultra_l1c-90sensor-dps-exposure_20250101_v000.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "imap_ultra_l1c-90sensor-dps-exposure_20250101_v000.csv",
),
(
"imap_ultra_l1b-45sensor-logistic-interpolation_20250101_v000.csv",
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l1"
/ "imap_ultra_l1b-45sensor-logistic-interpolation_20250101_v000.csv",
),
]

return test_data_path_list


def pytest_collection_modifyitems(items):
"""
The use of this hook allows modification of test `Items` after tests have
Expand Down Expand Up @@ -905,4 +617,4 @@ def mag_test_l1b_calibration_data():
if __name__ == "__main__":
# This is to enable downloading files easier by letting us
# run this file directly
_download_external_data(_test_data_paths())
_download_external_data()
Loading