diff --git a/changelog/158.feature.rst b/changelog/158.feature.rst new file mode 100644 index 0000000..0ba16a7 --- /dev/null +++ b/changelog/158.feature.rst @@ -0,0 +1 @@ +Restored the WIND/WAVES ``Fido`` client `~radiospectra.net.sources.wind.WAVESClient` and updated it to use NASA SPDF server and altered paths and filenames. diff --git a/radiospectra/net/__init__.py b/radiospectra/net/__init__.py index 3fba67b..bd1c570 100644 --- a/radiospectra/net/__init__.py +++ b/radiospectra/net/__init__.py @@ -4,11 +4,13 @@ from radiospectra.net.sources.ilofar import ILOFARMode357Client from radiospectra.net.sources.psp import RFSClient from radiospectra.net.sources.rstn import RSTNClient +from radiospectra.net.sources.wind import WAVESClient __all__ = [ "eCALLISTOClient", "EOVSAClient", "RFSClient", "RSTNClient", + "WAVESClient", "ILOFARMode357Client", ] diff --git a/radiospectra/net/sources/tests/test_wind_client.py b/radiospectra/net/sources/tests/test_wind_client.py new file mode 100644 index 0000000..cd8f744 --- /dev/null +++ b/radiospectra/net/sources/tests/test_wind_client.py @@ -0,0 +1,99 @@ +from datetime import datetime +from unittest import mock + +import numpy as np +import pytest + +import astropy.units as u + +from sunpy.net import Fido +from sunpy.net import attrs as a + +from radiospectra.net.sources.wind import WAVESClient + +MOCK_PATH = "sunpy.net.scraper.urlopen" + + +@pytest.fixture +def client(): + return WAVESClient() + + +@pytest.mark.remote_data +def test_fido(): + atr = a.Time("2020/01/02", "2020/01/03") + res = Fido.search(atr, a.Instrument("WAVES")) + + assert isinstance(res[0].client, WAVESClient) + assert len(res[0]) == 4 + + +# Taken from https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad1_idl_binary/2020/ +http_cont_rad1 = """ +wind_waves_rad1_20200102.R1 +wind_waves_rad1_20200103.R1 +""" + +# Taken from https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad2_idl_binary/2020/ +http_cont_rad2 = """ +wind_waves_rad2_20200102.R2 +wind_waves_rad2_20200103.R2 +""" + + +@pytest.fixture +def html_responses(): + return [http_cont_rad1, http_cont_rad2] + + +@mock.patch(MOCK_PATH) +def test_waves_client(mock_urlopen, client, html_responses): + mock_urlopen.return_value.read = mock.MagicMock() + mock_urlopen.return_value.read.side_effect = html_responses + mock_urlopen.close = mock.MagicMock(return_value=None) + atr = a.Time("2020/01/02", "2020/01/03") + query = client.search(atr) + + called_urls = [ + "https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad1_idl_binary/2020/", + "https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad2_idl_binary/2020/", + ] + assert called_urls == [call[0][0] for call in mock_urlopen.call_args_list] + assert len(query) == 4 + assert query[0]["Source"] == "WIND" + assert query[0]["Provider"] == "SPDF" + + wave = [20, 1040] * u.kHz + assert np.array_equal(query[0]["Wavelength"], wave) + assert query[0]["Start Time"].datetime == datetime(2020, 1, 2) + assert query[0]["End Time"].datetime == datetime(2020, 1, 2, 23, 59, 59, 999000) + + wave = [1075, 13825] * u.kHz + assert np.array_equal(query[3]["Wavelength"], wave) + assert query[3]["Start Time"].datetime == datetime(2020, 1, 3) + assert query[3]["End Time"].datetime == datetime(2020, 1, 3, 23, 59, 59, 999000) + + query_urls = [row["url"] for row in query] + assert ( + "https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad1_idl_binary/2020/wind_waves_rad1_20200102.R1" in query_urls + ) + assert ( + "https://spdf.gsfc.nasa.gov/pub/data/wind/waves/rad2_idl_binary/2020/wind_waves_rad2_20200103.R2" in query_urls + ) + + +@pytest.mark.parametrize( + ("query_wave", "receivers"), + [ + (a.Wavelength(1 * u.GHz, 2 * u.GHz), []), + (a.Wavelength(1 * u.Hz, 2 * u.Hz), []), + (a.Wavelength(20 * u.kHz, 150 * u.kHz), ["rad1"]), + (a.Wavelength(1.5 * u.MHz, 15 * u.MHz), ["rad2"]), + (a.Wavelength(5 * u.MHz, 10 * u.MHz), ["rad2"]), + (a.Wavelength(100 * u.Hz, 100 * u.kHz), ["rad1"]), + (a.Wavelength(20 * u.kHz, 15 * u.MHz), ["rad1", "rad2"]), + (a.Wavelength(5 * u.kHz, 20 * u.MHz), ["rad1", "rad2"]), + ], +) +def test_check_wavelength(query_wave, receivers, client): + assert set(client._check_wavelengths(query_wave)) == set(receivers) diff --git a/radiospectra/net/sources/wind.py b/radiospectra/net/sources/wind.py new file mode 100644 index 0000000..bb9dcaa --- /dev/null +++ b/radiospectra/net/sources/wind.py @@ -0,0 +1,145 @@ +import astropy.units as u + +from sunpy.net import attrs as a +from sunpy.net.dataretriever.client import GenericClient, QueryResponse +from sunpy.net.scraper import Scraper +from sunpy.time.timerange import TimeRange + +__all__ = ["WAVESClient"] + +RECEIVER_FREQUENCIES = { + "rad1": a.Wavelength(20 * u.kHz, 1040 * u.kHz), + "rad2": a.Wavelength(1.075 * u.MHz, 13.825 * u.MHz), +} + +RECEIVER_EXTENSIONS = { + "rad1": "R1", + "rad2": "R2", +} + + +class WAVESClient(GenericClient): + """ + Provides access to WIND/WAVES IDL binary data hosted at + `NASA Goddard Space Physics Data Facility (SPDF) + `__. + + Examples + -------- + >>> import radiospectra.net + >>> from sunpy.net import Fido, attrs as a + >>> results = Fido.search(a.Time("2020/01/01", "2020/01/02"), + ... a.Instrument("WAVES")) # doctest: +REMOTE_DATA + >>> results # doctest: +REMOTE_DATA + + Results from 1 Provider: + + 4 Results from the WAVESClient: + + Start Time End Time Instrument Source Provider Wavelength + kHz + ----------------------- ----------------------- ---------- ------ -------- ----------------- + 2020-01-01 00:00:00.000 2020-01-01 23:59:59.999 WAVES WIND SPDF 20.0 .. 1040.0 + 2020-01-02 00:00:00.000 2020-01-02 23:59:59.999 WAVES WIND SPDF 20.0 .. 1040.0 + 2020-01-01 00:00:00.000 2020-01-01 23:59:59.999 WAVES WIND SPDF 1075.0 .. 13825.0 + 2020-01-02 00:00:00.000 2020-01-02 23:59:59.999 WAVES WIND SPDF 1075.0 .. 13825.0 + + + """ + + pattern = ( + r"https://spdf.gsfc.nasa.gov/pub/data/wind/waves/{receiver}_idl_binary/{year_path}/" + r"wind_waves_{receiver}_{{year:4d}}{{month:2d}}{{day:2d}}.{ext}" + ) + + @classmethod + def _check_wavelengths(cls, wavelength): + """ + Check for overlap between given wavelength and receiver frequency coverage + defined in ``RECEIVER_FREQUENCIES``. + + Parameters + ---------- + wavelength : `sunpy.net.attrs.Wavelength` + Input wavelength range to check + + Returns + ------- + `list` + List of receivers names or empty list if no overlap + """ + # Input wavelength range is completely contained in one receiver range + receivers = [k for k, v in RECEIVER_FREQUENCIES.items() if wavelength in v] + # If not defined need to continue + if not receivers: + # Overlaps but not contained in, either max in low-frequency or min in high-frequency receiver + if wavelength.min in RECEIVER_FREQUENCIES["rad2"] or wavelength.max in RECEIVER_FREQUENCIES["rad2"]: + receivers.append("rad2") + if wavelength.min in RECEIVER_FREQUENCIES["rad1"] or wavelength.max in RECEIVER_FREQUENCIES["rad1"]: + receivers.append("rad1") + # min in rad1 and max in rad2 + # min and max of combined rad1 and rad2 contained in given wavelength range + if a.Wavelength(RECEIVER_FREQUENCIES["rad1"].min, RECEIVER_FREQUENCIES["rad2"].max) in wavelength: + receivers = ["rad1", "rad2"] + return receivers + + def search(self, *args, **kwargs): + """ + Query this client for a list of results. + + Parameters + ---------- + *args: `tuple` + `sunpy.net.attrs` objects representing the query. + **kwargs: `dict` + Any extra keywords to refine the search. + + Returns + ------- + A `QueryResponse` instance containing the query result. + """ + matchdict = self._get_match_dict(*args, **kwargs) + req_wave = matchdict.get("Wavelength", None) + receivers = RECEIVER_FREQUENCIES.keys() + if req_wave is not None: + receivers = self._check_wavelengths(req_wave) + + metalist = [] + start_year = matchdict["Start Time"].datetime.year + end_year = matchdict["End Time"].datetime.year + tr = TimeRange(matchdict["Start Time"], matchdict["End Time"]) + for receiver in receivers: + for year in range(start_year, end_year + 1): + pattern = ( + self.pattern.replace("{receiver}", receiver) + .replace("{ext}", RECEIVER_EXTENSIONS[receiver]) + .replace("{year_path}", str(year)) + ) + scraper = Scraper(format=pattern) + filesmeta = scraper._extract_files_meta(tr) + for i in filesmeta: + i["receiver"] = receiver + rowdict = self.post_search_hook(i, matchdict) + metalist.append(rowdict) + + return QueryResponse(metalist, client=self) + + def post_search_hook(self, exdict, matchdict): + """ + Convert receiver metadata to the receiver frequency ranges. + """ + rowdict = super().post_search_hook(exdict, matchdict) + receiver = rowdict.pop("receiver") + fr = RECEIVER_FREQUENCIES[receiver] + rowdict["Wavelength"] = u.Quantity([float(fr.min.value), float(fr.max.value)], unit=fr.unit) + return rowdict + + @classmethod + def register_values(cls): + adict = { + a.Instrument: [("WAVES", "WIND - WAVES")], + a.Source: [("WIND", "WIND")], + a.Provider: [("SPDF", "NASA Goddard Space Physics Data Facility")], + a.Wavelength: [("*")], + } + return adict diff --git a/radiospectra/spectrogram/sources/tests/test_waves.py b/radiospectra/spectrogram/sources/tests/test_waves.py index 6680346..4de0a5c 100644 --- a/radiospectra/spectrogram/sources/tests/test_waves.py +++ b/radiospectra/spectrogram/sources/tests/test_waves.py @@ -11,6 +11,7 @@ from radiospectra.spectrogram import Spectrogram from radiospectra.spectrogram.sources import WAVESSpectrogram +from radiospectra.spectrogram.spectrogram_factory import SpectrogramFactory @mock.patch("radiospectra.spectrogram.spectrogram_factory.parse_path") @@ -63,3 +64,13 @@ def test_waves_rad2(parse_path_moc): assert spec.end_time.datetime == datetime(2020, 11, 28, 23, 59) assert spec.wavelength.min == 1.075 * u.MHz assert spec.wavelength.max == 13.825 * u.MHz + + +@mock.patch("radiospectra.spectrogram.spectrogram_factory.readsav") +def test_waves_prefixed_filename_parses_date(readsav_mock): + data_array = np.zeros((256, 1441)) + readsav_mock.return_value = {"arrayb": data_array} + + _, meta = SpectrogramFactory._read_idl_sav(Path("wind_waves_rad1_20200711.R1"), instrument="waves") + + assert meta["start_time"].isot == "2020-07-11T00:00:00.000" diff --git a/radiospectra/spectrogram/spectrogram_factory.py b/radiospectra/spectrogram/spectrogram_factory.py index b233878..70d3fad 100644 --- a/radiospectra/spectrogram/spectrogram_factory.py +++ b/radiospectra/spectrogram/spectrogram_factory.py @@ -680,7 +680,7 @@ def _read_idl_sav(file, instrument=None): # bg which is already subtracted from data ? bg = data_array[:, -1] data = data_array[:, :-1] - start_time = Time.strptime(file.stem, "%Y%m%d") + start_time = Time.strptime(file.stem.split("_")[-1], "%Y%m%d") end_time = start_time + 86399 * u.s times = start_time + (np.arange(1440) * 60 + 30) * u.s meta = {