From 369970fce2fbc22aa3fb6fc84a8457c1ac785adb Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 14 May 2021 15:25:09 -0500 Subject: [PATCH 1/7] BUG: Corrected multi-day support --- pysatSpaceWeather/instruments/sw_dst.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index 5c3d39b1..e38d05ca 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -112,25 +112,30 @@ def load(fnames, tag=None, inst_id=None): """ - data = pds.DataFrame() + all_data = [] for filename in fnames: - # need to remove date appended to dst filename + # Need to remove date appended to dst filename fname = filename[0:-11] - all_data = [] + with open(fname) as open_f: lines = open_f.readlines() idx = 0 - # check if all lines are good + + # Check if all lines are good max_lines = 0 for line in lines: if len(line) > 1: max_lines += 1 + + # Prep memory yr = np.zeros(max_lines * 24, dtype=int) mo = np.zeros(max_lines * 24, dtype=int) day = np.zeros(max_lines * 24, dtype=int) ut = np.zeros(max_lines * 24, dtype=int) dst = np.zeros(max_lines * 24, dtype=int) + + # Read data for line in lines: if len(line) > 1: temp_year = int(line[14:16] + line[3:5]) @@ -148,21 +153,25 @@ def load(fnames, tag=None, inst_id=None): dst[idx:idx + 24] = temp2 idx += 24 + # Prep datetime index for the data start = dt.datetime(yr[0], mo[0], day[0], ut[0]) stop = dt.datetime(yr[-1], mo[-1], day[-1], ut[-1]) dates = pds.date_range(start, stop, freq='H') new_data = pds.DataFrame(dst, index=dates, columns=['dst']) - # pull out specific day + + # Pull out specific day new_date = dt.datetime.strptime(filename[-10:], '%Y-%m-%d') idx, = np.where((new_data.index >= new_date) & (new_data.index < new_date + pds.DateOffset(days=1))) new_data = new_data.iloc[idx, :] - # add specific day to all data loaded for filenames + + # Add specific day to all data loaded for filenames all_data.append(new_data) - # combine data together - data = pds.concat(all_data, sort=True, axis=0) + + # Combine data together + data = pds.concat(all_data, sort=True, axis=0) return data, pysat.Meta() From 00b5adbf1f5a0506ff01456c4a3bd63f86be90c0 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 14 May 2021 16:49:47 -0500 Subject: [PATCH 2/7] ENH: Improved load efficiency --- pysatSpaceWeather/instruments/sw_dst.py | 40 +++++++++++++++++-------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index e38d05ca..db9249e0 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -114,10 +114,20 @@ def load(fnames, tag=None, inst_id=None): all_data = [] + # Dst data is actually stored by year but users can load by day. + # Extract the actual dates from the input list of filenames as + # well as the names of the actual files. + fdates = [] + ufnames = [] for filename in fnames: - # Need to remove date appended to dst filename - fname = filename[0:-11] + fdates.append(dt.datetime.strptime(filename[-10:], '%Y-%m-%d')) + ufnames.append(filename[0:-11]) + # Get unique filenames that map to actual data + ufnames = np.unique(ufnames).tolist() + + # Load unique files + for fname in ufnames: with open(fname) as open_f: lines = open_f.readlines() idx = 0 @@ -153,26 +163,32 @@ def load(fnames, tag=None, inst_id=None): dst[idx:idx + 24] = temp2 idx += 24 - # Prep datetime index for the data + # Prep datetime index for the data and create DataFrame start = dt.datetime(yr[0], mo[0], day[0], ut[0]) stop = dt.datetime(yr[-1], mo[-1], day[-1], ut[-1]) dates = pds.date_range(start, stop, freq='H') - new_data = pds.DataFrame(dst, index=dates, columns=['dst']) - # Pull out specific day - new_date = dt.datetime.strptime(filename[-10:], '%Y-%m-%d') - idx, = np.where((new_data.index >= new_date) - & (new_data.index < new_date - + pds.DateOffset(days=1))) - new_data = new_data.iloc[idx, :] - - # Add specific day to all data loaded for filenames + # Add to all data loaded for filenames all_data.append(new_data) # Combine data together data = pds.concat(all_data, sort=True, axis=0) + # Pull out requested days + data = data.iloc[data.index >= fdates[0], :] + data = data.iloc[data.index < fdates[-1] + pds.DateOffset(days=1), :] + + # Create metadata + meta = pysat.Meta() + meta['dst'] = {meta.labels.units: 'nT', + meta.labels.name: 'Dst', + meta.labels.notes: 'Downloaded from NOAA/NGDC', + meta.labels.desc: 'Disturbance storm-time index', + meta.labels.fill_val: np.nan, + meta.labels.min_val: -np.inf, + meta.labels.max_val: np.inf} + return data, pysat.Meta() From ea83db7c6df40a79ee95abcf0162b4c75fb6e4ae Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 14 May 2021 16:50:35 -0500 Subject: [PATCH 3/7] BUG: Returned meta --- pysatSpaceWeather/instruments/sw_dst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index db9249e0..704bbd82 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -189,7 +189,7 @@ def load(fnames, tag=None, inst_id=None): meta.labels.min_val: -np.inf, meta.labels.max_val: np.inf} - return data, pysat.Meta() + return data, meta def list_files(tag=None, inst_id=None, data_path=None, format_str=None): From de06a5a32bbdd2a74ed063aaaf24f0084dd3b473 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 14 May 2021 17:05:28 -0500 Subject: [PATCH 4/7] DOC: Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e266a175..d7e223bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Deprecated F10.7 instrument tag 'all' and '', replacing them with 'historic' * Improved F10.7 instrument routines by combining similar code blocks * Fixed F10.7 load/list bugs that lead to duplicate data entries +* Fixed Dst load bugs when loading multiple days of data [0.0.3] - 2021-01-15 -------------------- From d104ba257d7f1add26c144ef2494735896ce7021 Mon Sep 17 00:00:00 2001 From: Russell Stoneback Date: Fri, 14 May 2021 17:15:51 -0500 Subject: [PATCH 5/7] DOC: Removed old comment --- pysatSpaceWeather/instruments/sw_dst.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index 704bbd82..5fae7e1e 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -14,9 +14,6 @@ ---- Will only work until 2057. -Download method should be invoked on a yearly frequency, -dst.download(date1, date2, freq='AS') - This material is based upon work supported by the National Science Foundation under Grant Number 1259508. From 5b6c672e7ec4d2beff0e0b5dda8c249904a309a7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 17 May 2021 08:46:40 -0400 Subject: [PATCH 6/7] ENH: updated Dst tag Updated Dst tag to be informative, and easily allow for future additions of historic data from Kyoto. --- pysatSpaceWeather/instruments/methods/dst.py | 5 +++-- pysatSpaceWeather/instruments/sw_dst.py | 23 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/dst.py b/pysatSpaceWeather/instruments/methods/dst.py index 5ad1afbc..cf1c2a9c 100644 --- a/pysatSpaceWeather/instruments/methods/dst.py +++ b/pysatSpaceWeather/instruments/methods/dst.py @@ -16,7 +16,8 @@ def acknowledgements(name, tag): """ - ackn = {'dst': {'': 'Dst is maintained at NCEI (formerly NGDC) at NOAA'}} + ackn = {'dst': + {'noaa': 'Dst is maintained at NCEI (formerly NGDC) at NOAA'}} return ackn[name][tag] @@ -33,7 +34,7 @@ def references(name, tag): """ - refs = {'dst': {'': ''.join([ + refs = {'dst': {'noaa': ''.join([ 'See referenece list and publication at: Sugiura M. and T. Kamei, ' 'http://wdc.kugi.kyoto-u.ac.jp/dstdir/dst2/onDstindex.html, ', 'last updated June 1991, accessed Dec 2020'])}} diff --git a/pysatSpaceWeather/instruments/sw_dst.py b/pysatSpaceWeather/instruments/sw_dst.py index 5fae7e1e..0bf15531 100644 --- a/pysatSpaceWeather/instruments/sw_dst.py +++ b/pysatSpaceWeather/instruments/sw_dst.py @@ -8,7 +8,9 @@ name 'dst' tag - None supported + 'noaa' - Historic Dst data coalated by and maintained by NOAA/NCEI +inst_id + '' Note ---- @@ -30,6 +32,7 @@ import pandas as pds import pysat + from pysatSpaceWeather.instruments.methods import dst as mm_dst logger = pysat.logger @@ -39,16 +42,16 @@ platform = 'sw' name = 'dst' -tags = {'': ''} -inst_ids = {'': ['']} +tags = {'noaa': 'Historic Dst data coalated by and maintained by NOAA/NCEI'} +inst_ids = {'': [tag for tag in tags]} # ---------------------------------------------------------------------------- # Instrument test attributes -_test_dates = {'': {'': dt.datetime(2007, 1, 1)}} +_test_dates = {'': {'noaa': dt.datetime(2007, 1, 1)}} # Other tags assumed to be True -_test_download_travis = {'': {'': False}} +_test_download_travis = {'': {'noaa': False}} # ---------------------------------------------------------------------------- # Instrument methods @@ -56,10 +59,6 @@ def init(self): """Initializes the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ self.acknowledgements = mm_dst.acknowledgements(self.name, self.tag) @@ -106,7 +105,6 @@ def load(fnames, tag=None, inst_id=None): ---- Called by pysat. Not intended for direct use by user. - """ all_data = [] @@ -180,7 +178,7 @@ def load(fnames, tag=None, inst_id=None): meta = pysat.Meta() meta['dst'] = {meta.labels.units: 'nT', meta.labels.name: 'Dst', - meta.labels.notes: 'Downloaded from NOAA/NGDC', + meta.labels.notes: tags[tag], meta.labels.desc: 'Disturbance storm-time index', meta.labels.fill_val: np.nan, meta.labels.min_val: -np.inf, @@ -219,7 +217,7 @@ def list_files(tag=None, inst_id=None, data_path=None, format_str=None): """ if data_path is not None: - if tag == '': + if tag == 'noaa': # files are by year, going to add date to yearly filename for # each day of the month. The load routine will load a month of # data and use the appended date to select out appropriate data. @@ -239,6 +237,7 @@ def list_files(tag=None, inst_id=None, data_path=None, format_str=None): else: raise ValueError(''.join(('A data_path must be passed to the loading ', 'routine for Dst'))) + return def download(date_array, tag, inst_id, data_path, user=None, password=None): From 741b64cf7e36cc391289ea64c46e2d033f8ad4d1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Tue, 18 May 2021 08:07:27 -0400 Subject: [PATCH 7/7] BUG: fixed meta assignment Fixed the metadata re-assignment of the fill value. --- pysatSpaceWeather/instruments/methods/ace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/methods/ace.py b/pysatSpaceWeather/instruments/methods/ace.py index 1920e8e5..48c1153d 100644 --- a/pysatSpaceWeather/instruments/methods/ace.py +++ b/pysatSpaceWeather/instruments/methods/ace.py @@ -91,7 +91,7 @@ def preprocess(inst): fill_val = inst.meta[col][inst.meta.labels.fill_val] if ~np.isnan(fill_val): inst.data[col] = inst.data[col].replace(fill_val, np.nan) - inst.meta[col][inst.meta.labels.fill_val] = np.nan + inst.meta[col] = {inst.meta.labels.fill_val: np.nan} return