From a407ada92cac1152635e48ab6a6088d58915af58 Mon Sep 17 00:00:00 2001 From: nick-gorman <40549624+nick-gorman@users.noreply.github.com> Date: Tue, 18 Oct 2022 19:52:38 +1100 Subject: [PATCH] fix date generator, update tests, update version number, relative imports for gui Date generator assumed that there was a fixed window of missing data, but actually, this is a changing window. Downloader now gives a warning about missing bid tables when they can't be downloaded. --- .gitignore | 1 + e.py | 12 --------- nemosis/date_generators.py | 19 ++----------- nemosis/downloader.py | 13 ++++++--- nemosis/gui.py | 6 ++--- setup.py | 2 +- tests/test_date_generators.py | 51 +++++++++-------------------------- 7 files changed, 28 insertions(+), 76 deletions(-) delete mode 100644 e.py diff --git a/.gitignore b/.gitignore index ba2e0b2..dd6e276 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ NEMOSIS.exe nemosis/smoke_tests.py nemosis/check_new_bid_table_functionality.py *.pyc +smoke.py diff --git a/e.py b/e.py deleted file mode 100644 index 735a198..0000000 --- a/e.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging - -from nemosis import dynamic_data_compiler - -logging.getLogger("nemosis").setLevel(logging.WARNING) - -start_time = '2017/01/01 00:00:00' -end_time = '2017/01/01 00:05:00' -table = 'DISPATCHPRICE' -raw_data_cache = 'D:/nemosis_cache' - -price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache) \ No newline at end of file diff --git a/nemosis/date_generators.py b/nemosis/date_generators.py index b88e53a..5e8b4db 100644 --- a/nemosis/date_generators.py +++ b/nemosis/date_generators.py @@ -120,21 +120,6 @@ def bid_table_gen(start_time, end_time): and year == end_year ): continue - if int(year) == 2021 and int(month) == 4 and int(day) == 1: - logger.warning( - "Offer data for 2021/04/01 is known to be missing from the AEMO public \n" - "archive, explicitly skipping. This file would also contain data for the first 4 hr of \n" - + "2021/04/02 so that data will also be missing from the returned dataframe." - ) - else: - yield str(year), month, str(day).zfill(2), None - + yield str(year), month, str(day).zfill(2), None else: - if int(year) == 2021 and int(month) == 3: - logger.warning( - "Offer data for March 2021 is known to be missing from the AEMO public \n" - "archive, explicitly skipping. This file would also contain data for the first 4 hr of \n" - + "2021/04/01 so that data will also be missing from the returned dataframe." - ) - else: - yield str(year), month, None, None + yield str(year), month, None, None diff --git a/nemosis/downloader.py b/nemosis/downloader.py index 6f0efb6..38f8a64 100644 --- a/nemosis/downloader.py +++ b/nemosis/downloader.py @@ -38,10 +38,15 @@ def run_bid_tables(year, month, day, index, filename_stub, down_load_to): if day is None: run(year, month, day, index, filename_stub, down_load_to) else: - _download_and_unpack_bid_move_complete_files( - year, month, day, index, filename_stub, down_load_to - ) - + try: + _download_and_unpack_bid_move_complete_files( + year, month, day, index, filename_stub, down_load_to + ) + except Exception: + logger.warning(f"{filename_stub} not downloaded. This is likely because this file is not being hosted \n" + + "online by AEMO. You can check this url to confirm: \n" + + "https://www.nemweb.com.au/REPORTS/Archive/Bidmove_Complete/. If the file is available but \n" + "this warning persists please contact the NEMOSIS maintainers.") def _download_and_unpack_bid_move_complete_files( year, month, day, index, filename_stub, down_load_to diff --git a/nemosis/gui.py b/nemosis/gui.py index 3858c5f..460dc68 100644 --- a/nemosis/gui.py +++ b/nemosis/gui.py @@ -1,6 +1,6 @@ -from . import rows -from . import defaults -from . import data_fetch_methods +from nemosis import rows +from nemosis import defaults +from nemosis import data_fetch_methods import pandas as pd import tkinter as tk import tkinter.ttk as ttk diff --git a/setup.py b/setup.py index 04b13f0..4aee71f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="nemosis", - version="3.1.0", + version="3.2.0", author="Nicholas Gorman, Abhijith Prakash", author_email="n.gorman305@gmail.com", description="A tool for accessing AEMO data.", diff --git a/tests/test_date_generators.py b/tests/test_date_generators.py index aef9dd3..c6d3609 100644 --- a/tests/test_date_generators.py +++ b/tests/test_date_generators.py @@ -185,7 +185,6 @@ def test_change_from_months_to_days(self): end_time = datetime.strptime("2021/04/03 00:00:00", "%Y/%m/%d %H:%M:%S") gen = date_generators.bid_table_gen(start_time, end_time) times = [(year, month, day, index) for year, month, day, index in gen] - # Note we expect the 1st of april to be skipped self.assertEqual(times[0][0], "2021") self.assertEqual(times[0][1], "01") self.assertEqual(times[0][2], None) @@ -194,49 +193,23 @@ def test_change_from_months_to_days(self): self.assertEqual(times[1][1], "02") self.assertEqual(times[1][2], None) self.assertEqual(times[1][3], None) - # Data for march and the first of april is missing from the AEMO website so we don't generate the dates - # for these times. self.assertEqual(times[2][0], "2021") - self.assertEqual(times[2][1], "04") - self.assertEqual(times[2][2], "02") + self.assertEqual(times[2][1], "03") + self.assertEqual(times[2][2], None) self.assertEqual(times[2][3], None) self.assertEqual(times[3][0], "2021") self.assertEqual(times[3][1], "04") - self.assertEqual(times[3][2], "03") + self.assertEqual(times[3][2], "01") self.assertEqual(times[3][3], None) - self.assertEqual(len(times), 4) - - def test_day_given_in_april_2021(self): - start_time = datetime.strptime("2021/04/01 00:00:00", "%Y/%m/%d %H:%M:%S") - end_time = datetime.strptime("2021/04/03 00:00:00", "%Y/%m/%d %H:%M:%S") - gen = date_generators.bid_table_gen(start_time, end_time) - times = [(year, month, day, index) for year, month, day, index in gen] - # Note we expect the 1st of april to be skipped - self.assertEqual(times[0][0], "2021") - self.assertEqual(times[0][1], "04") - self.assertEqual(times[0][2], "02") - self.assertEqual(times[0][3], None) - self.assertEqual(times[1][0], "2021") - self.assertEqual(times[1][1], "04") - self.assertEqual(times[1][2], "03") - self.assertEqual(times[1][3], None) - self.assertEqual(len(times), 2) - - def test_include_previous_market_day(self): - start_time = datetime.strptime("2021/05/10 01:00:00", "%Y/%m/%d %H:%M:%S") - end_time = datetime.strptime("2021/05/10 05:00:00", "%Y/%m/%d %H:%M:%S") - gen = date_generators.bid_table_gen(start_time, end_time) - times = [(year, month, day, index) for year, month, day, index in gen] - # Note we expect the 1st of april to be skipped - self.assertEqual(times[0][0], "2021") - self.assertEqual(times[0][1], "05") - self.assertEqual(times[0][2], "09") - self.assertEqual(times[0][3], None) - self.assertEqual(times[1][0], "2021") - self.assertEqual(times[1][1], "05") - self.assertEqual(times[1][2], "10") - self.assertEqual(times[1][3], None) - self.assertEqual(len(times), 2) + self.assertEqual(times[4][0], "2021") + self.assertEqual(times[4][1], "04") + self.assertEqual(times[4][2], "02") + self.assertEqual(times[4][3], None) + self.assertEqual(times[5][0], "2021") + self.assertEqual(times[5][1], "04") + self.assertEqual(times[5][2], "03") + self.assertEqual(times[5][3], None) + self.assertEqual(len(times), 6) def test_include_previous_month_if_1st_market_day_of_month(self): start_time = datetime.strptime("2021/05/01 05:00:00", "%Y/%m/%d %H:%M:%S")