-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit fd898df
Showing
10 changed files
with
1,754 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
.idea/* | ||
__pycache__/* | ||
venv/* | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,213 @@ | ||
names = {'DISPATCHLOAD': 'PUBLIC_DVD_DISPATCHLOAD', | ||
'DUDETAILSUMMARY': 'PUBLIC_DVD_DUDETAILSUMMARY', | ||
'DISPATCHCONSTRAINT': 'PUBLIC_DVD_DISPATCHCONSTRAINT', | ||
'GENCONDATA': 'PUBLIC_DVD_GENCONDATA', | ||
'DISPATCH_UNIT_SCADA': 'PUBLIC_DVD_DISPATCH_UNIT_SCADA', | ||
'DISPATCHPRICE': 'PUBLIC_DVD_DISPATCHPRICE', | ||
'SPDREGIONCONSTRAINT': 'PUBLIC_DVD_SPDREGIONCONSTRAINT', | ||
'SPDCONNECTIONPOINTCONSTRAINT': 'PUBLIC_DVD_SPDCONNECTIONPOINTCONSTRAINT', | ||
'SPDINTERCONNECTORCONSTRAINT': 'PUBLIC_DVD_SPDINTERCONNECTORCONSTRAINT', | ||
'BIDPEROFFER_D': 'PUBLIC_DVD_BIDPEROFFER_D', | ||
'DISPATCHINTERCONNECTORRES': 'PUBLIC_DVD_DISPATCHINTERCONNECTORRES', | ||
'BIDDAYOFFER_D': 'PUBLIC_DVD_BIDDAYOFFER_D', | ||
'DISPATCHREGIONSUM': 'PUBLIC_DVD_DISPATCHREGIONSUM', | ||
'FCAS_4_SECOND': 'FCAS', | ||
'ELEMENTS_FCAS_4_SECOND': 'Elements_FCAS.csv', | ||
'VARIABLES_FCAS_4_SECOND': '820-0079 csv.csv', | ||
'MASTER_REGISTRATION_LIST': 'NEM Registration and Exemption List'} | ||
|
||
return_tables = list(names.keys()) | ||
|
||
static_tables = ['ELEMENTS_FCAS_4_SECOND', 'VARIABLES_FCAS_4_SECOND', 'MASTER_REGISTRATION_LIST'] | ||
|
||
static_table_url = {'ELEMENTS_FCAS_4_SECOND': 'https://www.aemo.com.au/-/media/Files/Electricity/NEM/Data/Ancillary_Services/Elements_FCAS.csv', | ||
'VARIABLES_FCAS_4_SECOND': 'https://www.aemo.com.au/-/media/Files/CSV/820-0079-csv.csv', | ||
'MASTER_REGISTRATION_LIST': 'https://www.aemo.com.au/-/media/Files/Electricity/NEM/Participant_Information/NEM-Registration-and-Exemption-List.xls'} | ||
|
||
aemo_data_url = 'http://www.nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/{}/MMSDM_{}_{}/MMSDM_Historical_Data_SQLLoader/DATA/{}.zip' | ||
|
||
fcas_4_url ='http://www.nemweb.com.au/Reports/Current/Causer_Pays/FCAS_{}{}{}{}.zip' | ||
|
||
fcas_4_url_hist ='http://www.nemweb.com.au/Data_Archive/Wholesale_Electricity/FCAS_Causer_Pays/2017/FCAS_Causer_Pays_{}_{}/FCAS_{}{}{}{}.zip' | ||
|
||
data_url = {'DISPATCHLOAD': 'aemo_data_url', | ||
'DUDETAILSUMMARY': 'aemo_data_url', | ||
'DISPATCHCONSTRAINT': 'aemo_data_url', | ||
'GENCONDATA': 'aemo_data_url', | ||
'STATION': 'aemo_data_url', | ||
'STADUALLOC': 'aemo_data_url', | ||
'DISPATCH_UNIT_SCADA': 'aemo_data_url', | ||
'DUDETAIL': 'aemo_data_url', | ||
'GENUNITS': 'aemo_data_url', | ||
'DISPATCHPRICE': 'aemo_data_url', | ||
'PARTICIPANT': 'aemo_data_url', | ||
'SPDREGIONCONSTRAINT': 'aemo_data_url', | ||
'SPDCONNECTIONPOINTCONSTRAINT': 'aemo_data_url', | ||
'SPDINTERCONNECTORCONSTRAINT': 'aemo_data_url', | ||
'BIDPEROFFER_D': 'aemo_data_url', | ||
'DISPATCHINTERCONNECTORRES': 'aemo_data_url', | ||
'INTERCONNECTOR': 'aemo_data_url', | ||
'INTERCONNECTORCONSTRAINT': 'aemo_data_url', | ||
'MNSP_INTERCONNECTOR': 'aemo_data_url', | ||
'BIDDAYOFFER_D': 'aemo_data_url', | ||
'DISPATCHREGIONSUM': 'aemo_data_url', | ||
'MNSP_DAYOFFER': 'aemo_data_url', | ||
'MNSP_PEROFFER': 'aemo_data_url', | ||
'LOSSMODEL': 'aemo_data_url', | ||
'LOSSFACTORMODEL': 'aemo_data_url', | ||
'DISPATCHCASESOLUTION': 'aemo_data_url', | ||
'FCAS': 'fcas_4_url'} | ||
|
||
filterable_cols = ['DUID', 'REGIONID', 'STATIONID', 'PARTICIPANTID', 'STARTTYPE', 'SCHEDULE_TYPE', 'GENCONID', | ||
'BIDTYPE', 'VARIABLEID', 'INTERVENTION', 'DISPATCHMODE', 'STARTTYPE', 'CONNECTIONPOINTID', | ||
'DISPATCHTYPE', 'CONSTRAINTID', 'PREDISPATCH', 'STPASA', 'MTPASA', 'LIMITTYPE', 'STATIONNAME', | ||
'AGCFLAG', 'INTERCONNECTORID', 'NAME', 'Fuel Source - Primary', 'Fuel Source - Descriptor', | ||
'Technology Type - Primary', 'Technology Type - Descriptor'] | ||
|
||
table_columns = { | ||
|
||
'DISPATCHLOAD': ['SETTLEMENTDATE', 'DUID', 'INTERVENTION', 'DISPATCHMODE', 'AGCSTATUS', 'INITIALMW', | ||
'TOTALCLEARED', 'RAMPDOWNRATE', 'RAMPUPRATE', 'LOWER5MIN', 'LOWER60SEC', | ||
'LOWER6SEC', 'RAISE5MIN', 'RAISE60SEC', 'RAISE6SEC', 'LOWERREG', 'RAISEREG', | ||
'SEMIDISPATCHCAP', 'AVAILABILITY'], | ||
|
||
'DUDETAILSUMMARY': ['DUID', 'START_DATE', 'END_DATE', 'DISPATCHTYPE', 'CONNECTIONPOINTID', 'REGIONID', 'STATIONID', | ||
'PARTICIPANTID', 'LASTCHANGED', 'TRANSMISSIONLOSSFACTOR', 'STARTTYPE', 'DISTRIBUTIONLOSSFACTOR', | ||
'SCHEDULE_TYPE', 'MAX_RAMP_RATE_UP', 'MAX_RAMP_RATE_DOWN'], | ||
|
||
'DISPATCHCONSTRAINT': ['SETTLEMENTDATE', 'RUNNO', 'CONSTRAINTID', 'INTERVENTION', 'RHS', 'MARGINALVALUE', | ||
'VIOLATIONDEGREE', 'LASTCHANGED', 'GENCONID_EFFECTIVEDATE', 'GENCONID_VERSIONNO', 'LHS', | ||
'DISPATCHINTERVAL'], | ||
|
||
'GENCONDATA': ['GENCONID', 'EFFECTIVEDATE', 'VERSIONNO', 'CONSTRAINTTYPE', 'CONSTRAINTVALUE', 'DESCRIPTION', | ||
'GENERICCONSTRAINTWEIGHT', 'LASTCHANGED', 'DISPATCH', 'PREDISPATCH', 'STPASA', 'MTPASA', | ||
'LIMITTYPE', 'REASON'], | ||
|
||
'STATION': ['STATIONID', 'STATIONNAME', 'LASTCHANGED'], | ||
|
||
'STADUALLOC': ['DUID', 'EFFECTIVEDATE', 'STATIONID', 'VERSIONNO', 'LASTCHANGED'], | ||
|
||
'DISPATCH_UNIT_SCADA': ['SETTLEMENTDATE', 'DUID', 'SCADAVALUE'], | ||
|
||
'DUDETAIL': ['EFFECTIVEDATE', 'DUID', 'VERSIONNO', 'CONNECTIONPOINTID', 'REGISTEREDCAPACITY', 'AGCCAPABILITY', | ||
'DISPATCHTYPE', 'MAXCAPACITY', 'STARTTYPE', 'NORMALLYONFLAG', 'LASTCHANGED'], | ||
|
||
'GENUNITS': ['GENSETID', 'STATIONID', 'CDINDICATOR', 'AGCFLAG', 'REGISTEREDCAPACITY', 'DISPATCHTYPE', 'STARTTYPE', | ||
'NORMALSTATUS', 'LASTCHANGED', 'CO2E_ENERGY_SOURCE'], | ||
|
||
'DISPATCHPRICE': ['SETTLEMENTDATE', 'REGIONID', 'INTERVENTION', 'RRP', 'RAISE6SECRRP', 'RAISE60SECRRP', | ||
'RAISE5MINRRP', 'RAISEREGRRP', 'LOWER6SECRRP', 'LOWER60SECRRP', 'LOWERREGRRP', 'PRICE_STATUS'], | ||
|
||
'PARTICIPANT': ['PARTICIPANTID', 'PARTICIPANTCLASSID', 'NAME', 'LASTCHANGED'], | ||
|
||
'SPDREGIONCONSTRAINT': ['REGIONID', 'EFFECTIVEDATE', 'VERSIONNO', 'GENCONID', 'FACTOR', 'LASTCHANGED', 'BIDTYPE'], | ||
|
||
'SPDCONNECTIONPOINTCONSTRAINT': ['CONNECTIONPOINTID', 'EFFECTIVEDATE', 'VERSIONNO', 'GENCONID', 'FACTOR', 'BIDTYPE', | ||
'LASTCHANGED'], | ||
|
||
'SPDINTERCONNECTORCONSTRAINT': ['INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO', 'GENCONID', 'FACTOR', | ||
'LASTCHANGED'], | ||
|
||
'BIDPEROFFER_D': ['DUID', 'BANDAVAIL1', 'BANDAVAIL2', 'BANDAVAIL3', 'BANDAVAIL4', 'BANDAVAIL5','BANDAVAIL6', | ||
'BANDAVAIL7', 'BANDAVAIL8', 'BANDAVAIL9', 'BANDAVAIL10', 'MAXAVAIL', 'RAMPUPRATE', | ||
'RAMPDOWNRATE', 'BIDTYPE', 'SETTLEMENTDATE', 'ENABLEMENTMIN', 'ENABLEMENTMAX', 'LOWBREAKPOINT', | ||
'HIGHBREAKPOINT', 'INTERVAL_DATETIME'], | ||
'DISPATCHINTERCONNECTORRES': ['SETTLEMENTDATE', 'INTERCONNECTORID', 'DISPATCHINTERVAL', 'INTERVENTION', 'MWFLOW'], | ||
'INTERCONNECTOR': ['INTERCONNECTORID', 'REGIONFROM', 'REGIONTO', 'LASTCHANGED'], | ||
'INTERCONNECTORCONSTRAINT': ['INTERCONNECTORID', 'FROMREGIONLOSSSHARE', 'EFFECTIVEDATE', 'VERSIONNO', | ||
'LOSSCONSTANT', 'LOSSFLOWCOEFFICIENT', 'ICTYPE'], | ||
'MNSP_INTERCONNECTOR': ['INTERCONNECTORID', 'LINKID', 'FROMREGION', 'TOREGION', 'MAXCAPACITY', 'FROM_REGION_TLF', | ||
'TO_REGION_TLF', 'LHSFACTOR', 'EFFECTIVEDATE', 'VERSIONNO'], | ||
'BIDDAYOFFER_D': ['SETTLEMENTDATE', 'DUID', 'BIDTYPE', 'OFFERDATE', 'VERSIONNO', 'PRICEBAND1', 'PRICEBAND2', | ||
'PRICEBAND3', 'PRICEBAND4', 'PRICEBAND5', 'PRICEBAND6', 'PRICEBAND7', 'PRICEBAND8', | ||
'PRICEBAND9', 'PRICEBAND10', 'T1', 'T2', 'T3', 'T4'], | ||
'DISPATCHREGIONSUM': ['SETTLEMENTDATE', 'REGIONID', 'DISPATCHINTERVAL', 'INTERVENTION', 'TOTALDEMAND', | ||
'AVAILABLEGENERATION', 'AVAILABLELOAD', 'DEMANDFORECAST', 'DISPATCHABLEGENERATION', | ||
'DISPATCHABLELOAD', 'NETINTERCHANGE', 'EXCESSGENERATION', 'LOWER5MINLOCALDISPATCH', | ||
'LOWER60SECLOCALDISPATCH', 'LOWER6SECLOCALDISPATCH', 'RAISE5MINLOCALDISPATCH', | ||
'RAISE60SECLOCALDISPATCH', 'RAISE6SECLOCALDISPATCH', 'LOWERREGLOCALDISPATCH', | ||
'RAISEREGLOCALDISPATCH', 'INITIALSUPPLY', 'CLEAREDSUPPLY', 'TOTALINTERMITTENTGENERATION', | ||
'DEMAND_AND_NONSCHEDGEN', 'UIGF', 'SEMISCHEDULE_CLEAREDMW', 'SEMISCHEDULE_COMPLIANCEMW'], | ||
'MNSP_PEROFFER': ['SETTLEMENTDATE', 'OFFERDATE', 'VERSIONNO', 'PARTICIPANTID', 'LINKID', 'PERIODID', | ||
'BANDAVAIL1', 'BANDAVAIL2', 'BANDAVAIL3', 'BANDAVAIL4', 'BANDAVAIL5', 'BANDAVAIL6', | ||
'BANDAVAIL7', 'BANDAVAIL8', 'BANDAVAIL9', 'BANDAVAIL10'], | ||
'MNSP_DAYOFFER': ['SETTLEMENTDATE', 'OFFERDATE', 'VERSIONNO', 'PARTICIPANTID', 'LINKID', 'PERIODID', | ||
'PRICEBAND1', 'PRICEBAND2', 'PRICEBAND3', 'PRICEBAND4', 'PRICEBAND5', 'PRICEBAND6', | ||
'PRICEBAND7', 'PRICEBAND8', 'PRICEBAND9', 'PRICEBAND10'], | ||
'LOSSMODEL': ['EFFECTIVEDATE', 'VERSIONNO', 'INTERCONNECTORID', 'LOSSSEGMENT', 'MWBREAKPOINT'], | ||
'LOSSFACTORMODEL': ['EFFECTIVEDATE', 'VERSIONNO', 'INTERCONNECTORID', 'REGIONID', 'DEMANDCOEFFICIENT'], | ||
'DISPATCHCASESOLUTION': ['SETTLEMENTDATE', 'TOTALOBJECTIVE'], | ||
'FCAS_4_SECOND': ['TIMESTAMP', 'ELEMENTNUMBER', 'VARIABLENUMBER', 'VALUE', 'VALUEQUALITY'], | ||
'ELEMENTS_FCAS_4_SECOND': ['ELEMENTNUMBER', 'ELEMENTNAME', 'ELEMENTTYPE', 'NAME'], | ||
'VARIABLES_FCAS_4_SECOND': ['VARIABLENUMBER', 'VARIABLETYPE'], | ||
'MASTER_REGISTRATION_LIST': ['Participant', 'Station Name' ,'Region' ,'Dispatch Type', 'Category', 'Classification', | ||
'Fuel Source - Primary', 'Fuel Source - Descriptor', 'Technology Type - Primary', | ||
'Technology Type - Descriptor' , 'Aggregation', 'DUID']} | ||
|
||
table_primary_keys = {'DISPATCHCONSTRAINT': ['CONSTRAINTID', 'EFFECTIVEDATE', 'VERSIONNO'], | ||
'DUDETAILSUMMARY': ['DUID', 'START_DATE'], 'STATION': ['STATIONID'], | ||
'STADUALLOC': ['EFFECTIVEDATE', 'STATIONID', 'VERSIONNO'], | ||
'GENUNITS': ['GENSETID'], | ||
'PARTICIPANT': ['PARTICIPANTID'], | ||
'SPDREGIONCONSTRAINT': ['EFFECTIVEDATE', 'GENCONID','REGIONID', 'VERSIONNO', 'BIDTYPE'], | ||
'SPDCONNECTIONPOINTCONSTRAINT': ['EFFECTIVEDATE', 'GENCONID','CONNECTIONPOINTID', 'VERSIONNO', | ||
'BIDTYPE'], | ||
'SPDINTERCONNECTORCONSTRAINT': ['EFFECTIVEDATE','GENCONID','INTERCONNECTORID', 'VERSIONNO'], | ||
'GENCONDATA': ['GENCONID', 'EFFECTIVEDATE', 'VERSIONNO'], | ||
'MNSP_PEROFFER': ['SETTLEMENTDATE', 'OFFERDATE', 'VERSIONNO', 'PARTICIPANTID', 'LINKID'], | ||
'MNSP_DAYOFFER': ['SETTLEMENTDATE', 'OFFERDATE', 'VERSIONNO', 'PARTICIPANTID', 'LINKID'], | ||
'INTERCONNECTORCONSTRAINT': ['EFFECTIVEDATE', 'INTERCONNECTORID', 'VERSIONNO'], | ||
'MNSP_INTERCONNECTOR': ['EFFECTIVEDATE', 'LINKID', 'VERSIONNO'], | ||
'LOSSMODEL': ['EFFECTIVEDATE', 'INTERCONNECTORID', 'LOSSSEGMENT', 'VERSIONNO'], | ||
'LOSSFACTORMODEL': ['EFFECTIVEDATE', 'INTERCONNECTORID', 'REGIONID', 'VERSIONNO'], | ||
'BIDPEROFFER_D': ['BIDTYPE', 'DUID', 'OFFERDATE', 'PERIODID', 'SETTLEMENTDATE'], | ||
'DISPATCHINTERCONNECTORRES': ['DISPATCHINTERVAL', 'INTERCONNECTORID', 'INTERVENTION', | ||
'SETTLEMENTDATE'], | ||
'INTERCONNECTOR': ['INTERCONNECTORID'], | ||
'DISPATCHPRICE': ['DISPATCHINTERVAL', 'INTERVENTION', 'REGIONID', 'SETTLEMENTDATE'], | ||
'BIDDAYOFFER_D': ['BIDTYPE', 'DUID', 'SETTLEMENTDATE'], | ||
'DISPATCHREGIONSUM': ['DISPATCHINTERVAL', 'INTERVENTION', 'REGIONID', 'SETTLEMENTDATE']} | ||
|
||
effective_date_group_col = {'SPDREGIONCONSTRAINT': ['GENCONID'], | ||
'SPDCONNECTIONPOINTCONSTRAINT': ['GENCONID'], | ||
'SPDINTERCONNECTORCONSTRAINT': ['GENCONID'], | ||
'GENCONDATA': ['GENCONID'], | ||
'STADUALLOC': ['STATIONID'], | ||
'MNSP_INTERCONNECTOR': ['INTERCONNECTORID'], | ||
'INTERCONNECTORCONSTRAINT': ['INTERCONNECTORID'], | ||
'INTERCONNECTOR': ['INTERCONNECTORID'], | ||
'LOSSMODEL': ['INTERCONNECTORID'], | ||
'LOSSFACTORMODEL': ['INTERCONNECTORID'], | ||
'DUDETAILSUMMARY': ['DUID'], | ||
'MNSP_PEROFFER': ['LINKID'], | ||
'MNSP_DAYOFFER': ['LINKID']} | ||
|
||
|
||
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'] | ||
|
||
nem_data_model_start_time = '2009/07/01 00:00:00' | ||
|
||
fcas_start_index = 1 | ||
fcas_end_index = 2355 | ||
|
||
header_y_pad = 30 | ||
query_y_pad = (20, 0) | ||
query_row_offset = 2 | ||
row_height = 6 | ||
names_internal_row = 1 | ||
table_list_internal_row = 1 | ||
start_time_label_internal_row = 2 | ||
start_time_internal_row = 3 | ||
end_time_label_internal_row = 4 | ||
end_time_internal_row = 5 | ||
plus_internal_row = 6 | ||
plus_merge_internal_row = 7 | ||
list_row_span = 5 | ||
list_column_span = 2 | ||
save_field_column_span = 3 | ||
standard_x_pad = (0, 10) | ||
list_filter_row_span = 4 | ||
internal_filter_row = 2 | ||
delete_button_internal_row = 5 | ||
last_column = 100 | ||
join_type = ['inner', 'left', 'right'] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
import requests | ||
import zipfile | ||
import io | ||
|
||
|
||
def run(year, month, url, filename, down_load_to): | ||
"""This function""" | ||
|
||
# Add the year and month information to the generic AEMO data url | ||
url_formatted = format_aemo_url(url, year, month, filename) | ||
|
||
# Perform the download, unzipping saving of the file | ||
try: | ||
download_unzip_csv(url_formatted, down_load_to, filename) | ||
except: | ||
print('Warning {} not downloaded'.format(filename)) | ||
|
||
|
||
def run_fcas4s(year, month, day, index, url_latest, url_hist, filename, down_load_to): | ||
"""This function""" | ||
|
||
# Add the year and month information to the generic AEMO data url | ||
url_formatted_latest = url_latest.format(year, month, day, index) | ||
url_formatted_hist = url_hist.format(year, month, year, month, day, index) | ||
|
||
# Perform the download, unzipping saving of the file | ||
try: | ||
download_unzip_csv(url_formatted_latest, down_load_to, filename) | ||
except: | ||
try: | ||
download_unzip_csv(url_formatted_hist, down_load_to, filename) | ||
except: | ||
print('Warning {} not downloaded'.format(filename)) | ||
|
||
|
||
def download_unzip_csv(url, down_load_to, filename): | ||
"""This function downloads a zipped csv using a url, extracts the csv and saves it a specified location and with | ||
a specified filename""" | ||
r = requests.get(url) | ||
z = zipfile.ZipFile(io.BytesIO(r.content)) | ||
z.extractall(down_load_to) | ||
|
||
|
||
def download_csv(url, down_load_to, path_and_name): | ||
"""This function downloads a zipped csv using a url, extracts the csv and saves it a specified location and with | ||
a specified filename""" | ||
r = requests.get(url) | ||
with open(path_and_name, 'wb') as f: | ||
f.write(r.content) | ||
|
||
|
||
def download_xl(url, down_load_to, path_and_name): | ||
"""This function downloads a zipped csv using a url, extracts the csv and saves it a specified location and with | ||
a specified filename""" | ||
r = requests.get(url) | ||
with open(path_and_name, 'wb') as f: | ||
f.write(r.content) | ||
|
||
def format_aemo_url(url, year, month, filename): | ||
"""This fills in the missing information in the AEMO url so data for the right month, year and file name are | ||
downloaded""" | ||
year = str(year) | ||
return url.format(year, year, month, filename[:-4]) | ||
|
Oops, something went wrong.