diff --git a/YahooTickerDownloader.py b/YahooTickerDownloader.py index ab4c5ac..92d24fb 100755 --- a/YahooTickerDownloader.py +++ b/YahooTickerDownloader.py @@ -1,9 +1,13 @@ #!/usr/bin/env python +import sys import pickle from time import sleep import argparse import io +import logging + +import tablib from ytd import SimpleSymbolDownloader from ytd.downloader.GenericDownloader import GenericDownloader @@ -11,9 +15,9 @@ from ytd.compat import csv from ytd.compat import robotparser -import tablib -import sys +logger = logging.getLogger(__name__) + user_agent = SimpleSymbolDownloader.user_agent @@ -38,24 +42,23 @@ def downloadEverything(downloader, tickerType, insecure, sleeptime, pandantic): while not downloader.isDone(): symbols = downloader.nextRequest(insecure, pandantic) - print("Got " + str(len(symbols)) + " downloaded " + downloader.type + " symbols:") + logger.info(f"Got {str(len(symbols))} downloaded {downloader.type} symbols:") if(len(symbols) > 2): try: - print (" " + text(symbols[0])) - print (" " + text(symbols[1])) - print (" ect...") + logger.info(f" {text(symbols[0])}") + logger.info(f" {text(symbols[1])}") + logger.info(" ect...") except: - print (" Could not display some ticker symbols due to char encoding") + logger.warning(" Could not display some ticker symbols due to char encoding") downloader.printProgress() # Save download state occasionally. # We do this in case this long running is suddenly interrupted. loop = loop + 1 if loop % 200 == 0: - print ("Saving downloader to disk...") + logger.info("Saving downloader to disk...") saveDownloader(downloader, tickerType) - print ("Downloader successfully saved.") - print ("") + logger.info("Downloader successfully saved.") if not downloader.isDone(): sleep(sleeptime) # So we don't overload the server. @@ -75,22 +78,21 @@ def main(): protocol = 'http' if args.insecure else 'https' if args.insecure: - print("Using insecure connection") + logger.info("Using insecure connection") if args.export: - print("Exporting pickle file") + logger.info("Exporting pickle file") tickerType = args.type = args.type.lower() - print("Checking if we can resume a old download session") + logger.info("Checking if we can resume a old download session") try: downloader = loadDownloader(tickerType) - print("Downloader found on disk, resuming") + logger.info("Downloader found on disk, resuming") except: - print("No old downloader found on disk") - print("Starting a new session") + logger.info("No old downloader found on disk. Starting a new session") if tickerType not in options: - print("Error: " + tickerType + " is not a valid type option. See --help") + logger.error(f"Error: {tickerType} is not a valid type option. See --help") exit(1) else: downloader = options[tickerType] @@ -102,35 +104,31 @@ def main(): if not args.export: if(not rp.can_fetch(user_agent, protocol + '://finance.yahoo.com/_finance_doubledown/api/resource/searchassist')): - print('Execution of script halted due to robots.txt') + logger.warning('Execution of script halted due to robots.txt') return 1 if not downloader.isDone(): - print("Downloading " + downloader.type) - print("") + logger.info(f"Downloading {downloader.type}") downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic) - print ("Saving downloader to disk...") + logger.info("Saving downloader to disk...") saveDownloader(downloader, tickerType) - print ("Downloader successfully saved.") - print ("") + logger.info("Downloader successfully saved.") else: - print("The downloader has already finished downloading everything") - print("") + logger.info("The downloader has already finished downloading everything") except Exception as ex: - print("A exception occurred while downloading. Suspending downloader to disk") + logger.error("A exception occurred while downloading. Suspending downloader to disk") saveDownloader(downloader, tickerType) - print("Successfully saved download state") - print("Try removing {type}.pickle file if this error persists") - print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues") - print("") + logger.error("Successfully saved download state") + logger.error("Try removing {type}.pickle file if this error persists") + logger.error("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues") raise except KeyboardInterrupt as ex: - print("Suspending downloader to disk as .pickle file") + logger.warning("Suspending downloader to disk as pickle file") saveDownloader(downloader, tickerType) if downloader.isDone() or args.export: - print("Exporting "+downloader.type+" symbols") + logger.info(f"Exporting {downloader.type} symbols") data = tablib.Dataset() data.headers = downloader.getRowHeader() @@ -152,19 +150,19 @@ def main(): with open(downloader.type + '.xlsx', 'wb') as f: f.write(data.xlsx) except: - print("Could not export .xlsx due to a internal error") + logger.warning(f"Could not export .xlsx due to a internal error") try: with open(downloader.type + '.json', 'wb') as f: f.write(data.json.encode('UTF-8')) except: - print("Could not export .json due to a internal error") + logger.warning("Could not export .json due to a internal error") try: with open(downloader.type + '.yaml', 'wb') as f: f.write(data.yaml.encode('UTF-8')) except: - print("Could not export .yaml due to a internal error") + logger.warning("Could not export .yaml due to a internal error") if __name__ == "__main__": main()