Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

log events instead of printing for easier software interfacing #55

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

log events instead of printing for easier software interfacing #55

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 33 additions & 35 deletions YahooTickerDownloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
#!/usr/bin/env python

import sys
import pickle
from time import sleep
import argparse
import io
import logging

import tablib

from ytd import SimpleSymbolDownloader
from ytd.downloader.GenericDownloader import GenericDownloader
from ytd.compat import text
from ytd.compat import csv
from ytd.compat import robotparser

import tablib

import sys
logger = logging.getLogger(__name__)


user_agent = SimpleSymbolDownloader.user_agent

Expand All @@ -38,24 +42,23 @@ def downloadEverything(downloader, tickerType, insecure, sleeptime, pandantic):
while not downloader.isDone():

symbols = downloader.nextRequest(insecure, pandantic)
print("Got " + str(len(symbols)) + " downloaded " + downloader.type + " symbols:")
logger.info(f"Got {str(len(symbols))} downloaded {downloader.type} symbols:")
if(len(symbols) > 2):
try:
print (" " + text(symbols[0]))
print (" " + text(symbols[1]))
print (" ect...")
logger.info(f" {text(symbols[0])}")
logger.info(f" {text(symbols[1])}")
logger.info(" ect...")
except:
print (" Could not display some ticker symbols due to char encoding")
logger.warning(" Could not display some ticker symbols due to char encoding")
downloader.printProgress()

# Save download state occasionally.
# We do this in case this long running is suddenly interrupted.
loop = loop + 1
if loop % 200 == 0:
print ("Saving downloader to disk...")
logger.info("Saving downloader to disk...")
saveDownloader(downloader, tickerType)
print ("Downloader successfully saved.")
print ("")
logger.info("Downloader successfully saved.")

if not downloader.isDone():
sleep(sleeptime) # So we don't overload the server.
Expand All @@ -75,22 +78,21 @@ def main():

protocol = 'http' if args.insecure else 'https'
if args.insecure:
print("Using insecure connection")
logger.info("Using insecure connection")

if args.export:
print("Exporting pickle file")
logger.info("Exporting pickle file")

tickerType = args.type = args.type.lower()

print("Checking if we can resume a old download session")
logger.info("Checking if we can resume a old download session")
try:
downloader = loadDownloader(tickerType)
print("Downloader found on disk, resuming")
logger.info("Downloader found on disk, resuming")
except:
print("No old downloader found on disk")
print("Starting a new session")
logger.info("No old downloader found on disk. Starting a new session")
if tickerType not in options:
print("Error: " + tickerType + " is not a valid type option. See --help")
logger.error(f"Error: {tickerType} is not a valid type option. See --help")
exit(1)
else:
downloader = options[tickerType]
Expand All @@ -102,35 +104,31 @@ def main():
if not args.export:

if(not rp.can_fetch(user_agent, protocol + '://finance.yahoo.com/_finance_doubledown/api/resource/searchassist')):
print('Execution of script halted due to robots.txt')
logger.warning('Execution of script halted due to robots.txt')
return 1

if not downloader.isDone():
print("Downloading " + downloader.type)
print("")
logger.info(f"Downloading {downloader.type}")
downloadEverything(downloader, tickerType, args.insecure, args.sleep, args.pandantic)
print ("Saving downloader to disk...")
logger.info("Saving downloader to disk...")
saveDownloader(downloader, tickerType)
print ("Downloader successfully saved.")
print ("")
logger.info("Downloader successfully saved.")
else:
print("The downloader has already finished downloading everything")
print("")
logger.info("The downloader has already finished downloading everything")

except Exception as ex:
print("A exception occurred while downloading. Suspending downloader to disk")
logger.error("A exception occurred while downloading. Suspending downloader to disk")
saveDownloader(downloader, tickerType)
print("Successfully saved download state")
print("Try removing {type}.pickle file if this error persists")
print("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues")
print("")
logger.error("Successfully saved download state")
logger.error("Try removing {type}.pickle file if this error persists")
logger.error("Issues can be reported on https://github.com/Benny-/Yahoo-ticker-symbol-downloader/issues")
raise
except KeyboardInterrupt as ex:
print("Suspending downloader to disk as .pickle file")
logger.warning("Suspending downloader to disk as pickle file")
saveDownloader(downloader, tickerType)

if downloader.isDone() or args.export:
print("Exporting "+downloader.type+" symbols")
logger.info(f"Exporting {downloader.type} symbols")

data = tablib.Dataset()
data.headers = downloader.getRowHeader()
Expand All @@ -152,19 +150,19 @@ def main():
with open(downloader.type + '.xlsx', 'wb') as f:
f.write(data.xlsx)
except:
print("Could not export .xlsx due to a internal error")
logger.warning(f"Could not export .xlsx due to a internal error")

try:
with open(downloader.type + '.json', 'wb') as f:
f.write(data.json.encode('UTF-8'))
except:
print("Could not export .json due to a internal error")
logger.warning("Could not export .json due to a internal error")

try:
with open(downloader.type + '.yaml', 'wb') as f:
f.write(data.yaml.encode('UTF-8'))
except:
print("Could not export .yaml due to a internal error")
logger.warning("Could not export .yaml due to a internal error")

if __name__ == "__main__":
main()