Skip to content

Commit

Permalink
fix broken link
Browse files Browse the repository at this point in the history
  • Loading branch information
Ohadbasan committed Nov 27, 2023
1 parent 7b6d44b commit a9a19d1
Showing 1 changed file with 6 additions and 16 deletions.
22 changes: 6 additions & 16 deletions src/preprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import json
import time
import requests
import shutil


logging.basicConfig()
Expand Down Expand Up @@ -479,29 +480,18 @@ class PreprocessArguments:

# Mirrors for database
MIRRORS = [
'https://sponsor.ajay.app/database/sponsorTimes.csv', # Latest
'https://sb-mirror.mchang.xyz/sponsorTimes.csv', # 5 minute delay
'https://sb.ltn.fi/database/sponsorTimes.csv', # 5 minute delay
]
# TODO only download latest updates/changes



def download_file(url, filename):
"""
Helper method handling downloading large files from `url` to `filename`.
with requests.get(url, stream=True) as r:
with open(filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)

Adapted from https://stackoverflow.com/a/42071418
"""
chunk_size = 1024
r = requests.get(url, stream=True)
total_bytes = int(r.headers['Content-Length'])
with open(filename, 'wb') as f, tqdm(unit='B', total=total_bytes) as progress:
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk: # filter out keep-alive new chunks
progress.update(len(chunk))
f.write(chunk)

return total_bytes == os.path.getsize(filename)
return filename


def main():
Expand Down

0 comments on commit a9a19d1

Please sign in to comment.