diff --git a/.gitignore b/.gitignore index 1090913..ccfe2d1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ local.properties .classpath .settings/ .loadpath -.idea htmlcov/ # External tool builders .externalToolBuilders/ @@ -64,7 +63,6 @@ htmlcov/ *.tlb *.tli *.tlh -*.tmp *.vspscc .builds *.dotCover diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 780186c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -sudo: false -language: python -python: - - 3.8 -env: - - TOXENV=py-normal - -install: pip install tox -script: tox diff --git a/README.md b/README.md index 6911589..c1c8e73 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,7 @@ Mega.py ======= -[![Build -Status](https://travis-ci.org/odwyersoftware/mega.py.png?branch=master)](https://travis-ci.org/odwyersoftware/mega.py) -[![Downloads](https://pypip.in/d/mega.py/badge.png)](https://crate.io/packages/mega.py/) [![PyPI version](https://badge.fury.io/py/mega.py.svg)](https://pypi.org/project/mega.py/) - -Python library for the [Mega.co.nz](https://mega.nz/aff=Zo6IxNaHw14) +Python library for the [Mega.nz](https://mega.nz) API, currently supporting: - login @@ -26,14 +22,15 @@ How To Use ### Create a Mega account -First, [create an account with Mega](https://mega.nz/aff=Zo6IxNaHw14) . +For downloading links, no account is needed. To use upload capabilities, [create an account with Mega](https://mega.nz) . ### Install mega.py package -Run the following command, or run setup from the latest github source. +Clone the repository, and then run: -```python -pip install mega.py +```sh +cd mega.py +pip install . ``` ### Import mega.py @@ -123,7 +120,7 @@ m.upload('myfile.doc', folder[0]) ```python file = m.find('myfile.doc') m.download(file) -m.download_url('https://mega.co.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc') +m.download_url('https://mega.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc') m.download(file, '/home/john-smith/Desktop') # specify optional download filename (download_url() supports this also) m.download(file, '/home/john-smith/Desktop', 'myfile.zip') @@ -132,9 +129,9 @@ m.download(file, '/home/john-smith/Desktop', 'myfile.zip') ### Import a file from URL, optionally specify destination folder ```python -m.import_public_url('https://mega.co.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc') +m.import_public_url('https://mega.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc') folder_node = m.find('Documents')[1] -m.import_public_url('https://mega.co.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc', dest_node=folder_node) +m.import_public_url('https://mega.nz/#!utYjgSTQ!OM4U3V5v_W4N5edSo0wolg1D5H0fwSrLD3oLnLuS9pc', dest_node=folder_node) ``` ### Create a folder @@ -160,9 +157,3 @@ Returns a dict of folder node name and node\_id, e.g. file = m.find('myfile.doc') m.rename(file, 'my_file.doc') ``` - -## Contact Support - -For paid priority support contact [mega@odwyer.software](mailto:mega@odwyer.software). - -**[UK Python Development Agency](https://odwyer.software/)** diff --git a/requirements.txt b/requirements.txt index 1bc97c6..4c0d0b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests>=0.10 -pycryptodome>=3.9.6,<4.0.0 -pathlib==1.0.1 -tenacity>=5.1.5,<6.0.0 +requests>=2.27.1 +pycryptodome>=3.20.0,<4.0.0 +tenacity>=8.2.2 +tqdm>=4.64.1 diff --git a/setup.py b/setup.py index c49d2ee..6e2c1f7 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -* -from __future__ import absolute_import - import os from codecs import open @@ -22,16 +20,15 @@ setup(name='mega.py', version='1.0.9.dev0', + python_requires='>=3.6', packages=find_packages('src', exclude=('tests', )), package_dir={'': 'src'}, include_package_data=True, zip_safe=False, - url='https://github.com/odwyersoftware/mega.py', - description='Python lib for the Mega.co.nz API', + url='https://github.com/pgp/mega.py', + description='Python lib for the Mega.nz API', long_description=readme + '\n\n' + history, long_description_content_type='text/markdown', - author='O\'Dwyer Software', - author_email='hello@odwyer.software', license='Creative Commons Attribution-Noncommercial-Share Alike license', install_requires=install_requires, classifiers=[ @@ -40,4 +37,10 @@ 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Topic :: Internet :: WWW/HTTP', - ]) + ], + entry_points={ + 'console_scripts': [ + 'meganz=mega.maincli:maincli' + ] + }, + ) diff --git a/src/mega/crypto.py b/src/mega/crypto.py index 61ddf15..e0933c6 100644 --- a/src/mega/crypto.py +++ b/src/mega/crypto.py @@ -1,37 +1,28 @@ -from Crypto.Cipher import AES -import json import base64 -import struct import binascii +import codecs +import json import random -import sys +import struct -# Python3 compatibility -if sys.version_info < (3, ): +from Crypto.Cipher import AES - def makebyte(x): - return x +EMPTY_IV = b'\0'*16 - def makestring(x): - return x -else: - import codecs +def makebyte(x): + return codecs.latin_1_encode(x)[0] - def makebyte(x): - return codecs.latin_1_encode(x)[0] - def makestring(x): - return codecs.latin_1_decode(x)[0] +def makestring(x): + return codecs.latin_1_decode(x)[0] def aes_cbc_encrypt(data, key): - aes_cipher = AES.new(key, AES.MODE_CBC, makebyte('\0' * 16)) - return aes_cipher.encrypt(data) + return AES.new(key, AES.MODE_CBC, EMPTY_IV).encrypt(data) def aes_cbc_decrypt(data, key): - aes_cipher = AES.new(key, AES.MODE_CBC, makebyte('\0' * 16)) - return aes_cipher.decrypt(data) + return AES.new(key, AES.MODE_CBC, EMPTY_IV).decrypt(data) def aes_cbc_encrypt_a32(data, key): @@ -42,8 +33,8 @@ def aes_cbc_decrypt_a32(data, key): return str_to_a32(aes_cbc_decrypt(a32_to_str(data), a32_to_str(key))) -def stringhash(str, aeskey): - s32 = str_to_a32(str) +def stringhash(s, aeskey): + s32 = str_to_a32(s) h32 = [0, 0, 0, 0] for i in range(len(s32)): h32[i % 4] ^= s32[i] @@ -65,6 +56,8 @@ def prepare_key(arr): def encrypt_key(a, key): + # this sum, which is applied to a generator of tuples, actually flattens the output list of lists of that generator + # i.e. it's equivalent to tuple([item for t in generatorOfLists for item in t]) return sum((aes_cbc_encrypt_a32(a[i:i + 4], key) for i in range(0, len(a), 4)), ()) @@ -85,7 +78,18 @@ def decrypt_attr(attr, key): attr = aes_cbc_decrypt(attr, a32_to_str(key)) attr = makestring(attr) attr = attr.rstrip('\0') - return json.loads(attr[4:]) if attr[:6] == 'MEGA{"' else False + + prefix = 'MEGA{"' + if attr.startswith(prefix): + i1 = 4 + i2 = attr.find('}') + if i2 >= 0: + i2+=1 + return json.loads(attr[i1:i2]) + else: + raise RuntimeError(f'Unable to properly decode filename, raw content is: {attr}') + else: + return False def a32_to_str(a): @@ -149,6 +153,8 @@ def a32_to_base64(a): return base64_url_encode(a32_to_str(a)) +# generates a list of chunks of the kind (offset, chunk_size), where offset refers to the file start +# chunk_size starts at 0x20000 (100 KiB), and then increments linearly till saturation to 0x100000 (1 MiB) def get_chunks(size): p = 0 s = 0x20000 diff --git a/src/mega/maincli.py b/src/mega/maincli.py new file mode 100644 index 0000000..d31520f --- /dev/null +++ b/src/mega/maincli.py @@ -0,0 +1,20 @@ +import argparse +import os + +from mega import Mega + +def maincli(): + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--direct', + action='store_true', + dest='noTempFile', + help='Do not write to a temp file (write directly to output file)') + parser.add_argument('downloadUrl', nargs=1) + parser.add_argument('destinationDirectory', nargs='?', default=os.path.realpath('.')) + args = parser.parse_args() + m = Mega().login() + m.download_url(url=args.downloadUrl[0], dest_path=args.destinationDirectory, no_temp_file=args.noTempFile) + + +if __name__ == '__main__': + maincli() \ No newline at end of file diff --git a/src/mega/mega.py b/src/mega/mega.py index 906b0db..675db4b 100644 --- a/src/mega/mega.py +++ b/src/mega/mega.py @@ -1,6 +1,5 @@ import math import re -import json import logging import secrets from pathlib import Path @@ -15,14 +14,16 @@ import shutil import requests +import tqdm + from tenacity import retry, wait_exponential, retry_if_exception_type from .errors import ValidationError, RequestError from .crypto import (a32_to_base64, encrypt_key, base64_url_encode, encrypt_attr, base64_to_a32, base64_url_decode, decrypt_attr, a32_to_str, get_chunks, str_to_a32, - decrypt_key, mpi_to_int, stringhash, prepare_key, make_id, - makebyte, modular_inverse) + decrypt_key, mpi_to_int, stringhash, prepare_key, + make_id, modular_inverse) logger = logging.getLogger(__name__) @@ -30,7 +31,8 @@ class Mega: def __init__(self, options=None): self.schema = 'https' - self.domain = 'mega.co.nz' + self.domain = 'mega.nz' + self.api_domain = 'mega.co.nz' # g.api.mega.nz doesn't exist, have to use the old domain name for api access self.timeout = 160 # max secs to wait for resp from api requests self.sid = None self.sequence_num = random.randint(0, 0xFFFFFFFF) @@ -54,7 +56,6 @@ def _login_user(self, email, password): logger.info('Logging in user...') email = email.lower() get_user_salt_resp = self._api_request({'a': 'us0', 'user': email}) - user_salt = None try: user_salt = base64_to_a32(get_user_salt_resp['s']) except KeyError: @@ -162,14 +163,14 @@ def _api_request(self, data): if not isinstance(data, list): data = [data] - url = f'{self.schema}://g.api.{self.domain}/cs' + url = f'{self.schema}://g.api.{self.api_domain}/cs' response = requests.post( url, params=params, - data=json.dumps(data), + json=data, timeout=self.timeout, ) - json_resp = json.loads(response.text) + json_resp = response.json() try: if isinstance(json_resp, list): int_resp = json_resp[0] if isinstance(json_resp[0], @@ -188,8 +189,27 @@ def _api_request(self, data): raise RequestError(int_resp) return json_resp[0] - def _parse_url(self, url): - """Parse file id and key from url.""" + def _is_mega_link(self, url: str): + return url.startswith(f'{self.schema}://{self.domain}') or url.startswith(f'{self.schema}://{self.api_domain}') + + def _follow_redirects(self, url: str): + for i in range(10): + if self._is_mega_link(url): + return url + resp = requests.get(url, allow_redirects=False) + if resp.is_redirect or resp.is_permanent_redirect: + url = resp.headers['Location'] + print(f'Found redirect: {url}') + continue + else: + raise RuntimeError('Url is not a redirect nor a mega link') + raise RuntimeError('Too many redirects') + + def _parse_url(self, url: str): + # Parse file id and key from url. + # Follow redirects from URL shortening services, if any + url = self._follow_redirects(url) + if '/file/' in url: # V2 URL structure url = url.replace(' ', '') @@ -324,7 +344,6 @@ def find(self, filename=None, handle=None, exclude_deleted=False): filename = path.name parent_dir_name = path.parent.name for file in list(files.items()): - parent_node_id = None try: if parent_dir_name: parent_node_id = self.find_path_descriptor(parent_dir_name, @@ -391,7 +410,8 @@ def get_link(self, file): else: raise ValidationError('File id and key must be present') - def _node_data(self, node): + @staticmethod + def _node_data(node): try: return node[1] except (IndexError, KeyError): @@ -417,7 +437,7 @@ def get_user(self): user_data = self._api_request({'a': 'ug'}) return user_data - def get_node_by_type(self, type): + def get_node_by_type(self, node_type): """ Get a node by it's numeric type id, e.g: 0: file @@ -428,7 +448,7 @@ def get_node_by_type(self, type): """ nodes = self.get_files() for node in list(nodes.items()): - if node[1]['t'] == type: + if node[1]['t'] == node_type: return node def get_files_in_node(self, target): @@ -457,7 +477,8 @@ def get_id_from_public_handle(self, public_handle): node_id = self.get_id_from_obj(node_data) return node_id - def get_id_from_obj(self, node_data): + @staticmethod + def get_id_from_obj(node_data): """ Get node id from a file object """ @@ -630,7 +651,7 @@ def export(self, path=None, node_id=None): nodes = self.get_files() return self.get_folder_link(nodes[node_id]) - def download_url(self, url, dest_path=None, dest_filename=None): + def download_url(self, url, dest_path=None, dest_filename=None, no_temp_file=False): """ Download a file by it's public url """ @@ -643,6 +664,7 @@ def download_url(self, url, dest_path=None, dest_filename=None): dest_path=dest_path, dest_filename=dest_filename, is_public=True, + no_temp_file=no_temp_file ) def _download_file(self, @@ -651,21 +673,16 @@ def _download_file(self, dest_path=None, dest_filename=None, is_public=False, - file=None): + file=None, + no_temp_file=False): if file is None: if is_public: file_key = base64_to_a32(file_key) - file_data = self._api_request({ - 'a': 'g', - 'g': 1, - 'p': file_handle - }) - else: - file_data = self._api_request({ - 'a': 'g', - 'g': 1, - 'n': file_handle - }) + file_data = self._api_request({ + 'a': 'g', + 'g': 1, + 'p' if is_public else 'n': file_handle + }) k = (file_key[0] ^ file_key[4], file_key[1] ^ file_key[5], file_key[2] ^ file_key[6], file_key[3] ^ file_key[7]) @@ -678,7 +695,7 @@ def _download_file(self, meta_mac = file['meta_mac'] # Seems to happens sometime... When this occurs, files are - # inaccessible also in the official also in the official web app. + # inaccessible also in the official web app. # Strangely, files can come back later. if 'g' not in file_data: raise RequestError('File not accessible anymore') @@ -698,52 +715,59 @@ def _download_file(self, dest_path = '' else: dest_path += '/' + output_path = Path(dest_path + file_name) - with tempfile.NamedTemporaryFile(mode='w+b', + with (tempfile.NamedTemporaryFile(mode='w+b', prefix='megapy_', - delete=False) as temp_output_file: - k_str = a32_to_str(k) - counter = Counter.new(128, - initial_value=((iv[0] << 32) + iv[1]) << 64) - aes = AES.new(k_str, AES.MODE_CTR, counter=counter) - - mac_str = '\0' * 16 - mac_encryptor = AES.new(k_str, AES.MODE_CBC, - mac_str.encode("utf8")) - iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) - - for chunk_start, chunk_size in get_chunks(file_size): - chunk = input_file.read(chunk_size) - chunk = aes.decrypt(chunk) - temp_output_file.write(chunk) - - encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) - for i in range(0, len(chunk) - 16, 16): - block = chunk[i:i + 16] - encryptor.encrypt(block) + delete=False) if not no_temp_file else open(output_path, 'xb')) as temp_output_file: + with tqdm.tqdm(total=file_size, unit='iB', unit_scale=True) as progress_bar: + k_str = a32_to_str(k) + counter = Counter.new(128, + initial_value=((iv[0] << 32) + iv[1]) << 64) + aes = AES.new(k_str, AES.MODE_CTR, counter=counter) + + # mega.nz improperly uses CBC as a MAC mode, so after each chunk, the computed mac_bytes are used as IV for the next chunk MAC accumulation + mac_bytes = b'\0' * 16 + mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_bytes) + iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) - # fix for files under 16 bytes failing - if file_size > 16: - i += 16 - else: - i = 0 - - block = chunk[i:i + 16] - if len(block) % 16: - block += b'\0' * (16 - (len(block) % 16)) - mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) - - file_info = os.stat(temp_output_file.name) - logger.info('%s of %s downloaded', file_info.st_size, - file_size) - file_mac = str_to_a32(mac_str) - # check mac integrity - if (file_mac[0] ^ file_mac[1], - file_mac[2] ^ file_mac[3]) != meta_mac: - raise ValueError('Mismatched mac') - output_path = Path(dest_path + file_name) - shutil.move(temp_output_file.name, output_path) - return output_path + for chunk_start, chunk_size in get_chunks(file_size): + # print('Chunk size from generator: '+chunk_size) + chunk = input_file.read(chunk_size) + chunk = aes.decrypt(chunk) + temp_output_file.write(chunk) + progress_bar.update(len(chunk)) + + encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) + + # take last 16-N bytes from chunk (with N between 1 and 16, including extremes) + mv = memoryview(chunk) # avoid copying memory for the entire chunk when slicing + modchunk = len(chunk) % 16 + if modchunk == 0: + # ensure we reserve the last 16 bytes anyway, we have to feed them into mac_encryptor + modchunk = 16 + last_block = chunk[-modchunk:] # fine to copy bytes here, they're only a few bytes + else: + last_block = chunk[-modchunk:] + (b'\0' * (16 - modchunk)) # pad last block to 16 bytes + rest_of_chunk = mv[:-modchunk] + + encryptor.encrypt(rest_of_chunk) + input_to_mac = encryptor.encrypt(last_block) + mac_bytes = mac_encryptor.encrypt(input_to_mac) + + # file_info = os.stat(temp_output_file.name) + # logger.info('%s of %s downloaded', file_info.st_size, + # file_size) + file_mac = str_to_a32(mac_bytes) + # check mac integrity + if (file_mac[0] ^ file_mac[1], + file_mac[2] ^ file_mac[3]) != meta_mac: + raise ValueError('Mismatched mac') + output_name = temp_output_file.name + if not no_temp_file: + print('Moving temporary file to destination path') + shutil.move(output_name, output_path) + return output_path def upload(self, filename, dest=None, dest_filename=None): # determine storage node @@ -759,18 +783,17 @@ def upload(self, filename, dest=None, dest_filename=None): ul_url = self._api_request({'a': 'u', 's': file_size})['p'] # generate random aes key (128) for file - ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] - k_str = a32_to_str(ul_key[:4]) + ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] # generate 192 bits of random data + k_str = a32_to_str(ul_key[:4]) # use 128 bits for the key... count = Counter.new( - 128, initial_value=((ul_key[4] << 32) + ul_key[5]) << 64) + 128, initial_value=((ul_key[4] << 32) + ul_key[5]) << 64) # and 64 bits for the IV (which has size 128 bits anyway) aes = AES.new(k_str, AES.MODE_CTR, counter=count) upload_progress = 0 completion_file_handle = None - mac_str = '\0' * 16 - mac_encryptor = AES.new(k_str, AES.MODE_CBC, - mac_str.encode("utf8")) + mac_bytes = b'\0' * 16 + mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_bytes) iv_str = a32_to_str([ul_key[4], ul_key[5], ul_key[4], ul_key[5]]) if file_size > 0: for chunk_start, chunk_size in get_chunks(file_size): @@ -789,9 +812,10 @@ def upload(self, filename, dest=None, dest_filename=None): i = 0 block = chunk[i:i + 16] - if len(block) % 16: - block += makebyte('\0' * (16 - len(block) % 16)) - mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) + modlenblock = len(block) % 16 + if modlenblock: + block += (b'\0' * (16 - modlenblock)) + mac_bytes = mac_encryptor.encrypt(encryptor.encrypt(block)) # encrypt file and upload chunk = aes.encrypt(chunk) @@ -811,7 +835,7 @@ def upload(self, filename, dest=None, dest_filename=None): logger.info('Chunks uploaded') logger.info('Setting attributes to complete upload') logger.info('Computing attributes') - file_mac = str_to_a32(mac_str) + file_mac = str_to_a32(mac_bytes) # determine meta mac meta_mac = (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) diff --git a/src/tests/test_mega.py b/src/tests/test_mega.py index 5cf364e..9225d0d 100644 --- a/src/tests/test_mega.py +++ b/src/tests/test_mega.py @@ -7,7 +7,7 @@ from mega import Mega -TEST_CONTACT = 'test@mega.co.nz' +TEST_CONTACT = 'test@mega.nz' TEST_PUBLIC_URL = ( 'https://mega.nz/#!hYVmXKqL!r0d0-WRnFwulR_shhuEDwrY1Vo103-am1MyUy8oV6Ps') TEST_FILE = os.path.basename(__file__) @@ -80,7 +80,7 @@ def test_export_folder(self, mega, folder_name): if not public_url: public_url = result_public_share_url - assert result_public_share_url.startswith('https://mega.co.nz/#F!') + assert result_public_share_url.startswith('https://mega.nz/#F!') assert result_public_share_url == public_url def test_export_folder_within_folder(self, mega, folder_name): @@ -89,14 +89,14 @@ def test_export_folder_within_folder(self, mega, folder_name): result_public_share_url = mega.export(path=folder_path) - assert result_public_share_url.startswith('https://mega.co.nz/#F!') + assert result_public_share_url.startswith('https://mega.nz/#F!') def test_export_folder_using_node_id(self, mega, folder_name): node_id = mega.find(folder_name)[0] result_public_share_url = mega.export(node_id=node_id) - assert result_public_share_url.startswith('https://mega.co.nz/#F!') + assert result_public_share_url.startswith('https://mega.nz/#F!') def test_export_single_file(self, mega, folder_name): # Upload a single file into a folder @@ -109,7 +109,7 @@ def test_export_single_file(self, mega, folder_name): for _ in range(2): result_public_share_url = mega.export(path) - assert result_public_share_url.startswith('https://mega.co.nz/#!') + assert result_public_share_url.startswith('https://mega.nz/#!') def test_import_public_url(mega): @@ -244,6 +244,6 @@ def test_when_api_returns_int_raises_exception( response_text, ): with requests_mock.Mocker() as m: - m.post(f'{mega.schema}://g.api.{mega.domain}/cs', + m.post(f'{mega.schema}://g.api.{mega.api_domain}/cs', text=response_text) mega._api_request(data={})