diff --git a/.gitignore b/.gitignore index 2f295e5f..21ecb6f8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ eval* .pytest_cache .tox docs_api +.idea +venv diff --git a/dump.txt.gz b/dump.txt.gz deleted file mode 100644 index b3b4af48..00000000 Binary files a/dump.txt.gz and /dev/null differ diff --git a/examples/advanced.py b/examples/advanced.py index 25d17076..08776949 100644 --- a/examples/advanced.py +++ b/examples/advanced.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from koala.ExcelCompiler import ExcelCompiler from koala.Spreadsheet import Spreadsheet from koala.excellib import xsum diff --git a/examples/basic.py b/examples/basic.py index ddeb323c..280934bd 100644 --- a/examples/basic.py +++ b/examples/basic.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from koala.ExcelCompiler import ExcelCompiler from koala.Spreadsheet import Spreadsheet diff --git a/koala/Cell.py b/koala/Cell.py index df893ccd..3dd97d9c 100644 --- a/koala/Cell.py +++ b/koala/Cell.py @@ -1,13 +1,9 @@ # cython: profile=True -from __future__ import absolute_import, division - from koala.CellBase import CellBase from koala.Range import RangeCore from koala.utils import * -from openpyxl.compat import unicode - class Cell(CellBase): ctr = 0 @@ -62,11 +58,7 @@ def __init__( self.__row = None self.__col_idx = None - # `unicode` != `str` in Python2. See `from openpyxl.compat import unicode` - if type(formula) == str and str != unicode: - self.__formula = unicode(formula, 'utf-8') if formula else None - else: - self.__formula = formula if formula else None + self.__formula = formula if formula else None self.__value = value self.python_expression = None diff --git a/koala/ExcelCompiler.py b/koala/ExcelCompiler.py index 8d228a34..79604206 100644 --- a/koala/ExcelCompiler.py +++ b/koala/ExcelCompiler.py @@ -1,15 +1,5 @@ -from __future__ import print_function # cython: profile=True -import os.path - -import networkx - -from koala.reader import read_archive, read_named_ranges, read_cells -from koala.utils import * -from koala.ast import graph_from_seeds, shunting_yard, build_ast, prepare_pointer -from koala.Cell import Cell -from koala.Range import RangeFactory from koala.Spreadsheet import Spreadsheet import warnings diff --git a/koala/Range.py b/koala/Range.py index f3c2c0cd..8ceadf12 100644 --- a/koala/Range.py +++ b/koala/Range.py @@ -1,12 +1,7 @@ -from __future__ import absolute_import, division, print_function - from koala.CellBase import CellBase from koala.ExcelError import ErrorCodes, ExcelError from koala.utils import * -from openpyxl.compat import unicode - - # WARNING: Range should never be imported directly. Import Range from excelutils instead. ### Range Utils ### @@ -58,7 +53,7 @@ def check_value(a): return ExcelError(a) try: # This is to avoid None or Exception returned by Range operations - if isinstance(a, (unicode, str)): + if isinstance(a, str): return a elif float(a): return a @@ -570,7 +565,7 @@ def multiply(a, b): @staticmethod def divide(a, b): try: - return old_div(float(check_value(a)), float(check_value(b))) + return float(check_value(a)) / float(check_value(b)) except Exception as e: return ExcelError('#DIV/0!', e) @@ -584,9 +579,9 @@ def power(a, b): @staticmethod def is_equal(a, b): try: - if not isinstance(a, (str, unicode)): + if not isinstance(a, str): a = check_value(a) - if not isinstance(b, (str, unicode)): + if not isinstance(b, str): b = check_value(b) return is_almost_equal(a, b, precision=0.00001) @@ -596,9 +591,9 @@ def is_equal(a, b): @staticmethod def is_not_equal(a, b): try: - if not isinstance(a, (str, unicode)): + if not isinstance(a, str): a = check_value(a) - if not isinstance(a, (str, unicode)): + if not isinstance(a, str): b = check_value(b) return a != b diff --git a/koala/Spreadsheet.py b/koala/Spreadsheet.py index f4ac5db5..f427e723 100644 --- a/koala/Spreadsheet.py +++ b/koala/Spreadsheet.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, print_function # cython: profile=True from koala.Range import get_cell_address, parse_cell_address @@ -17,9 +16,6 @@ import networkx from networkx.readwrite import json_graph -from openpyxl.compat import unicode - - class Spreadsheet(object): def __init__(self, file=None, ignore_sheets=[], ignore_hidden=False, debug=False): # print("___### Initializing Excel Compiler ###___") @@ -56,7 +52,7 @@ def __init__(self, file=None, ignore_sheets=[], ignore_hidden=False, debug=False else: # assume file path archive = read_archive(os.path.abspath(file)) # Parse cells - self.cells = read_cells(archive, ignore_sheets, ignore_hidden) + self.cells, self.sheets = read_cells(archive, ignore_sheets, ignore_hidden) # Parse named_range { name (ExampleName) -> address (Sheet!A1:A10)} self.named_ranges = read_named_ranges(archive) self.range = RangeFactory(self.cells) @@ -306,7 +302,7 @@ def cell_set_formula(self, address, formula): for index, c in enumerate(cell.range.cells): # for each cell of the range, translate the formula if index == 0: c.formula = formula - translator = Translator(unicode('=' + formula), c.address().split('!')[1]) # the Translator needs a reference without sheet + translator = Translator('=' + formula, c.address().split('!')[1]) # the Translator needs a reference without sheet else: translated = translator.translate_formula(c.address().split('!')[1]) # the Translator needs a reference without sheet c.formula = translated[1:] # to get rid of the '=' diff --git a/koala/__init__.py b/koala/__init__.py index 012c30aa..322478f4 100644 --- a/koala/__init__.py +++ b/koala/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from openpyxl import * from .ast import * from .Cell import * diff --git a/koala/ast/__init__.py b/koala/ast/__init__.py index 6d1f8d21..fd243ac2 100644 --- a/koala/ast/__init__.py +++ b/koala/ast/__init__.py @@ -1,12 +1,9 @@ -from __future__ import absolute_import # cython: profile=True import collections -import six import networkx from networkx.classes.digraph import DiGraph -from openpyxl.compat import unicode from koala.utils import uniqueify, flatten, max_dimension, col2num, resolve_range from koala.Cell import Cell @@ -118,7 +115,7 @@ def shunting_yard(expression, named_ranges, ref = None, tokenize_range = False): for index, token in enumerate(tokens): new_tokens.append(token) - if type(token.tvalue) == str or type(token.tvalue) == unicode: + if type(token.tvalue) == str: if token.tvalue.startswith(':'): # example -> :OFFSET( or simply :A10 depth = 0 @@ -139,7 +136,7 @@ def shunting_yard(expression, named_ranges, ref = None, tokenize_range = False): if depth == 0: new_tokens.pop() # these 2 lines are needed to remove INDEX() new_tokens.pop() - expr = six.next(rev).tvalue + expr + expr = next(rev).tvalue + expr break expr += token.tvalue @@ -377,13 +374,11 @@ def cell2code(cell, named_ranges): else: ast = None - if isinstance(cell.value, unicode): - code = u'u"' + cell.value.replace(u'"', u'\\"') + u'"' - elif isinstance(cell.value, str): - raise RuntimeError("Got unexpected non-unicode str") + if isinstance(cell.value, str): + code = '"' + cell.value.replace('"', r'\"') + '"' else: code = str(cell.value) - return code,ast + return code, ast def prepare_pointer(code, names, ref_cell = None): diff --git a/koala/ast/astnodes.py b/koala/ast/astnodes.py index 03bceb44..91569214 100644 --- a/koala/ast/astnodes.py +++ b/koala/ast/astnodes.py @@ -1,25 +1,21 @@ -from __future__ import print_function # cython: profile=True from networkx import NetworkXError -from openpyxl.compat import unicode - from koala.excellib import FUNCTION_MAP, IND_FUN from koala.utils import is_range, split_range, split_address, resolve_range from koala.ExcelError import * def to_str(my_string): - # `unicode` != `str` in Python2. See `from openpyxl.compat import unicode` - if type(my_string) == str and str != unicode: - return unicode(my_string, 'utf-8') - elif type(my_string) == unicode: + if isinstance(my_string, bytes): + return my_string.decode("utf-8") + elif isinstance(my_string, str): return my_string else: try: return str(my_string) - except: + except Exception: print('Couldnt parse as string', type(my_string)) return my_string # elif isinstance(my_string, (int, float, tuple, Ra): @@ -45,7 +41,7 @@ def __getattr__(self, name): def children(self, ast): try: args = ast.predecessors(self) - args = sorted(args, key=lambda x: ast.node[x]['pos']) + args = sorted(args, key=lambda x: ast.nodes[x]['pos']) except NetworkXError: args = '' return args diff --git a/koala/excellib.py b/koala/excellib.py index 46b85008..01172e5a 100644 --- a/koala/excellib.py +++ b/koala/excellib.py @@ -6,10 +6,9 @@ # source: https://github.com/dgorissen/pycel/blob/master/src/pycel/excellib.py -from __future__ import absolute_import, division - import itertools import numpy as np +import numpy_financial as npf import scipy.optimize import datetime import random @@ -18,8 +17,6 @@ from calendar import monthrange from dateutil.relativedelta import relativedelta -from openpyxl.compat import unicode - from koala.utils import * from koala.Range import RangeCore as Range from koala.ExcelError import * @@ -427,7 +424,7 @@ def irr(values, guess = None): raise ValueError('guess value for excellib.irr() is %s and not 0' % guess) else: try: - return np.irr(values) + return npf.irr(values) except Exception as e: return ExcelError('#NUM!', e) @@ -729,7 +726,7 @@ def randbetween(bottom, top): def right(text,n): #TODO: hack to deal with naca section numbers - if isinstance(text, unicode) or isinstance(text,str): + if isinstance(text, str): return text[-n:] else: # TODO: get rid of the decimal diff --git a/koala/reader.py b/koala/reader.py index 8a679204..9403d902 100644 --- a/koala/reader.py +++ b/koala/reader.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from io import BytesIO import re import os @@ -108,6 +106,8 @@ def read_cells(archive, ignore_sheets = [], ignore_hidden = False): cells = {} + sheets = [] + functions = set() cts = dict(read_content_types(archive)) @@ -127,6 +127,9 @@ def read_cells(archive, ignore_sheets = [], ignore_hidden = False): if sheet_name in ignore_sheets: continue + if sheet_name not in sheets: + sheets.append(sheet_name) + root = fromstring(archive.read(sheet['path'])) # it is necessary to use cElementTree from xml module, otherwise root.findall doesn't work as it should hidden_cols = False @@ -225,7 +228,7 @@ def read_cells(archive, ignore_sheets = [], ignore_hidden = False): # if f not in existing: # print('== Missing function: %s' % f) - return cells + return cells, sheets def read_rels(archive): diff --git a/koala/serializer.py b/koala/serializer.py index 27dd2d79..f85b8a28 100644 --- a/koala/serializer.py +++ b/koala/serializer.py @@ -1,13 +1,9 @@ -from __future__ import absolute_import, print_function - import json import gzip import networkx from networkx.classes.digraph import DiGraph -from networkx.readwrite import json_graph from networkx.drawing.nx_pydot import write_dot -from openpyxl.compat import unicode from koala.Cell import Cell from koala.Range import RangeCore, RangeFactory @@ -51,10 +47,10 @@ def parse_cell_info(cell): for cell in simple_cells: parse_cell_info(cell) value = cell.value - if isinstance(value, unicode): + if isinstance(value, str): outfile.write(cell.value.encode('utf-8') + b"\n") else: - outfile.write((str(cell.value) + u"\n").encode('utf-8')) + outfile.write((str(cell.value) + "\n").encode('utf-8')) outfile.write(b"====" + b"\n") outfile.write(b"-----" + b"\n") @@ -205,41 +201,15 @@ def dump_json(self, fname): def load_json(fname): - def _decode_list(data): - rv = [] - for item in data: - if isinstance(item, unicode) and unicode != str: - item = item.encode('utf-8') - elif isinstance(item, list) and unicode != str: - item = _decode_list(item) - elif isinstance(item, dict): - item = _decode_dict(item) - rv.append(item) - return rv - - def _decode_dict(data): - rv = {} - for key, value in data.items(): - if isinstance(key, unicode) and unicode != str: - key = key.encode('utf-8') - if isinstance(value, unicode) and unicode != str: - value = value.encode('utf-8') - elif isinstance(value, list): - value = _decode_list(value) - elif isinstance(value, dict): - value = _decode_dict(value) - rv[key] = value - return rv - with gzip.GzipFile(fname, 'r') as infile: - data = json.loads(infile.read().decode('utf-8'), object_hook=_decode_dict) + data = json.loads(infile.read().decode('utf-8')) return data ########### based on dot ################# -def export_to_dot(self,fname): - write_dot(self.G,fname) +def export_to_dot(self, fname): + write_dot(self.G, fname) ########### plotting ################# diff --git a/koala/tokenizer.py b/koala/tokenizer.py index 03fbdf45..60df934f 100644 --- a/koala/tokenizer.py +++ b/koala/tokenizer.py @@ -1,5 +1,3 @@ -from __future__ import division, print_function - # cython: profile=True #======================================================================== @@ -27,11 +25,8 @@ #======================================================================== import re -import six import collections -from koala.utils import old_div - #======================================================================== # Class: ExcelParserTokens @@ -433,7 +428,7 @@ def EOF(): # standard postfix operators if ("%".find(currentChar()) != -1): if (len(token) > 0): - tokens.add(old_div(float(token), 100), self.TOK_TYPE_OPERAND) + tokens.add(float(token) / 100, self.TOK_TYPE_OPERAND) token = "" else: tokens.add('*', self.TOK_TYPE_OP_IN) @@ -501,9 +496,9 @@ def EOF(): ): pass elif (not( - ((six.next(tokens).ttype == self.TOK_TYPE_FUNCTION) and (tokens.next().tsubtype == self.TOK_SUBTYPE_START)) or - ((six.next(tokens).ttype == self.TOK_TYPE_SUBEXPR) and (tokens.next().tsubtype == self.TOK_SUBTYPE_START)) or - (six.next(tokens).ttype == self.TOK_TYPE_OPERAND) + ((next(tokens).ttype == self.TOK_TYPE_FUNCTION) and (tokens.next().tsubtype == self.TOK_SUBTYPE_START)) or + ((next(tokens).ttype == self.TOK_TYPE_SUBEXPR) and (tokens.next().tsubtype == self.TOK_SUBTYPE_START)) or + (next(tokens).ttype == self.TOK_TYPE_OPERAND) ) ): pass diff --git a/koala/utils.py b/koala/utils.py index 626f1980..242d2062 100644 --- a/koala/utils.py +++ b/koala/utils.py @@ -1,20 +1,11 @@ # cython: profile=True -from __future__ import absolute_import, division - -import collections -import numbers +import collections.abc import re import datetime as dt -try: - from functools import lru_cache -except ImportError: # fix for Python 2.7 - from backports.functools_lru_cache import lru_cache -from six import string_types +from functools import lru_cache from copy import deepcopy -from openpyxl.compat import unicode - from .ExcelError import ExcelError ASCII = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -150,12 +141,6 @@ def resolve_range(rng, should_flatten = False, sheet=''): else: pass - # `unicode` != `str` in Python2. See `from openpyxl.compat import unicode` - if type(sheet) == str and str != unicode: - sheet = unicode(sheet, 'utf-8') - if type(rng) == str and str != unicode: - rng = unicode(rng, 'utf-8') - key = rng+str(should_flatten)+sheet if key in resolve_range_cache: @@ -347,10 +332,10 @@ def get_linest_degree(excel,cl): return (max(degree,1),coef) def flatten(l, only_lists = False): - instance = list if only_lists else collections.Iterable + instance = list if only_lists else collections.abc.Iterable for el in l: - if isinstance(el, instance) and not isinstance(el, string_types): + if isinstance(el, instance) and not isinstance(el, str): for sub in flatten(el, only_lists = only_lists): yield sub else: @@ -587,7 +572,7 @@ def extract_numeric_values(*args): values = [] for arg in args: - if isinstance(arg, collections.Iterable) and type(arg) != list and type(arg) != tuple and type(arg) != str and type(arg) != unicode: # does not work fo other Iterable than RangeCore, but can t import RangeCore here for circular reference issues + if isinstance(arg, collections.abc.Iterable) and type(arg) != list and type(arg) != tuple and type(arg) != str: # does not work fo other Iterable than RangeCore, but can t import RangeCore here for circular reference issues values.extend([x for x in arg.values if is_number(x) and type(x) is not bool]) # for x in arg.values: # if is_number(x) and type(x) is not bool: # excludes booleans from nested ranges @@ -602,21 +587,6 @@ def extract_numeric_values(*args): return values - -def old_div(a, b): - """ - Equivalent to ``a / b`` on Python 2 without ``from __future__ import - division``. - - Copied from: - https://github.com/PythonCharmers/python-future/blob/master/src/past/utils/__init__.py - """ - if isinstance(a, numbers.Integral) and isinstance(b, numbers.Integral): - return a // b - else: - return a / b - - def safe_iterator(node, tag=None): """Return an iterator or an empty list""" if node is None: diff --git a/requirements.txt b/requirements.txt index c676f783..4486046f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ -networkx==2.1 -openpyxl==2.5.3 +networkx>=2.4 +openpyxl>=3.0.3 numpy>=1.14.2 -Cython==0.28.2 -lxml==4.1.1 -six==1.11.0 +numpy-financial>=1.0.0 +Cython>=0.29.15 +lxml>=4.5.0 scipy>=1.0.0 -python-dateutil==2.8.0 -backports.functools_lru_cache==1.5 +python-dateutil>=2.8.0 diff --git a/setup.py b/setup.py index ace79353..9ad7b5b3 100644 --- a/setup.py +++ b/setup.py @@ -63,14 +63,13 @@ install_requires=[ - 'networkx >= 2.1', - 'openpyxl >= 2.5.3', + 'networkx >= 2.4', + 'openpyxl >= 3.0.3', 'numpy >= 1.14.2', - 'Cython >= 0.28.2', - 'lxml >= 4.1.1', - 'six >= 1.11.0', + 'numpy-financial>=1.0.0', + 'Cython >= 0.29.15', + 'lxml >= 4.5.0', 'scipy>=1.0.0', - 'python-dateutil==2.8.0', - 'backports.functools_lru_cache==1.5' + 'python-dateutil==2.8.0' ] ) diff --git a/tests/ast/test_range.py b/tests/ast/test_range.py index c51be38e..346045fd 100644 --- a/tests/ast/test_range.py +++ b/tests/ast/test_range.py @@ -1,4 +1,3 @@ -from __future__ import print_function import unittest from koala.Range import RangeFactory diff --git a/tests/excel/test_functions.py b/tests/excel/test_functions.py index 1d1b76a9..7be69dee 100644 --- a/tests/excel/test_functions.py +++ b/tests/excel/test_functions.py @@ -1,6 +1,5 @@ -from __future__ import absolute_import - -import pyximport; pyximport.install() +import pyximport +pyximport.install() import unittest diff --git a/tests/excel/test_utils.py b/tests/excel/test_utils.py index c36b207f..7ad16685 100644 --- a/tests/excel/test_utils.py +++ b/tests/excel/test_utils.py @@ -1,6 +1,6 @@ -from __future__ import absolute_import +import pyximport -import pyximport; pyximport.install() +pyximport.install() import unittest