From 1a123e51705483dc36fa99b86bfb6b3722387c98 Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Wed, 12 Jan 2022 16:52:55 -0800 Subject: [PATCH 1/7] first round of unit tests for node transformer. --- .../unit/transformer/test_node_transformer.py | 48 +++++++++++++++++++ .../{ => transformer}/test_source_giver.py | 13 +++-- 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 tests/unit/transformer/test_node_transformer.py rename tests/unit/{ => transformer}/test_source_giver.py (61%) diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py new file mode 100644 index 000000000..d979f365a --- /dev/null +++ b/tests/unit/transformer/test_node_transformer.py @@ -0,0 +1,48 @@ +# +# +# type:ignore +import ast + +import pytest +from mock import MagicMock + +from lineapy.transformer.node_transformer import NodeTransformer + + +class TestNodeTransformer: + nt: NodeTransformer + + @pytest.fixture(autouse=True) + def before_everything(self): + nt = NodeTransformer( + "", MagicMock(), MagicMock() # SourceCodeLocation(0, 0, 0, 0) + ) + assert nt is not None + self.nt = nt + + def test_lambda_executes_as_expression(self): + self.nt._exec_expression = MagicMock() + self.nt._exec_statement = MagicMock() + + # this inits an ast.Module containing one expression whose value is a ast.lambda + test_node = ast.parse("lambda x: x + 10") + lambda_node = test_node.body[0].value + self.nt.generic_visit(lambda_node) + self.nt._exec_statement.assert_not_called() + self.nt._exec_expression.assert_called_once() + + def test_assign_executes(self): + test_node = ast.parse("a = 10") + self.nt.visit_Assign = MagicMock() + self.nt.visit(test_node.body[0]) + self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) + + def test_assign_calls_tracer_assign(self): + self.nt.get_source = MagicMock() + test_node = ast.parse("a = 10") + tracer = self.nt.tracer + self.nt.visit(test_node.body[0]) + # self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) + tracer.assign.assert_called_once_with( + "a", self.nt.tracer.literal.return_value + ) diff --git a/tests/unit/test_source_giver.py b/tests/unit/transformer/test_source_giver.py similarity index 61% rename from tests/unit/test_source_giver.py rename to tests/unit/transformer/test_source_giver.py index 28d9a31cf..8e6661ef7 100644 --- a/tests/unit/test_source_giver.py +++ b/tests/unit/transformer/test_source_giver.py @@ -6,6 +6,8 @@ from lineapy.transformer.source_giver import SourceGiver +# from astpretty import pprint + @pytest.mark.parametrize( "code,lineno", @@ -28,14 +30,19 @@ def test_source_giver_adds_end_lineno(code, lineno): from astpretty import pprint tree = ast.parse(code) + # ensure that the end_lineno is not available and fetching it raises exceptions with pytest.raises(AttributeError): print(tree.body[0].end_lineno) + # now we invoke the SourceGiver and add end_linenos in 2 steps - first we run the tree thr asttokens asttokens.ASTTokens(code, parse=False, tree=tree) - pprint(tree) - print(tree.body[0].last_token) + # pprint(tree) + # print(tree.body[0].last_token) + # double check that the line numbers cooked up by asttokens are correct assert tree.body[0].last_token.end[0] == lineno + # and in step 2, run the tree thr SourceGiver and copy the asttokens's token values + # so that the tree looks like 3.8+ tree with all the end_linenos etc SourceGiver().transform(tree) assert tree.body[0].end_lineno == lineno - print(tree) + # print(tree) From 751d21b0f7ae40895270cead79995f13ca1949bb Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Thu, 13 Jan 2022 17:46:07 -0800 Subject: [PATCH 2/7] remove unused code portions, add tests for multiple visitors in node_transformer. --- lineapy/transformer/node_transformer.py | 20 +-- lineapy/transformer/source_giver.py | 31 ---- .../unit/transformer/test_node_transformer.py | 132 +++++++++++++++++- 3 files changed, 133 insertions(+), 50 deletions(-) diff --git a/lineapy/transformer/node_transformer.py b/lineapy/transformer/node_transformer.py index 691b5253d..843704657 100644 --- a/lineapy/transformer/node_transformer.py +++ b/lineapy/transformer/node_transformer.py @@ -569,21 +569,11 @@ def visit_Subscript(self, node: ast.Subscript) -> CallNode: args = [self.visit(node.value)] index = node.slice args.append(self.visit(index)) - if isinstance(node.ctx, ast.Load): - return self.tracer.call( - self.tracer.lookup_node(GET_ITEM), - self.get_source(node), - *args, - ) - elif isinstance(node.ctx, ast.Del): - raise NotImplementedError( - "Subscript with ctx=ast.Del() not supported." - ) - else: - raise ValueError( - "Subscript with ctx=ast.Store() should have been handled by" - " visit_Assign." - ) + return self.tracer.call( + self.tracer.lookup_node(GET_ITEM), + self.get_source(node), + *args, + ) def visit_Attribute(self, node: ast.Attribute) -> CallNode: diff --git a/lineapy/transformer/source_giver.py b/lineapy/transformer/source_giver.py index de13459b3..a7286cf29 100644 --- a/lineapy/transformer/source_giver.py +++ b/lineapy/transformer/source_giver.py @@ -2,9 +2,6 @@ class SourceGiver: - def __init__(self): - pass - def transform(self, nodes: ast.Module) -> None: """ This call should only happen once asttoken has run its magic @@ -26,31 +23,3 @@ def transform(self, nodes: ast.Module) -> None: node.end_col_offset = node.last_token.end[1] # type: ignore # if isinstance(node, ast.ListComp): node.col_offset = node.first_token.start[1] # type: ignore - - def transform_inhouse(self, nodes: ast.Module) -> None: - curr_line_start: int = -1 - curr_line_offset: int = -1 - prev_line_start: int = -1 - prev_line_offset: int = -1 - prev_node = None - node: ast.AST - # TODO check if the ast type is a Module instead of simply relying on mypy - for node in ast.walk(nodes): - if not hasattr(node, "lineno"): - continue - - curr_line_start = node.lineno - curr_line_offset = node.col_offset - if prev_node is not None: - prev_node.end_lineno = max( # type: ignore - prev_line_start, curr_line_start - 1 - ) - if prev_line_start == curr_line_start: - prev_node.end_col_offset = max( - prev_line_offset, curr_line_offset - 1 - ) - else: - prev_node.end_col_offset = -1 - prev_node = node - prev_line_start = curr_line_start - prev_line_offset = curr_line_offset diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py index d979f365a..e7ac863d7 100644 --- a/tests/unit/transformer/test_node_transformer.py +++ b/tests/unit/transformer/test_node_transformer.py @@ -4,14 +4,79 @@ import ast import pytest -from mock import MagicMock +from mock import MagicMock, patch -from lineapy.transformer.node_transformer import NodeTransformer +from lineapy.transformer.node_transformer import NodeTransformer, transform + + +@patch( + "lineapy.transformer.node_transformer.NodeTransformer", +) +def test_transform_fn(nt_mock: MagicMock): + """ + Test that the transform function calls the NodeTransformer + """ + mocked_tracer = MagicMock() + source_location = MagicMock() + transform("x = 1", source_location, mocked_tracer) + nt_mock.assert_called_once_with("x = 1", source_location, mocked_tracer) + mocked_tracer.db.commit.assert_called_once() + # TODO - test that source giver is called only for 3.7 and below class TestNodeTransformer: nt: NodeTransformer + basic_tests = ( + "code, visitor, call_count", + [ + ("a[2:3]", "Slice", 2), + ("a[2:3:2]", "Slice", 2), + ("a[2:3]", "Subscript", 2), + ("a.x", "Attribute", 1), + ("{'a': 1}", "Dict", 1), + ("a < b", "Compare", 1), + ("a not in []", "Compare", 3), + ("a + b", "BinOp", 1), + ("a or b", "BoolOp", 1), + ("[1,2]", "List", 1), + # ("{1,2}", "Set", 1), + # tuple eventually calls tracer.call but we've mocked out the whole thing + ("(1,2)", "Tuple", 0), + ("not True", "UnaryOp", 1), + ("10", "Constant", 0), # this will break with 3.7 + ("assert True", "Assert", 1), + ("fn(*args)", "Expr", 1), + ("fn(*args)", "Call", 1), + ("fn(*args)", "Starred", 1), + ("fn(*args)", "Name", 1), + ], + ) + # TODO pull out constant to handle 3.7 vs 3.9 etc + # TODO handle visit_Name as its own test + # TODO import, assert, module, starred + basic_test_ids = [ + "slice", + "slice_with_step", + "subscript", + "attribute", + "dict", + "compare", + "compare_notin", + "binop", + "boolop", + "list", + # "set", + "tuple", + "unaryop", + "constant", + "assert", + "expr", + "call", + "starred", + "name", + ] + @pytest.fixture(autouse=True) def before_everything(self): nt = NodeTransformer( @@ -43,6 +108,65 @@ def test_assign_calls_tracer_assign(self): tracer = self.nt.tracer self.nt.visit(test_node.body[0]) # self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) - tracer.assign.assert_called_once_with( - "a", self.nt.tracer.literal.return_value + tracer.assign.assert_called_once_with("a", tracer.literal.return_value) + + @pytest.mark.parametrize( + "code", + ["a[3]=10", "a.x =10"], + ids=["assign_subscript", "assign_attribute"], + ) + def test_assign_subscript_attribute_calls_tracer_assign(self, code): + self.nt.get_source = MagicMock() + test_node = ast.parse(code) + tracer = self.nt.tracer + self.nt.visit(test_node.body[0]) + tracer.call.assert_called_once_with( + tracer.lookup_node.return_value, + self.nt.get_source.return_value, + tracer.lookup_node.return_value, + tracer.literal.return_value, + tracer.literal.return_value, + ) + + def test_visit_delete_executes(self): + test_node = ast.parse("del a") + with pytest.raises(NotImplementedError): + self.nt.visit_Delete(test_node.body[0]) + + self.nt.visit_Delete = MagicMock() + self.nt.visit(test_node.body[0]) + self.nt.visit_Delete.assert_called_once_with(test_node.body[0]) + + @pytest.mark.parametrize( + "code", ["del a[3]", "del a.x"], ids=["delitem", "delattr"] + ) + def test_visit_delete_subscript_attribute_calls_tracer_call(self, code): + self.nt.get_source = MagicMock() + test_node = ast.parse(code) + tracer = self.nt.tracer + self.nt.visit(test_node.body[0]) + tracer.call.assert_called_once_with( + tracer.lookup_node.return_value, + self.nt.get_source.return_value, + tracer.lookup_node.return_value, + tracer.literal.return_value, ) + + # catch all for any ast nodes that do not have if conditions and/or very little logic + + @pytest.mark.parametrize(*basic_tests, ids=basic_test_ids) + def test_code_visited_calls_tracer_call(self, code, visitor, call_count): + test_node = ast.parse(code) + self.nt.visit(test_node) + # doing this so that we can select which function in tracer gets called. + # might be overkill though so leaving it at this + tracer_fn = getattr(self.nt.tracer, "call") + assert tracer_fn.call_count == call_count + + @pytest.mark.parametrize(*basic_tests, ids=basic_test_ids) + def test_code_visits_right_visitor(self, code, visitor, call_count): + test_node = ast.parse(code) + self.nt.__setattr__("visit_" + visitor, MagicMock()) + nt_visitor = self.nt.__getattribute__("visit_" + visitor) + self.nt.visit(test_node) + nt_visitor.assert_called_once() From 5fec4f5f48bce997edda5c4a30dfcb775cc106b5 Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Fri, 14 Jan 2022 10:39:14 -0800 Subject: [PATCH 3/7] add fixes for py37, update versions --- requirements.txt | 46 +++++----- setup.py | 2 +- .../unit/transformer/test_node_transformer.py | 88 +++++++++++-------- 3 files changed, 77 insertions(+), 59 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0a3bac3eb..7c1e7eae4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ alabaster==0.7.12 altair==4.2.0 -anyio==3.4.0 +anyio==3.5.0 appnope==0.1.2 -argcomplete==2.0.0 argon2-cffi==21.3.0 argon2-cffi-bindings==21.2.0 astor==0.8.1 @@ -27,42 +26,42 @@ coveralls==3.3.1 cramjam==2.5.0 cycler==0.11.0 debugpy==1.5.1 -decorator==5.1.0 +decorator==5.1.1 defusedxml==0.7.1 distlib==0.3.4 docopt==0.6.2 docutils==0.17.1 entrypoints==0.3 execnet==1.9.0 +executing==0.8.2 fancycompleter==0.9.1 fastparquet==0.7.2 filelock==3.4.2 flake8==4.0.1 fonttools==4.28.5 -fsspec==2021.11.1 +fsspec==2022.1.0 graphviz==0.19.1 greenlet==1.1.2 -identify==2.4.1 +identify==2.4.4 idna==3.3 imagesize==1.3.0 -importlib-metadata==4.2.0 importlib-resources==5.4.0 iniconfig==1.1.1 -ipykernel==6.6.1 -ipython==7.31.0 +ipykernel==6.7.0 +ipython==8.0.0 ipython-genutils==0.2.0 isort==5.10.1 jedi==0.18.1 Jinja2==3.0.3 joblib==1.1.0 json5==0.9.6 -jsonschema==4.3.3 +jsonschema==4.4.0 jupyter-client==7.1.0 jupyter-core==4.9.1 -jupyter-server==1.13.1 -jupyterlab==3.2.5 +jupyter-server==1.13.2 +jupyterlab==3.2.8 jupyterlab-pygments==0.1.2 -jupyterlab-server==2.10.2 +jupyterlab-server==2.10.3 kiwisolver==1.3.2 MarkupSafe==2.0.1 matplotlib==3.5.1 @@ -70,23 +69,23 @@ matplotlib-inline==0.1.3 mccabe==0.6.1 mistune==0.8.4 mock==4.0.3 -mypy==0.910 +mypy==0.931 mypy-extensions==0.4.3 -nbclassic==0.3.4 -nbclient==0.5.9 +nbclassic==0.3.5 +nbclient==0.5.10 nbconvert==6.4.0 nbformat==5.1.3 nbval==0.9.6 nest-asyncio==1.5.4 networkx==2.6.3 nodeenv==1.6.0 -notebook==6.4.6 -numpy==1.21.5 +notebook==6.4.7 +numpy==1.22.0 packaging==21.3 pandas==1.3.5 pandocfilters==1.5.0 parso==0.8.3 -path==16.2.0 +path==16.3.0 path.py==12.5.0 pathspec==0.9.0 pdbpp==0.10.3 @@ -101,12 +100,13 @@ prometheus-client==0.12.0 prompt-toolkit==3.0.24 psycopg2==2.9.3 ptyprocess==0.7.0 +pure-eval==0.2.1 py==1.11.0 pycodestyle==2.8.0 pycparser==2.21 pydantic==1.9.0 pyflakes==2.4.0 -Pygments==2.11.1 +Pygments==2.11.2 PyOpenGL==3.1.5 pyparsing==3.0.6 pyrepl==0.9.0 @@ -123,7 +123,7 @@ pytz==2021.3 PyYAML==6.0 pyzmq==22.3.0 requests==2.27.1 -rich==10.16.2 +rich==11.0.0 scikit-learn==1.0.2 scipy==1.7.3 scour==0.38.2 @@ -143,6 +143,7 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 SQLAlchemy==1.4.29 sqlalchemy2-stubs==0.0.2a19 +stack-data==0.1.3 syrupy==1.4.5 termcolor==1.1.0 terminado==0.12.1 @@ -154,10 +155,9 @@ tomli==1.2.3 toolz==0.11.2 tornado==6.1 traitlets==5.1.1 -typed-ast==1.4.3 -types-PyYAML==6.0.1 +types-PyYAML==6.0.3 typing_extensions==4.0.1 -urllib3==1.26.7 +urllib3==1.26.8 virtualenv==20.13.0 wcwidth==0.2.5 webencodings==0.5.1 diff --git a/setup.py b/setup.py index 70950630c..4d3b6949a 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ def version(path): "flake8", "fastparquet", "syrupy==1.4.5", - "mypy<0.920", + "mypy", "isort", "pytest", "matplotlib", diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py index e7ac863d7..351621011 100644 --- a/tests/unit/transformer/test_node_transformer.py +++ b/tests/unit/transformer/test_node_transformer.py @@ -2,11 +2,23 @@ # # type:ignore import ast +import sys +import asttokens import pytest from mock import MagicMock, patch from lineapy.transformer.node_transformer import NodeTransformer, transform +from lineapy.transformer.source_giver import SourceGiver + + +def _get_ast_node(code): + node = ast.parse(code) + if sys.version_info < (3, 8): # give me endlines! + asttokens.ASTTokens(code, parse=False, tree=node) + SourceGiver().transform(node) + + return node @patch( @@ -27,34 +39,31 @@ def test_transform_fn(nt_mock: MagicMock): class TestNodeTransformer: nt: NodeTransformer - basic_tests = ( - "code, visitor, call_count", - [ - ("a[2:3]", "Slice", 2), - ("a[2:3:2]", "Slice", 2), - ("a[2:3]", "Subscript", 2), - ("a.x", "Attribute", 1), - ("{'a': 1}", "Dict", 1), - ("a < b", "Compare", 1), - ("a not in []", "Compare", 3), - ("a + b", "BinOp", 1), - ("a or b", "BoolOp", 1), - ("[1,2]", "List", 1), - # ("{1,2}", "Set", 1), - # tuple eventually calls tracer.call but we've mocked out the whole thing - ("(1,2)", "Tuple", 0), - ("not True", "UnaryOp", 1), - ("10", "Constant", 0), # this will break with 3.7 - ("assert True", "Assert", 1), - ("fn(*args)", "Expr", 1), - ("fn(*args)", "Call", 1), - ("fn(*args)", "Starred", 1), - ("fn(*args)", "Name", 1), - ], - ) + basic_tests_list = [ + ("a[2:3]", "Slice", 2), + ("a[2:3:2]", "Slice", 2), + ("a[2:3]", "Subscript", 2), + ("a.x", "Attribute", 1), + ("{'a': 1}", "Dict", 1), + ("a < b", "Compare", 1), + ("a not in []", "Compare", 3), + ("a + b", "BinOp", 1), + ("a or b", "BoolOp", 1), + ("[1,2]", "List", 1), + # ("{1,2}", "Set", 1), + # tuple eventually calls tracer.call but we've mocked out the whole thing + ("(1,2)", "Tuple", 0), + ("not True", "UnaryOp", 1), + ("assert True", "Assert", 1), + ("fn(*args)", "Expr", 1), + ("fn(*args)", "Call", 1), + ("fn(*args)", "Starred", 1), + ("fn(*args)", "Name", 1), + ] + # TODO pull out constant to handle 3.7 vs 3.9 etc # TODO handle visit_Name as its own test - # TODO import, assert, module, starred + # TODO import, module basic_test_ids = [ "slice", "slice_with_step", @@ -69,7 +78,6 @@ class TestNodeTransformer: # "set", "tuple", "unaryop", - "constant", "assert", "expr", "call", @@ -77,6 +85,16 @@ class TestNodeTransformer: "name", ] + if sys.version_info < (3, 8): + basic_tests_list += (("10", "Num", 0),) + basic_test_ids += ["num"] + else: + # this will break with 3.7 + basic_tests_list += (("10", "Constant", 0),) + basic_test_ids += ["constant"] + + basic_tests = ("code, visitor, call_count", basic_tests_list) + @pytest.fixture(autouse=True) def before_everything(self): nt = NodeTransformer( @@ -90,21 +108,21 @@ def test_lambda_executes_as_expression(self): self.nt._exec_statement = MagicMock() # this inits an ast.Module containing one expression whose value is a ast.lambda - test_node = ast.parse("lambda x: x + 10") + test_node = _get_ast_node("lambda x: x + 10") lambda_node = test_node.body[0].value self.nt.generic_visit(lambda_node) self.nt._exec_statement.assert_not_called() self.nt._exec_expression.assert_called_once() def test_assign_executes(self): - test_node = ast.parse("a = 10") + test_node = _get_ast_node("a = 10") self.nt.visit_Assign = MagicMock() self.nt.visit(test_node.body[0]) self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) def test_assign_calls_tracer_assign(self): self.nt.get_source = MagicMock() - test_node = ast.parse("a = 10") + test_node = _get_ast_node("a = 10") tracer = self.nt.tracer self.nt.visit(test_node.body[0]) # self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) @@ -117,7 +135,7 @@ def test_assign_calls_tracer_assign(self): ) def test_assign_subscript_attribute_calls_tracer_assign(self, code): self.nt.get_source = MagicMock() - test_node = ast.parse(code) + test_node = _get_ast_node(code) tracer = self.nt.tracer self.nt.visit(test_node.body[0]) tracer.call.assert_called_once_with( @@ -129,7 +147,7 @@ def test_assign_subscript_attribute_calls_tracer_assign(self, code): ) def test_visit_delete_executes(self): - test_node = ast.parse("del a") + test_node = _get_ast_node("del a") with pytest.raises(NotImplementedError): self.nt.visit_Delete(test_node.body[0]) @@ -142,7 +160,7 @@ def test_visit_delete_executes(self): ) def test_visit_delete_subscript_attribute_calls_tracer_call(self, code): self.nt.get_source = MagicMock() - test_node = ast.parse(code) + test_node = _get_ast_node(code) tracer = self.nt.tracer self.nt.visit(test_node.body[0]) tracer.call.assert_called_once_with( @@ -156,7 +174,7 @@ def test_visit_delete_subscript_attribute_calls_tracer_call(self, code): @pytest.mark.parametrize(*basic_tests, ids=basic_test_ids) def test_code_visited_calls_tracer_call(self, code, visitor, call_count): - test_node = ast.parse(code) + test_node = _get_ast_node(code) self.nt.visit(test_node) # doing this so that we can select which function in tracer gets called. # might be overkill though so leaving it at this @@ -165,7 +183,7 @@ def test_code_visited_calls_tracer_call(self, code, visitor, call_count): @pytest.mark.parametrize(*basic_tests, ids=basic_test_ids) def test_code_visits_right_visitor(self, code, visitor, call_count): - test_node = ast.parse(code) + test_node = _get_ast_node(code) self.nt.__setattr__("visit_" + visitor, MagicMock()) nt_visitor = self.nt.__getattribute__("visit_" + visitor) self.nt.visit(test_node) From 52d31c81693c7e91758697ea812069642b655302 Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Fri, 14 Jan 2022 11:18:38 -0800 Subject: [PATCH 4/7] rollback the list to python 3.7. always generate from 3.7 to avoid issues with older versions. --- requirements.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7c1e7eae4..4495a6a1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,6 @@ docopt==0.6.2 docutils==0.17.1 entrypoints==0.3 execnet==1.9.0 -executing==0.8.2 fancycompleter==0.9.1 fastparquet==0.7.2 filelock==3.4.2 @@ -45,10 +44,11 @@ greenlet==1.1.2 identify==2.4.4 idna==3.3 imagesize==1.3.0 +importlib-metadata==4.2.0 importlib-resources==5.4.0 iniconfig==1.1.1 ipykernel==6.7.0 -ipython==8.0.0 +ipython==7.31.0 ipython-genutils==0.2.0 isort==5.10.1 jedi==0.18.1 @@ -58,7 +58,7 @@ json5==0.9.6 jsonschema==4.4.0 jupyter-client==7.1.0 jupyter-core==4.9.1 -jupyter-server==1.13.2 +jupyter-server==1.13.3 jupyterlab==3.2.8 jupyterlab-pygments==0.1.2 jupyterlab-server==2.10.3 @@ -80,7 +80,7 @@ nest-asyncio==1.5.4 networkx==2.6.3 nodeenv==1.6.0 notebook==6.4.7 -numpy==1.22.0 +numpy==1.21.5 packaging==21.3 pandas==1.3.5 pandocfilters==1.5.0 @@ -100,7 +100,6 @@ prometheus-client==0.12.0 prompt-toolkit==3.0.24 psycopg2==2.9.3 ptyprocess==0.7.0 -pure-eval==0.2.1 py==1.11.0 pycodestyle==2.8.0 pycparser==2.21 @@ -143,7 +142,6 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 SQLAlchemy==1.4.29 sqlalchemy2-stubs==0.0.2a19 -stack-data==0.1.3 syrupy==1.4.5 termcolor==1.1.0 terminado==0.12.1 @@ -155,6 +153,7 @@ tomli==1.2.3 toolz==0.11.2 tornado==6.1 traitlets==5.1.1 +typed-ast==1.5.1 types-PyYAML==6.0.3 typing_extensions==4.0.1 urllib3==1.26.8 From d51f6db035784b79690cd95d624eee15ab00b73d Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Fri, 14 Jan 2022 12:07:27 -0800 Subject: [PATCH 5/7] more missing test cases --- .../unit/transformer/test_node_transformer.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py index 351621011..ad50642a0 100644 --- a/tests/unit/transformer/test_node_transformer.py +++ b/tests/unit/transformer/test_node_transformer.py @@ -50,6 +50,7 @@ class TestNodeTransformer: ("a + b", "BinOp", 1), ("a or b", "BoolOp", 1), ("[1,2]", "List", 1), + # set is xfailing right now # ("{1,2}", "Set", 1), # tuple eventually calls tracer.call but we've mocked out the whole thing ("(1,2)", "Tuple", 0), @@ -59,6 +60,15 @@ class TestNodeTransformer: ("fn(*args)", "Call", 1), ("fn(*args)", "Starred", 1), ("fn(*args)", "Name", 1), + ("print(*'mystring')", "Starred", 1), + # calls tracer.trace_import but this list checks for tracer.call calls + ("import math", "Import", 0), + # calls tracer.trace_import but this list checks for tracer.call calls + # TODO - import from calls transform utils which should really be mocked out and + # tested on their own, in true spirit of unit testing + ("from math import sqrt", "ImportFrom", 0), + ("a, b = (1,2)", "Assign", 2), + ("lambda x: x + 10", "Lambda", 1), ] # TODO pull out constant to handle 3.7 vs 3.9 etc @@ -83,15 +93,26 @@ class TestNodeTransformer: "call", "starred", "name", + "starred_str", + "import", + "import_from", + "assign_tuple", + "lambda", ] if sys.version_info < (3, 8): basic_tests_list += (("10", "Num", 0),) + # extslice does not call tracer.call but it contains a slice node. + # that along with subscript will result in two calls + basic_tests_list += (("a[:,3]", "ExtSlice", 2),) basic_test_ids += ["num"] + basic_test_ids += ["extslice"] else: # this will break with 3.7 basic_tests_list += (("10", "Constant", 0),) + basic_tests_list += (("a[:,3]", "Slice", 2),) basic_test_ids += ["constant"] + basic_test_ids += ["slice_with_ext"] basic_tests = ("code, visitor, call_count", basic_tests_list) From 8bac28bd066237a342fbc865bedf69de81a1c9ef Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Fri, 14 Jan 2022 13:59:15 -0800 Subject: [PATCH 6/7] fix lambda/statment test cases that need to see the relevant segment of code to proceed. (mocking that out for now) --- tests/unit/transformer/test_node_transformer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py index ad50642a0..8f07c87d7 100644 --- a/tests/unit/transformer/test_node_transformer.py +++ b/tests/unit/transformer/test_node_transformer.py @@ -195,6 +195,7 @@ def test_visit_delete_subscript_attribute_calls_tracer_call(self, code): @pytest.mark.parametrize(*basic_tests, ids=basic_test_ids) def test_code_visited_calls_tracer_call(self, code, visitor, call_count): + self.nt._get_code_from_node = MagicMock() test_node = _get_ast_node(code) self.nt.visit(test_node) # doing this so that we can select which function in tracer gets called. From d3b6858e05568bf14e1dc1eb8c307723be435e11 Mon Sep 17 00:00:00 2001 From: Shardul Sardesai Date: Tue, 18 Jan 2022 14:21:33 -0800 Subject: [PATCH 7/7] deadcode run --- tests/unit/transformer/test_node_transformer.py | 11 +++++------ tests/unit/transformer/test_source_giver.py | 4 ---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/unit/transformer/test_node_transformer.py b/tests/unit/transformer/test_node_transformer.py index 8f07c87d7..14502940d 100644 --- a/tests/unit/transformer/test_node_transformer.py +++ b/tests/unit/transformer/test_node_transformer.py @@ -1,5 +1,6 @@ -# -# +# Setting up for mypy to ignore this file +# This file uses mocks which have dynamically defined functions. +# this does not sit well with mypy who needs to know what functions a class has # type:ignore import ast import sys @@ -64,16 +65,15 @@ class TestNodeTransformer: # calls tracer.trace_import but this list checks for tracer.call calls ("import math", "Import", 0), # calls tracer.trace_import but this list checks for tracer.call calls - # TODO - import from calls transform utils which should really be mocked out and + # TODO - importfrom calls transform_utils which should really be mocked out and # tested on their own, in true spirit of unit testing ("from math import sqrt", "ImportFrom", 0), ("a, b = (1,2)", "Assign", 2), ("lambda x: x + 10", "Lambda", 1), ] - # TODO pull out constant to handle 3.7 vs 3.9 etc # TODO handle visit_Name as its own test - # TODO import, module + # TODO module basic_test_ids = [ "slice", "slice_with_step", @@ -146,7 +146,6 @@ def test_assign_calls_tracer_assign(self): test_node = _get_ast_node("a = 10") tracer = self.nt.tracer self.nt.visit(test_node.body[0]) - # self.nt.visit_Assign.assert_called_once_with(test_node.body[0]) tracer.assign.assert_called_once_with("a", tracer.literal.return_value) @pytest.mark.parametrize( diff --git a/tests/unit/transformer/test_source_giver.py b/tests/unit/transformer/test_source_giver.py index 8e6661ef7..6e2d16500 100644 --- a/tests/unit/transformer/test_source_giver.py +++ b/tests/unit/transformer/test_source_giver.py @@ -27,7 +27,6 @@ def test_source_giver_adds_end_lineno(code, lineno): if sys.version_info >= (3, 8): pytest.skip("SourceGiver not invoked for Python 3.8+") import asttokens - from astpretty import pprint tree = ast.parse(code) # ensure that the end_lineno is not available and fetching it raises exceptions @@ -36,8 +35,6 @@ def test_source_giver_adds_end_lineno(code, lineno): # now we invoke the SourceGiver and add end_linenos in 2 steps - first we run the tree thr asttokens asttokens.ASTTokens(code, parse=False, tree=tree) - # pprint(tree) - # print(tree.body[0].last_token) # double check that the line numbers cooked up by asttokens are correct assert tree.body[0].last_token.end[0] == lineno @@ -45,4 +42,3 @@ def test_source_giver_adds_end_lineno(code, lineno): # so that the tree looks like 3.8+ tree with all the end_linenos etc SourceGiver().transform(tree) assert tree.body[0].end_lineno == lineno - # print(tree)