Improve SPdb (#652)

Now that we have astcompile, we can finally fix a long time bug of SPdb:
until now, from the spdb prompt you could only print symbols which were
already in the symtable. This means that you could access e.g. a builtin
only if it was already referenced from the body of the function you were
inspecting.

With astcompile, now we know that we are in interactive mode and we can
insert a special `ast.NameInteractive`, which then do a fully dynamic
lookup in the astframe.

While we are at it, we also improve/simplify how values are printed from
spdb.

Author: antocuni

spy/ast.py

@@ -447,6 +447,21 @@ class NameImportRef(Expr):
     sym: Symbol
 
 
+@astnode(">= astcompiled")
+class NameInteractive(Expr):
+    """
+    A Name lookup which is resolved dynamically.
+
+    This is generated only by astcompile_interactive, when the name is not found in the
+    surrounding symtable.  It's mostly meant to be used by SPdb.
+
+    See e.g. test_astcompile::test_NameInteractive and test_spdb::test_NameInteractive
+    """
+
+    precedence = 100
+    id: str
+
+
 @astnode(">= astcompiled")
 class NameError(Expr):
     """

spy/astcompile.py

@@ -570,7 +570,7 @@ def compile_expr_Name(self, name: ast.Name) -> ast.Expr:
             # the SPdb prompt, compiled against the symtable of a live frame), else
             # it means that there is a bug in symtable.
             assert self.interactive, "sym not found"
-            return ast.NameError(name.loc, name.id)
+            return ast.NameInteractive(name.loc, name.id)
 
         if sym.impref is not None:
             return ast.NameImportRef(name.loc, sym)

spy/tests/compiler/test_spdb.py

@@ -253,16 +253,13 @@ def foo(x: int, session: str) -> int:
             |     spdb_interact(session)
             |     |____________________|
         (spdb) print x
-        static type:  <spy type 'i32'>
-        dynamic type: <spy type 'i32'>
+        type: i32
         41
         (spdb) y
-        static type:  <spy type 'i32'>
-        dynamic type: <spy type 'i32'>
+        type: i32
         42
         (spdb) y * 2
-        static type:  <spy type 'i32'>
-        dynamic type: <spy type 'i32'>
+        type: i32
         84
         (spdb) continue
         """
@@ -283,8 +280,7 @@ def foo(x: int, session: str) -> None:
             |     spdb_interact(session)
             |     |____________________|
         (spdb) x
-        static type:  <spy type 'i32'>
-        dynamic type: <spy type 'i32'>
+        type: i32
         42
         (spdb) y
         *** NameError: name `y` is not defined
@@ -293,6 +289,37 @@ def foo(x: int, session: str) -> None:
         mod = self.compile(src)
         mod.foo(42, session)
 
+    def test_NameInteractive(self):
+        src = """
+        from _test import spdb_interact
+
+        X = 10
+        var Y = 20
+
+        def foo(x: int, session: str) -> None:
+            spdb_interact(session)
+        """
+        session = f"""
+        --- entering applevel debugger ---
+           [0] test::foo at {self.filename}:8
+            |     spdb_interact(session)
+            |     |____________________|
+        (spdb) X    # outer direct
+        type: i32
+        10
+        (spdb) Y    # outer cell
+        type: i32
+        20
+        (spdb) Z    # not found
+        *** NameError: name `Z` is not defined
+        (spdb) str  # builtin
+        type: type
+        <spy type 'str'>
+        (spdb) continue
+        """
+        mod = self.compile(src)
+        mod.foo(42, session)
+
     def test_ParseError(self):
         src = """
         from _test import spdb_interact
@@ -330,8 +357,7 @@ def foo() -> None:
            3         x = 1
            4  ->     raise ValueError("hello")
         (spdb) x
-        static type:  <spy type 'i32'>
-        dynamic type: <spy type 'i32'>
+        type: i32
         1
         (spdb) continue
         """

spy/tests/test_astcompile.py

@@ -3,9 +3,13 @@
 
 import pytest
 
+import spy.ast as ast
 from spy.analyze.importing import ImportAnalyzer
+from spy.astcompile import astcompile_interactive
 from spy.backend.spy import AST_FORMAT, FQN_FORMAT, SPyBackend
 from spy.fqn import FQN
+from spy.parser import Parser
+from spy.tests.test_parser import assert_node_dump
 from spy.util import print_diff
 from spy.vm.function import W_ASTFunc
 from spy.vm.vm import SPyVM
@@ -36,6 +40,20 @@ def write_src(self, src: str) -> None:
         src = textwrap.dedent(src)
         f.write(src)
 
+    def compile_interactive(self, src: str) -> ast.Expr:
+        """
+        Parse a single expression and astcompile it in interactive mode against the
+        symtable of `test::foo`. This is meant to be similar to what SPdb does when
+        evaluating interactive exprs.
+        """
+        fqn = FQN("test::foo")
+        w_foo = self.vm.globals_w[fqn]
+        assert isinstance(w_foo, W_ASTFunc)
+        parser = Parser(src, "<test>")
+        stmt = parser.parse_single_stmt()
+        assert isinstance(stmt, ast.StmtExpr)
+        return astcompile_interactive(stmt.value, w_foo.funcdef.symtable)
+
     def assert_dump(
         self,
         expected: str,
@@ -215,3 +233,24 @@ def foo(obj: dynamic, val: i32) -> None:
             _$aug_target0.x = _$aug_target0.x + val
         """
         self.assert_dump(expected)
+
+    def test_NameInteractive(self):
+        self.compile_src("""
+        X = 10
+
+        def foo() -> None:
+            Y = 20
+        """)
+        expr_X = self.compile_interactive("X")
+        expected = """
+        NameInteractive(id='X')
+        """
+        assert_node_dump(expr_X, expected)
+
+        expr_Y = self.compile_interactive("Y")
+        expected = """
+        NameLocalDirect(
+            sym=Symbol('Y', 'const', 'direct'),
+        )
+        """
+        assert_node_dump(expr_Y, expected)

spy/tests/test_parser.py

@@ -11,6 +11,15 @@
 from spy.vm.b import B
 
 
+def assert_node_dump(node: ast.Node, expected: str):
+    dumped = dump(node, use_colors=False, fields_to_ignore=("symtable",))
+    dumped = dumped.strip()
+    expected = textwrap.dedent(expected).strip()
+    if dumped != expected:
+        print_diff(expected, dumped, "expected", "got")
+        pytest.fail("assert_dump failed")
+
+
 @pytest.mark.usefixtures("init")
 class TestParser:
     @pytest.fixture
@@ -30,14 +39,9 @@ def expect_errors(self, src: str, main: str, *anns: MatchAnnotation):
             self.parse(src)
 
     def assert_dump(self, node: ast.Node, expected: str):
-        dumped = dump(node, use_colors=False, fields_to_ignore=("symtable",))
-        dumped = dumped.strip()
-        expected = textwrap.dedent(expected).strip()
         if "{tmpdir}" in expected:
             expected = expected.format(tmpdir=self.tmpdir)
-        if dumped != expected:
-            print_diff(expected, dumped, "expected", "got")
-            pytest.fail("assert_dump failed")
+        assert_node_dump(node, expected)
 
     def test_Module(self):
         src = """

spy/vm/astframe.py

@@ -747,6 +747,47 @@ def eval_expr_NameError(self, name: ast.NameError) -> W_MetaArg:
             name.loc,
         )
 
+    def eval_expr_NameInteractive(self, name: ast.NameInteractive) -> W_MetaArg:
+        # NameInteractive is generated only during interactive sessions, like SPdb.  See
+        # e.g. test_astcompile::test_NameInteractive and
+        # test_spdb::test_NameInteractive.
+        #
+        # We want to lookup a name which is NOT found in the symtable. We basically need
+        # to do at runtime usually is done at ScopeAnalyzer time:
+        w_val: Optional[W_Object]
+        for level in range(1, len(self.closure) + 1):
+            outervars = self.closure[-level]
+            lv = outervars.get(name.id)
+            if lv is None:
+                continue
+            if isinstance(lv.w_val, W_Cell):
+                w_val = lv.w_val.get()
+            else:
+                w_val = lv.w_val
+            assert w_val is not None
+            return W_MetaArg(self.vm, lv.color, lv.w_T, w_val, name.loc)
+
+        # name not found. Let's try the builtins
+        sym = SymTable.from_builtins().lookup_maybe(name.id)
+        if sym is not None:
+            assert sym.impref is not None
+            w_val = self.vm.lookup_ImportRef(sym.impref)
+            if w_val is None:
+                # this is likely a builtin which triggers an implicit import. Do the
+                # import and redo the lookup
+                self.vm.import_(sym.impref.modname)
+                w_val = self.vm.lookup_ImportRef(sym.impref)
+                assert w_val is not None
+            w_T = self.vm.dynamic_type(w_val)
+            return W_MetaArg(self.vm, "blue", w_T, w_val, name.loc)
+
+        raise SPyError.simple(
+            "W_NameError",
+            f"name `{name.id}` is not defined",
+            "not found in this scope",
+            name.loc,
+        )
+
     def eval_expr_AssignExprConstError(
         self, node: ast.AssignExprConstError
     ) -> W_MetaArg:

spy/vm/debugger/spdb.py

@@ -53,12 +53,18 @@ def print_wam(
     if file is None:
         file = sys.stdout
     w_T = vm.dynamic_type(wam_arg.w_val)
+    w_static_T = wam_arg.w_static_T
     wam_s = vm.repr_wam(wam_arg, loc=Loc.here())
     s = vm.unwrap_str(wam_s.w_val)
     #
     color = ColorFormatter(use_colors=use_colors)
-    print(color.set("green", "static type: "), wam_arg.w_static_T, file=file)
-    print(color.set("green", "dynamic type:"), w_T, file=file)
+    T = w_T.fqn.human_name(vm)
+    static_T = w_static_T.fqn.human_name(vm)
+    if w_T is w_static_T:
+        print(color.set("green", "type:"), T, file=file)
+    else:
+        print(color.set("green", "static type: "), static_T, file=file)
+        print(color.set("green", "dynamic type:"), T, file=file)
     print(s, file=file)