Dev state management (#265)

Changes
- State management: Manageable subtrait of DebugView which allows
sending, receiving and setting Refs using the codecs
- Breakpoints: Better written as a LazyParsley / StrictParsley
- DebugTree: New field for highlighting specific nodes

Author: j-mie6

parsley-debug/shared/src/main/scala/parsley/debug/Codec.scala

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors>
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package parsley.debug
+
+import scala.util.Try
+import parsley.state.Ref
+
+abstract class RefCodec {
+    type A
+
+    val ref: Ref[A]
+    val codec: Codec[A]
+}
+
+object RefCodec {
+    
+    // Ref address and String value passed to RemoteView
+    type CodedRef = (Int, String)
+
+}
+
+
+/** An encoder-decoder for `A` and a string representation
+  * 
+  * @tparam A the type being encoded and decoded
+  */
+trait Codec[A] {
+    /** Encode `a` into a string
+      * 
+      * @param a the value to encode
+      * @return the string encoding of `a`
+      */
+    def encode(a: A): String
+
+    /** Attempt to decode a string into type `A`
+      *
+      * @param s the string to decode
+      * @return a `Option[A]` containing the decoded value if successful
+      */
+    def decode(s: String): Option[A]
+}
+
+object Codec {
+    implicit val booleanCodec: Codec[Boolean] = new Codec[Boolean] {
+        def encode(b: Boolean): String = b.toString
+
+        def decode(s: String): Option[Boolean] = Try(s.toBoolean).toOption
+    }   
+
+    implicit val byteCodec: Codec[Byte] = new Codec[Byte] {
+        def encode(x: Byte): String = x.toString
+
+        def decode(s: String): Option[Byte] = Try(s.toByte).toOption
+    }
+
+    implicit val shortCodec: Codec[Short] = new Codec[Short] {
+        def encode(x: Short): String = x.toString
+
+        def decode(s: String): Option[Short] = Try(s.toShort).toOption
+    }
+
+    implicit val intCodec: Codec[Int] = new Codec[Int] {
+        def encode(x: Int): String = x.toString
+
+        def decode(s: String): Option[Int] = Try(s.toInt).toOption
+    }
+
+    implicit val longCodec: Codec[Long] = new Codec[Long] {
+        def encode(x: Long): String = x.toString
+
+        def decode(s: String): Option[Long] = Try(s.toLong).toOption
+    }
+
+    implicit val floatCodec: Codec[Float] = new Codec[Float] {
+        def encode(x: Float): String = x.toString
+
+        def decode(s: String): Option[Float] = Try(s.toFloat).toOption
+    }
+
+    implicit val doubleCodec: Codec[Double] = new Codec[Double] {
+        def encode(x: Double): String = x.toString
+
+        def decode(s: String): Option[Double] = Try(s.toDouble).toOption
+    }
+
+    implicit val charCodec: Codec[Char] = new Codec[Char] {
+        def encode(c: Char): String = c.toString
+
+        def decode(s: String): Option[Char] = if (s.length() == 1) Some(s.charAt(0)) else None
+    }
+
+    implicit val stringCodec: Codec[String]  = new Codec[String] {
+        def encode(s: String): String = s
+
+        def decode(s: String): Option[String] = Some(s)
+    }
+}

parsley-debug/shared/src/main/scala/parsley/debug/DebugTree.scala

@@ -50,6 +50,9 @@ private [debug] abstract class DebugTree extends Iterable[DebugTree] {
     /** Is this parser iterative? */
     def isIterative: Boolean
 
+    /** Has this node been generated since the last time it hit a breakpoint */
+    def isNewlyGenerated: Boolean
+
 // $COVERAGE-OFF$
     /** Provides a depth-first view of the tree as an iterator. */
     override def iterator: Iterator[DebugTree] = Iterator(this) ++ nodeChildren.iterator.flatMap(_.iterator)

parsley-debug/shared/src/main/scala/parsley/debug/DebugView.scala

@@ -5,6 +5,7 @@
  */
 package parsley.debug
 
+import parsley.debug.RefCodec.CodedRef
 import parsley.debug.internal.XIllegalStateException
 import org.typelevel.scalaccompat.annotation.unused
 
@@ -54,7 +55,17 @@ object DebugView {
         *           n == 0 to just step through this breakpoint.
         *           n >= 1 to step through the next n breakpoints.
         */
-      private [debug] def renderWait(input: => String, tree: => DebugTree): Int
+        private [debug] def renderWait(input: => String, tree: => DebugTree): Int
+    }
+
+    /** Signifies that the debug view inheriting from this can wait on a certain render call, updating the state of Refs.
+      * 
+      * This can be extended to make remote state management, breakpoint stepping possible.
+      *
+      * @see [[DebugView]]
+      */
+    trait Manageable extends DebugView with Pauseable {
+        private [debug] def renderManage(input: => String, tree: => DebugTree, refs: CodedRef*): (Int, Seq[CodedRef])
     }
 
     /** Signifies that the debug view inheriting from this can only be run once.

parsley-debug/shared/src/main/scala/parsley/debug/combinator.scala

@@ -266,13 +266,15 @@ object combinator {
       *
       * @param parser The parser to debug.
       * @param break Indicate whether to break on entry or exit to the parser.
+      * @param refs Stateful references to be updated.
       * @tparam A Output type of original parser.
       * @return A modified parser which will pause parsing and ask the view to render the produced
       *         debug tree after a call to [[Parsley.parse]] is made.
       */
-    def break[A](parser: Parsley[A], break: Breakpoint): Parsley[A] = new Parsley(new RemoteBreak(parser.internal, break))
 
-    /** Dot accessor versions of the combinators.  */
+    def break[A](parser: Parsley[A], break: Breakpoint, refs: RefCodec*): Parsley[A] = new Parsley(new RemoteBreak(parser.internal, break, refs*))
+
+    /** Dot accessor versions of the combinators. */
     implicit class DebuggerOps[A](par: Parsley[A]) {
         //def attachDebugger(toStringRules: PartialFunction[Any, Boolean]): DebuggedPair[A] = combinator.attachDebugger(par, toStringRules)
         //def attachReusable(toStringRules: PartialFunction[Any, Boolean]): () => DebuggedPair[A] = combinator.attachReusable(par, toStringRules)
@@ -286,7 +288,7 @@ object combinator {
         def attachReusable(view: =>DebugView.Reusable): () => Parsley[A] = combinator.attachReusable(par, view, DefaultStringRules)
         //def attach(implicit view: DebugFrontend): Parsley[A] = combinator.attach(par, defaultRules)
         def named(name: String): Parsley[A] = combinator.named(par, name)
-        def break(break: Breakpoint): Parsley[A] = combinator.break(par, break)
+        def break(break: Breakpoint, refs: RefCodec*): Parsley[A] = combinator.break(par, break, refs*)
     }
     // $COVERAGE-ON$
 

parsley-debug/shared/src/main/scala/parsley/debug/internal/DebugContext.scala

@@ -10,13 +10,14 @@ import scala.collection.mutable
 import parsley.XAssert
 import parsley.debug.ParseAttempt
 import parsley.internal.deepembedding.frontend.LazyParsley
-import parsley.internal.deepembedding.frontend.debug.RemoteBreak
-import parsley.debug.*
+import parsley.debug.DebugView
+import parsley.debug.RefCodec.CodedRef
 
 // Class used to hold details about a parser being debugged.
 // This is normally held as a value inside an implicit variable.
 // Anything caught by the toStringRules will have a parse result of that type toString-ed for memory
 // efficiency.
+
 private [parsley] class DebugContext(private val toStringRules: PartialFunction[Any, Boolean], private val view: DebugView) {
     // Create a new dummy root of the tree that will act as filler for the rest of the tree to build
     // off of (as there is no "nil" representation for the tree... other than null, which should be
@@ -45,7 +46,7 @@ private [parsley] class DebugContext(private val toStringRules: PartialFunction[
         XAssert.assert(!(ch.size > 1), s"The root tree has somehow gained multiple children. (${ch.size})")
 
         // This should never fail.
-        ch.head
+        ch.head.withoutBreakpoints().withoutNewlyGeneratedFlags()
     }
 
     // Add an attempt of parsing at the current stack point.
@@ -83,48 +84,73 @@ private [parsley] class DebugContext(private val toStringRules: PartialFunction[
       */
     private var breakpointSkips: Int = 0
 
-    /** Handle a breakpoint.
+    /** True if associated DebugView extends the Manageable trait */
+    def manageableView: Boolean = view match {
+        case _: DebugView.Manageable => true 
+        case _ => false
+    } 
+
+    private var firstBreakpoint: Boolean = true
+
+    /** Trigger a breakpoint.
       *
-      * @param tree         The debug tree that has been created thus far.
       * @param fullInput    The full parser input.
-      * @param view         The DebugView instance. This must extend DebugView.Pauseable to work.
+      * @param refs         References managed by this breakpoint.
       */
-    private def handleBreak(tree: TransientDebugTree, fullInput: String): Unit = view match {
-        case view: DebugView.Pauseable => {
-            if (breakpointSkips > 0) {
-                breakpointSkips -= 1
-            } else {
-                breakpointSkips = view.renderWait(fullInput, tree)
+    def triggerBreak(fullInput: String, isAfter: Boolean, codedRefs: Option[Seq[CodedRef]]): Option[Seq[CodedRef]] = {
+        if (firstBreakpoint) {
+            builderStack.head.resetNewlyGeneratedFlags()
+            firstBreakpoint = false
+        }
+
+        val debugTree: TransientDebugTree = (if (isAfter) builderStack.head else {
+             // FIXME: Instead of `init`, this should take(n) where n is the number of children the next node would have.
+             // This can only be known after the next parser runs, and needs to be passed in after saving/restoring the builderStack
+            builderStack.head.copy(children = builderStack.tail.init)
+        }).copy(parse = Some(new ParseAttempt("", 0, 0, (0,0), (0,0), Some(()))))
+
+
+        val newRefs: Option[Seq[CodedRef]] = view match {
+            case view: DebugView.Pauseable => {
+                if (breakpointSkips > 0) { // Skip to next breakpoint
+                    breakpointSkips -= 1
+                    None
+                } else if (breakpointSkips != -1) { // Breakpoint exit
+                    view match {
+                        case view: DebugView.Manageable => {
+                            
+                            // Wait for RemoteView to return breakpoint skips and updated state
+                            val (newSkips, newRefs): (Int, Seq[CodedRef]) = view.renderManage(fullInput, debugTree, codedRefs.get*)
+                            
+                            // Update breakpoint skips
+                            breakpointSkips = newSkips
+                            Some(newRefs)
+                        }
+
+                        // Update breakpoint using Pausable render call
+                        case _ => {
+                            breakpointSkips = view.renderWait(fullInput, debugTree)
+                            None
+                        }
+                    }
+                } else None
             }
+            
+            case _ => None
         }
-        case _ => 
+
+        debugTree.resetNewlyGeneratedFlags()
+        newRefs
     }
 
     // Push a new parser onto the parser callstack.
     def push(fullInput: String, parser: LazyParsley[_], isIterative: Boolean, userAssignedName: Option[String]): Unit = {
-        // Send the debug tree here if EntryBreak
-        parser match {
-            case break: RemoteBreak[_] => break.break match {
-                case EntryBreak | FullBreak => handleBreak(builderStack.head, fullInput)
-                case _ => 
-            }
-            case _ =>
-        }
 
         val newTree = new TransientDebugTree(fullInput = fullInput)
         newTree.name = Renamer.nameOf(userAssignedName, parser)
         newTree.internal = Renamer.internalName(parser)
         newTree.iterative = isIterative
 
-        // Send the debug tree here if ExitBreak
-        parser match {
-            case break: RemoteBreak[_] => break.break match {
-                case ExitBreak | FullBreak => handleBreak(newTree, fullInput)
-                case _ => 
-            }
-            case _ =>
-        }
-
         //val uid = nextUid()
         //builderStack.head.children(s"${newTree.name}-#$uid") = newTree
         builderStack.head.children += newTree

parsley-debug/shared/src/main/scala/parsley/debug/internal/TransientDebugTree.scala

@@ -22,7 +22,7 @@ import parsley.debug.{DebugTree, ParseAttempt}
 private [parsley] class TransientDebugTree(var name: String = "", var internal: String = "", val fullInput: String,
                                            var parse: Option[ParseAttempt] = None, var cNumber: Option[Long] = None,
                                            val children: mutable.ListBuffer[TransientDebugTree] = mutable.ListBuffer.empty,
-                                           var iterative: Boolean = false) extends DebugTree {
+                                           var iterative: Boolean = false, var newlyGenerated: Boolean = true) extends DebugTree {
     // These are user-facing, and will depend heavily on what the parser looks like.
     // $COVERAGE-OFF$
     override def parserName: String = name
@@ -44,6 +44,8 @@ private [parsley] class TransientDebugTree(var name: String = "", var internal:
 
     override def isIterative: Boolean = iterative
 
+    override def isNewlyGenerated: Boolean = newlyGenerated
+
     // Factors out inputs or results for parsers with children.
     private type Augment  = (Long, (Int, Int))
     private var augmentId = 0L
@@ -76,4 +78,29 @@ private [parsley] class TransientDebugTree(var name: String = "", var internal:
             })
         }
     }
+
+    // Make a copy with possibly different fields
+    private [debug] def copy(name: String = name, internal: String = internal, fullInput: String = fullInput,
+            parse: Option[ParseAttempt] = parse, cNumber: Option[Long] = cNumber,
+            children: mutable.ListBuffer[TransientDebugTree] = children,
+            iterative: Boolean = iterative, newlyGenerated: Boolean = newlyGenerated): TransientDebugTree
+        = new TransientDebugTree(name, internal, fullInput, parse, cNumber, children, iterative, isNewlyGenerated)
+        
+    // Strips all `remoteBreak` nodes from the tree
+    private [debug] def withoutBreakpoints(): TransientDebugTree = {
+        val childrenWithoutBreak = children.map(_.withoutBreakpoints())
+        if (internalName == "remoteBreak") {
+            childrenWithoutBreak.head // Debug nodes *should* only ever have one and only one child
+        } else new TransientDebugTree(name, internal, fullInput, parse, cNumber, childrenWithoutBreak, iterative)
+    }
+
+    private [debug] def resetNewlyGeneratedFlags(): Unit = {
+        newlyGenerated = false
+        children.foreach(_.resetNewlyGeneratedFlags())
+    }
+
+    private [debug] def withoutNewlyGeneratedFlags(): TransientDebugTree = {
+        resetNewlyGeneratedFlags()
+        this
+    }
 }

parsley-debug/shared/src/main/scala/parsley/internal/deepembedding/backend/RemoteBreak.scala

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors>
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package parsley.internal.deepembedding.backend
+
+import parsley.debug.{Breakpoint, EntryBreak, ExitBreak, FullBreak, RefCodec}
+import parsley.debug.internal.DebugContext
+import parsley.internal.deepembedding.ContOps
+import parsley.internal.deepembedding.ContOps.{suspend, ContAdapter}
+import parsley.internal.machine.instructions.debug.TriggerBreakpoint
+
+private [deepembedding] final class InertBreak[A](p: StrictParsley[A], break: Breakpoint, refs: RefCodec*) extends StrictParsley[A] {
+  override protected[backend] def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: StrictParsley.InstrBuffer, state: CodeGenState): M[R,Unit] = p.codeGen(producesResults)
+
+  override private [deepembedding] def inlinable: Boolean = p.inlinable
+
+  override private [deepembedding] def pretty: String = f"inertBreak(${p.pretty})"
+
+  private [deepembedding] def activate(dbgCtx: DebugContext) = new ActiveBreak[A](p, break, dbgCtx, refs*)
+}
+
+private [deepembedding] final class ActiveBreak[A](p: StrictParsley[A], break: Breakpoint, dbgCtx: DebugContext, refs: RefCodec*) extends StrictParsley[A] {
+  override protected[backend] def codeGen[M[_, +_]: ContOps, R](producesResults: Boolean)(implicit instrs: StrictParsley.InstrBuffer, state: CodeGenState): M[R,Unit] = {
+    break match {
+      case EntryBreak | FullBreak => instrs += new TriggerBreakpoint(dbgCtx, false, refs*)
+      case _ =>
+    }
+    suspend[M, R, Unit](p.codeGen[M, R](producesResults)) |> {
+        break match {
+          case ExitBreak | FullBreak => instrs += new TriggerBreakpoint(dbgCtx, true, refs*)
+          case _ =>
+        }
+    }
+  }
+
+  override private [deepembedding] def inlinable: Boolean = p.inlinable
+
+  override private [deepembedding] def pretty: String = f"activeBreak(${p.pretty})"
+}