Fully thread-safe AST (#58)

* Added some infrastructure for tracking subroutines with labels

* Added label tracking to letFinder

* More work done, but now nested subroutines don't factor out

* Fixed the bugs, we're passing tests now, but work is being duplicated, and that needs fixing

* Fixed for 2.12.12

* Fixed explosion problem

* Cleaned up the shop a bit, next step, removing the synchronized and using laziness

* Removed redundant information, mutable subroutines are actually ok, because they are past letFinding

* Removed superfluous synchronisation in favour of lazy vals, our work here is done

* Removed superfluous import

Author: j-mie6

src/main/scala-2.12/parsley/XCompat.scala

@@ -4,15 +4,15 @@ import scala.collection.mutable
 import scala.language.higherKinds
 
 private [parsley] object XCompat {
-  def isIdentityWrap[A, B](f: A => B): Boolean = f eq $conforms[A]
+    def isIdentityWrap[A, B](f: A => B): Boolean = f eq $conforms[A]
 
-  def refl[A]: A =:= A = implicitly[A =:= A]
+    def refl[A]: A =:= A = implicitly[A =:= A]
 
-  implicit class Subtitution[A, B](ev: A =:= B) {
-    def substituteCo[F[_]](fa: F[A]): F[B] = fa.asInstanceOf[F[B]]
-  }
+    implicit class Subtitution[A, B](ev: A =:= B) {
+        def substituteCo[F[_]](fa: F[A]): F[B] = fa.asInstanceOf[F[B]]
+    }
 
-  implicit class MapValuesInPlace[K, V](m: mutable.Map[K, V]) {
-    def mapValuesInPlace(f: (K, V) => V): mutable.Map[K, V] = m.transform(f)
-  }
+    implicit class MapValuesInPlace[K, V](m: mutable.Map[K, V]) {
+        def mapValuesInPlace(f: (K, V) => V): mutable.Map[K, V] = m.transform(f)
+    }
 }

src/main/scala/parsley/expr/Fixity.scala

@@ -1,5 +1,7 @@
 package parsley.expr
 
+import scala.language.higherKinds
+
 /**
   * Denotes the fixity and associativity of an operator. Importantly, it also specifies the type of the
   * of the operations themselves. For non-monolithic structures this is described by `fixity.GOp` and

src/main/scala/parsley/internal/deepembedding/GeneralisedEmbedding.scala

@@ -7,9 +7,10 @@ import scala.language.higherKinds
 
 // Core Embedding
 private [parsley] abstract class Singleton[A](pretty: String, instr: =>instructions.Instr) extends Parsley[A] {
+    final override def findLetsAux[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit, Unit] = result(())
     final override def preprocess[Cont[_, +_]: ContOps, A_ >: A](implicit seen: Set[Parsley[_]], sub: SubMap,
                                                                  label: UnsafeOption[String]): Cont[Unit, Parsley[A_]] = result(this)
-    final override def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit, Unit] = result(())
     final override def codeGen[Cont[_, +_]: ContOps](implicit instrs: InstrBuffer, state: CodeGenState): Cont[Unit, Unit] = {
         result(instrs += instr)
     }
@@ -20,12 +21,13 @@ private [parsley] abstract class Singleton[A](pretty: String, instr: =>instructi
 
 private [deepembedding] abstract class SingletonExpect[A](pretty: String, builder: UnsafeOption[String] => SingletonExpect[A], instr: instructions.Instr)
     extends Parsley[A] {
+    final override def findLetsAux[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit, Unit] = result(())
     final override def preprocess[Cont[_, +_]: ContOps, A_ >: A](implicit seen: Set[Parsley[_]], sub: SubMap,
-                                                           label: UnsafeOption[String]): Cont[Unit, Parsley[A]] = {
+                                                                 label: UnsafeOption[String]): Cont[Unit, Parsley[A]] = {
         if (label == null) result(this)
         else result(builder(label))
     }
-    final override def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit, Unit] = result(())
     final override def codeGen[Cont[_, +_]: ContOps](implicit instrs: InstrBuffer, state: CodeGenState): Cont[Unit, Unit] = {
         result(instrs += instr)
     }
@@ -34,22 +36,17 @@ private [deepembedding] abstract class SingletonExpect[A](pretty: String, builde
     // $COVERAGE-ON$
 }
 
-private [deepembedding] abstract class Unary[A, B](_p: =>Parsley[A])(pretty: String => String, empty: String => Unary[A, B]) extends Parsley[B] {
+private [deepembedding] abstract class Unary[A, B](__p: =>Parsley[A])(pretty: String => String, empty: String => Unary[A, B]) extends Parsley[B] {
+    private lazy val _p = __p
     private [deepembedding] var p: Parsley[A] = _
     protected val childRepeats: Int = 1
     protected val numInstrs: Int
-    final override def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit,Unit] = this.synchronized {
-        processed = false
-        p = _p
-        p.findLets
-    }
+    final override def findLetsAux[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit,Unit] = _p.findLets
     override def preprocess[Cont[_, +_]: ContOps, B_ >: B](implicit seen: Set[Parsley[_]], sub: SubMap,
-                                                   label: UnsafeOption[String]): Cont[Unit, Parsley[B_]] =
-        if (label == null && processed) result(this) else for (p <- this.p.optimised) yield {
-            val self = /*if (label == null) this else*/ empty(label)
-            self.ready(p)
-        }
-    private [deepembedding] def ready(p: Parsley[A]): this.type = this.synchronized {
+                                                           label: UnsafeOption[String]): Cont[Unit, Parsley[B_]] =
+        for (p <- _p.optimised) yield empty(label).ready(p)
+    private [deepembedding] def ready(p: Parsley[A]): this.type = {
         processed = true
         this.p = p
         size = p.size + numInstrs
@@ -60,26 +57,23 @@ private [deepembedding] abstract class Unary[A, B](_p: =>Parsley[A])(pretty: Str
     // $COVERAGE-ON$
 }
 
-private [deepembedding] abstract class Binary[A, B, C](_left: =>Parsley[A], _right: =>Parsley[B])(pretty: (String, String) => String, empty: =>Binary[A, B, C])
-    extends Parsley[C] {
+private [deepembedding] abstract class Binary[A, B, C](__left: =>Parsley[A], __right: =>Parsley[B])
+                                                      (pretty: (String, String) => String, empty: =>Binary[A, B, C]) extends Parsley[C] {
+    private lazy val _left = __left
+    private lazy val _right = __right
     private [deepembedding] var left: Parsley[A] = _
     private [deepembedding] var right: Parsley[B] = _
     protected val numInstrs: Int
     protected val leftRepeats: Int = 1
     protected val rightRepeats: Int = 1
-    final override def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit,Unit] = this.synchronized {
-        processed = false
-        left = _left
-        right = _right
-        left.findLets >> right.findLets
-    }
+    final override def findLetsAux[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit,Unit] = _left.findLets >> _right.findLets
     final override def preprocess[Cont[_, +_]: ContOps, C_ >: C](implicit seen: Set[Parsley[_]], sub: SubMap,
                                                    label: UnsafeOption[String]): Cont[Unit, Parsley[C_]] =
-        if (label == null && processed) result(this) else for (left <- this.left.optimised; right <- this.right.optimised) yield {
-            val self = /*if (label == null) this else*/ empty
-            self.ready(left, right)
+        for (left <- _left.optimised; right <- _right.optimised) yield {
+            empty.ready(left, right)
         }
-    private [deepembedding] def ready(left: Parsley[A], right: Parsley[B]): this.type = this.synchronized {
+    private [deepembedding] def ready(left: Parsley[A], right: Parsley[B]): this.type = {
         processed = true
         this.left = left
         this.right = right
@@ -93,34 +87,32 @@ private [deepembedding] abstract class Binary[A, B, C](_left: =>Parsley[A], _rig
     // $COVERAGE-ON$
 }
 
-private [deepembedding] abstract class Ternary[A, B, C, D](_first: =>Parsley[A], _second: =>Parsley[B], _third: =>Parsley[C])
+private [deepembedding] abstract class Ternary[A, B, C, D](__first: =>Parsley[A], __second: =>Parsley[B], __third: =>Parsley[C])
                                                           (pretty: (String, String, String) => String, empty: =>Ternary[A, B, C, D]) extends Parsley[D] {
+    private lazy val _first: Parsley[A] = __first
+    private lazy val _second: Parsley[B] = __second
+    private lazy val _third: Parsley[C] = __third
     private [deepembedding] var first: Parsley[A] = _
     private [deepembedding] var second: Parsley[B] = _
     private [deepembedding] var third: Parsley[C] = _
     protected val numInstrs: Int
+    final override def findLetsAux[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit, Unit] = {
+        _first.findLets >> _second.findLets >> _third.findLets
+    }
     final override def preprocess[Cont[_, +_]: ContOps, D_ >: D](implicit seen: Set[Parsley[_]], sub: SubMap,
                                                            label: UnsafeOption[String]): Cont[Unit, Parsley[D_]] =
-        if (label == null && processed) result(this) else
-            for (first <- this.first.optimised; second <- this.second.optimised; third <- this.third.optimised) yield {
-                val self = /*if (label == null) this else*/ empty
-                self.ready(first, second, third)
-            }
-    private [deepembedding] def ready(first: Parsley[A], second: Parsley[B], third: Parsley[C]): this.type = this.synchronized {
+        for (first <- _first.optimised; second <- _second.optimised; third <- _third.optimised) yield {
+            empty.ready(first, second, third)
+        }
+    private [deepembedding] def ready(first: Parsley[A], second: Parsley[B], third: Parsley[C]): this.type = {
         processed = true
         this.first = first
         this.second = second
         this.third = third
         size = first.size + second.size + third.size + numInstrs
         this
     }
-    final override def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit, Unit] = this.synchronized {
-        processed = false
-        first = _first
-        second = _second
-        third = _third
-        first.findLets >> second.findLets >> third.findLets
-    }
     // $COVERAGE-OFF$
     final override def prettyASTAux[Cont[_, +_]: ContOps]: Cont[String, String] =
         for (f <- first.prettyASTAux; s <- second.prettyASTAux; t <- third.prettyASTAux) yield pretty(f, s, t)

src/main/scala/parsley/internal/deepembedding/Parsley.scala

@@ -39,30 +39,33 @@ private [parsley] abstract class Parsley[+A] private [deepembedding]
     }
 
     // Internals
-    final private [deepembedding] def findLets[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit, Unit] = {
-        state.addPred(this)
+    final private [deepembedding] def findLets[Cont[_, +_]: ContOps]
+        (implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit, Unit] = {
+        state.addPred(this, label)
         if (seen(this)) result(state.addRec(this))
-        else if (state.notProcessedBefore(this)) {
+        else if (state.notProcessedBefore(this, label)) {
             this match {
                 case self: UsesRegister => state.addReg(self.reg)
                 case _ =>
             }
-            findLetsAux(implicitly[ContOps[Cont]], seen + this, state)
+            findLetsAux(implicitly[ContOps[Cont]], seen + this, state, label)
         }
         else result(())
     }
     final private def fix(implicit seen: Set[Parsley[_]], sub: SubMap, label: UnsafeOption[String]): Parsley[A] = {
         // We use the seen set here to prevent cascading sub-routines
         val wasSeen = seen(this)
-        val self = sub(this)
+        val self = sub(label, this)
         if (wasSeen && (self eq this)) new Rec(this, label)
         else if (wasSeen) this
         else self
     }
     final private [deepembedding] def optimised[Cont[_, +_]: ContOps, A_ >: A](implicit seen: Set[Parsley[_]],
                                                                                         sub: SubMap,
-                                                                                        label: UnsafeOption[String] = null): Cont[Unit, Parsley[A_]] = {
-        for (p <- this.fix.preprocess(implicitly[ContOps[Cont]], seen + this, sub, label)) yield p.optimise
+                                                                                        label: UnsafeOption[String]): Cont[Unit, Parsley[A_]] = {
+        val fixed = this.fix
+        if (fixed.processed) result(fixed.optimise)
+        else for (p <- fixed.preprocess(implicitly[ContOps[Cont]], seen + this, sub, label)) yield p.optimise
     }
     final private [deepembedding] var safe = true
     final private var cps = false
@@ -87,20 +90,21 @@ private [parsley] abstract class Parsley[+A] private [deepembedding]
 
     final private def pipeline[Cont[_, +_]: ContOps](implicit instrs: InstrBuffer, state: CodeGenState): Unit = {
         perform {
-            implicit val letFinderState: LetFinderState = new LetFinderState
             implicit val seenSet: Set[Parsley[_]] = Set.empty
+            implicit val label: UnsafeOption[String] = null
+            implicit val letFinderState: LetFinderState = new LetFinderState
+            implicit lazy val subMap: SubMap = new SubMap(letFinderState.lets)
             findLets >> {
-                implicit val subMap: SubMap = new SubMap(letFinderState.lets)
                 optimised.flatMap(p => generateCalleeSave(p.codeGen, allocateRegisters(letFinderState.usedRegs)))
             }
         }
         if (state.subsExist) {
             val end = state.freshLabel()
             instrs += new instructions.Jump(end)
             while (state.more) {
-                val p = state.nextSub()
-                instrs += new instructions.Label(state.getSubLabel(p))
-                perform(p.codeGen)
+                val sub = state.nextSub()
+                instrs += new instructions.Label(state.getSubLabel(sub))
+                perform(sub.p.codeGen)
                 instrs += instructions.Return
             }
             instrs += new instructions.Label(end)
@@ -149,7 +153,7 @@ private [parsley] abstract class Parsley[+A] private [deepembedding]
                                                             sub: SubMap,
                                                             label: UnsafeOption[String]): Cont[Unit, Parsley[A_]]
     // Let-finder recursion
-    protected def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState): Cont[Unit, Unit]
+    protected def findLetsAux[Cont[_, +_]: ContOps](implicit seen: Set[Parsley[_]], state: LetFinderState, label: UnsafeOption[String]): Cont[Unit, Unit]
     // Optimisation - Bottom-up
     protected def optimise: Parsley[A] = this
     // Peephole optimisation and code generation - Top-down
@@ -194,51 +198,49 @@ private [deepembedding] trait UsesRegister {
 // Internals
 private [parsley] class CodeGenState {
     private var current = 0
-    private val queue = mutable.ListBuffer.empty[Parsley[_]]
-    private val map = mutable.Map.empty[Parsley[_], Int]
+    private val queue = mutable.ListBuffer.empty[Subroutine[_]]
+    private val map = mutable.Map.empty[Subroutine[_], Int]
     def freshLabel(): Int = {
         val next = current
         current += 1
         next
     }
     def nlabels: Int = current
 
-    def getSubLabel(p: Parsley[_]): Int = map.getOrElseUpdate(p,
-    {
-        p +=: queue
+    def getSubLabel(sub: Subroutine[_]): Int = map.getOrElseUpdate(sub, {
+        sub +=: queue
         freshLabel()
     })
 
-    def nextSub(): Parsley[_] = queue.remove(0)
+    def nextSub(): Subroutine[_] = queue.remove(0)
     def more: Boolean = queue.nonEmpty
     def subsExist: Boolean = map.nonEmpty
 }
 
 private [parsley] class LetFinderState {
     private val _recs = mutable.Set.empty[Parsley[_]]
-    private val _preds = mutable.Map.empty[Parsley[_], Int]
+    private val _preds = mutable.Map.empty[(UnsafeOption[String], Parsley[_]), Int]
     private val _usedRegs = mutable.Set.empty[Reg[_]]
 
-    def addPred(p: Parsley[_]): Unit = _preds += p -> (_preds.getOrElseUpdate(p, 0) + 1)
+    def addPred(p: Parsley[_], label: UnsafeOption[String]): Unit = _preds += (label, p) -> (_preds.getOrElseUpdate((label, p), 0) + 1)
     def addRec(p: Parsley[_]): Unit = _recs += p
     def addReg(reg: Reg[_]): Unit = _usedRegs += reg
-    def notProcessedBefore(p: Parsley[_]): Boolean = _preds(p) == 1
-
-    def lets: Map[Parsley[_], Parsley[_]] = {
-        (for ((k, v) <- _preds;
-            if v >= 2 && !_recs.contains(k))
-        yield k -> {
-            val sub = Subroutine(k, null)
-            sub.processed = false
-            sub
-        }).toMap
+    def notProcessedBefore(p: Parsley[_], label: UnsafeOption[String]): Boolean = _preds((label, p)) == 1
+
+    def lets: Iterable[(UnsafeOption[String], Parsley[_])] = _preds.toSeq.collect {
+        case (k@(label, p), refs) if refs >= 2 && !_recs(p) => k
     }
+
     def recs: Set[Parsley[_]] = _recs.toSet
     def usedRegs: Set[Reg[_]] = _usedRegs.toSet
 }
 
-private [parsley] class SubMap(val subMap: Map[Parsley[_], Parsley[_]]) extends AnyVal {
-    def apply[A](p: Parsley[A]): Parsley[A] = subMap.getOrElse(p, p).asInstanceOf[Parsley[A]]
+private [parsley] class SubMap(subs: Iterable[(UnsafeOption[String], Parsley[_])]) {
+    private val subMap: Map[(UnsafeOption[String], Parsley[_]), Subroutine[_]] = subs.map {
+        case k@(label, p) => k -> new Subroutine(p, label)
+    }.toMap
+
+    def apply[A](label: UnsafeOption[String], p: Parsley[A]): Parsley[A] = subMap.getOrElse((label, p), p).asInstanceOf[Parsley[A]]
     // $COVERAGE-OFF$
     override def toString: String = subMap.toString
     // $COVERAGE-ON$