Work around node snapshot limitation for better REPL errors

build.boot

@@ -159,7 +159,9 @@
   (cljs :compiler-options {:optimizations :simple
                            :main 'lumo.core
                            :cache-analysis true
-                           :source-map false
+                           :source-map true
+                           :source-map-timestamp false
+                           :aot-cache true
                            :dump-core false
                            :static-fns true
                            :optimize-constants false

scripts/package.js

@@ -39,7 +39,9 @@ function deflate(fname) {
 const outputPath = `build/${/^Windows/.test(os.type()) ? 'lumo.exe' : 'lumo'}`;
 const resources = getDirContents('target').filter(
   fname =>
-    fname.endsWith('.aot.js.map') ||
+    /target[\\\/]cljs[\\\/].*(\$macros)?\.js\.map/.test(fname) ||
+    /target[\\\/]lumo[\\\/]repl(\$macros)?\.js\.map/.test(fname) ||
+    /target[\\\/]main\.js\.map/.test(fname) ||
     (!fname.endsWith('main.js') &&
       !fname.endsWith('bundle.js') &&
       !fname.endsWith('bundle.min.js') &&

src/cljs/snapshot/lumo/repl.cljs

@@ -221,6 +221,36 @@
 ;; Monkey-patch cljs.js/run-async! to instead be our more stack-efficient run-sync!
 (set! cljs/run-async! run-sync!)
 
+(defn ns->bundled-js-path
+  "Return the path to the bundled js expected for the input ns
+  symbol. Note that no 1:1 is guaranteed, for instance cljs.core maps to
+  main.js."
+  [ns-sym]
+  (cond
+    (nil? ns-sym) nil
+    (= ns-sym 'cljs.core) "main.js"
+    :else (str (cljs/ns->relpath ns-sym) ".js")))
+
+(defn- load-bundled-source-maps!
+  [ns-syms]
+  (let [load-source-maps (fn [ns-sym]
+                           (when-not (get-in @st [:source-maps ns-sym])
+                             (if-let [sm-text (-> ns-sym
+                                                  ns->bundled-js-path
+                                                  (str ".map")
+                                                  js/$$LUMO_GLOBALS.load)]
+                               ;; Detect if we have source maps in need of decoding
+                               ;; or if they are AOT decoded.
+                               (if (or (string/starts-with? sm-text "{\"version\"")
+                                       (string/starts-with? sm-text "{\n\"version\""))
+                                 (cljs/load-source-map! st ns-sym sm-text)
+                                 (swap! st assoc-in [:source-maps ns-sym] (common/transit-json->cljs sm-text)))
+                               (swap! st assoc-in [:source-maps ns-sym] {}))))]
+    (run! load-source-maps ns-syms)))
+
+(defn- load-core-macros-source-maps! []
+  (load-bundled-source-maps! '[cljs.core$macros]))
+
 (defn- load-bundled [name path file-path source cb]
   (when-let [cache-json (or (js/$$LUMO_GLOBALS.load (str file-path ".cache.json"))
                             (js/$$LUMO_GLOBALS.load (str path ".cache.json")))]
@@ -449,49 +479,63 @@
   {:pre [(symbol? ns)]}
   (let [ns-str        (str ns)
         munged-ns-str (string/escape ns-str {\- \_ \. \$})]
-    (into {} (for [sym (ns-syms ns)]
-               [(str munged-ns-str "$" (munge sym)) (symbol ns-str (str sym))]))))
+    (into {munged-ns-str ns}      ;; adds the namespace itself
+          (for [sym (ns-syms ns)]
+            [(str munged-ns-str "$" (munge sym)) (symbol ns-str (str sym))]))))
 
-(def ^:private core-demunge-map
+(def core-demunge-map
   (delay (form-demunge-map 'cljs.core)))
 
-(defn- non-core-demunge-maps
+(defn non-core-demunge-maps
   []
   (let [non-core-nss (remove #{'cljs.core 'cljs.core$macros} (all-ns))]
     (map form-demunge-map non-core-nss)))
 
-(defn- lookup-sym
+(defn lookup-sym
   [demunge-maps munged-sym]
   (some #(% munged-sym) demunge-maps))
 
-(defn- demunge-local
+(defn demunge-local
   [demunge-maps munged-sym]
   (let [[_ fn local] (re-find #"(.*)_\$_(.*)" munged-sym)]
     (when fn
       (when-let [fn-sym (lookup-sym demunge-maps fn)]
-        (str fn-sym " " (demunge local))))))
+        {:sym fn-sym
+         :local (demunge local)}))))
 
-(defn- demunge-protocol-fn
+(defn demunge-protocol-fn
   [demunge-maps munged-sym]
   (let [[_ obj ns prot fn] (re-find #"(.*)\.(.*)\$(.*)\$(.*)\$arity\$.*" munged-sym)]
-    (when ns
+    (when-let [ns-sym (lookup-sym demunge-maps ns)]
       (when-let [prot-sym (lookup-sym demunge-maps (str ns "$" prot))]
         (when-let [fn-sym (lookup-sym demunge-maps (str ns "$" fn))]
-          (str fn-sym " [" prot-sym "]"))))))
+          {:obj (symbol obj)
+           :ns ns-sym
+           :sym fn-sym
+           :protocol prot-sym})))))
 
 (defn- gensym?
   [sym]
   (string/starts-with? (name sym) "G__"))
 
 (defn- demunge-sym
+  "Demunge a string to a symbol.
+
+  Note that the string has to contain dollar signs for functions and
+  vars, but no dollars for something like protocol."
   [munged-sym]
+  ;; these are two maps core first - from Planck
   (let [demunge-maps (cons @core-demunge-map (non-core-demunge-maps))]
-    (str (or (lookup-sym demunge-maps munged-sym)
-             (demunge-protocol-fn demunge-maps munged-sym)
-             (demunge-local demunge-maps munged-sym)
-             (if (gensym? munged-sym)
-               munged-sym
-               (demunge munged-sym))))))
+    (or (when-let [sym (lookup-sym demunge-maps munged-sym)]
+          {:ns (some-> sym namespace symbol)
+           :sym sym})
+        (demunge-protocol-fn demunge-maps munged-sym)
+        (demunge-local demunge-maps munged-sym)
+        (if (gensym? munged-sym)
+          {:sym munged-sym}
+          (when-let [sym (some-> munged-sym not-empty demunge symbol)]
+            (merge {:ns (some-> sym namespace symbol)
+                    :sym sym}))))))
 
 (def ^:private demunge-sym-memo
   (memoize demunge-sym))
@@ -512,16 +556,40 @@
              (js-file? file)))
     (str (file->ns-sym file) "/")))
 
+(defn stacktrace-function->sym
+  [s]
+  (let [protocol? #(re-find #"arity\$[0-9]+" %)
+        ;; the below includes function objects
+        obj? #(re-find #"Object|Function\." %)]
+    (cond
+      (string/blank? s) nil
+      (protocol? s) s
+      (obj? s) (->> (string/split s #"\.")
+                    (drop 1)
+                    (string/join "$"))
+      :else (string/escape s {\- \_ \. \$}))))
+
+(defn demunge-stacktrace-entry
+  [st-entry]
+  (let [{:keys [ns sym protocol local] :as ds} (demunge-sym-memo
+                                                (stacktrace-function->sym
+                                                 (:function st-entry)))
+        s (cond
+            protocol (str sym " [" protocol "]")
+            local (str sym " " local)
+            sym (str sym)
+            :else nil)]
+    (and s (qualify s (:file st-entry)))))
+
 (defn- mapped-stacktrace-str
   ([stacktrace sms]
    (mapped-stacktrace-str stacktrace sms nil))
   ([stacktrace sms opts]
    (apply str
-          (for [{:keys [function file line column]} (st/mapped-stacktrace stacktrace sms opts)
-                :let [demunged (-> (str (when function (demunge-sym-memo function)))
-                                   (qualify file))]
+          (for [{:keys [function file line column] :as st-entry} (st/mapped-stacktrace stacktrace sms opts)
+                :let [demunged (when function (demunge-stacktrace-entry st-entry))]
                 :when (not= demunged "cljs.core/-invoke [cljs.core/IFn]")]
-            (str \tab demunged " (" file (when line (str ":" line))
+            (str \tab (or demunged "NO_SOURCE_FILE") " (" file (when line (str ":" line))
                  (when column (str ":" column)) ")" \newline)))))
 
 (defn- ^:boolean could-not-eval? [msg]
@@ -577,6 +645,33 @@
   [s]
   (str s (when-not (= \newline (last s)) \newline)))
 
+(def lumo-embedded-string? #{"<embedded>" "evalmachine.<anonymous>"})
+
+(defn patch-missing-function-stacktrace
+  "Patch a stacktrace entry that does not have :function but has :file."
+  [{:keys [function file] :as st-entry}]
+  (if (and (not function) file)
+    (merge st-entry
+           {:function file
+            :file "<embedded>"})
+    st-entry))
+
+(defn patch-embedded-stacktrace
+  [{:keys [function file] :as st-entry}]
+  (if (and function (lumo-embedded-string? file))
+    (let [{:keys [ns]} (demunge-sym-memo (stacktrace-function->sym function))
+          bundled-js-path (ns->bundled-js-path ns)]
+      (if (and bundled-js-path (js/$$LUMO_GLOBALS.isBundled (str bundled-js-path ".map")))
+        (assoc st-entry :file (str (cljs/ns->relpath ns) ".js"))
+        st-entry))
+    st-entry))
+
+(def ^{:doc "Patch the stacktrace coming from the JS enging"}
+  stacktrace-patch-xf
+  (comp (map patch-missing-function-stacktrace)
+        (map patch-embedded-stacktrace)
+        (filter (complement (comp lumo-embedded-string? :file)))))
+
 (defn- print-error
   ([error stacktrace?]
    (print-error error stacktrace? nil))
@@ -602,11 +697,22 @@
              (print-value data {::as-code? false}))
          (when stacktrace?
            (let [opts {:output-dir "file://(/goog/..)?"}
-                 canonical-stacktrace (->> (st/parse-stacktrace
-                                            {}
-                                            (.-stack error)
-                                            {:ua-product :nodejs}
-                                            opts))]
+                 canonical-stacktrace (into []
+                                            stacktrace-patch-xf
+                                            (st/parse-stacktrace
+                                             {}
+                                             (.-stack error)
+                                             {:ua-product :nodejs}
+                                             opts))
+                 namespaces (->> canonical-stacktrace
+                                 (remove (comp lumo-embedded-string? :file))
+                                 (map (comp file->ns-sym :file))
+                                 distinct)
+                 core-ns? (some #(= 'cljs.core %) namespaces)]
+             (when core-ns?
+               (load-core-macros-source-maps!))
+             (when (seq namespaces)
+               (load-bundled-source-maps! namespaces))
              (print
               (mapped-stacktrace-str
                canonical-stacktrace

src/js/cljs.js

@@ -196,6 +196,7 @@ function newDevelopmentContext(): vm$Context {
       addSourcePaths: lumo.addSourcePaths,
       getSourcePaths: lumo.getSourcePaths,
       removeSourcePath: lumo.removeSourcePath,
+      isBundled: lumo.isBundled,
     },
     global: undefined,
   };
@@ -230,6 +231,7 @@ function newClojureScriptContext(): vm$Context {
     addSourcePaths: lumo.addSourcePaths,
     getSourcePaths: lumo.getSourcePaths,
     removeSourcePath: lumo.removeSourcePath,
+    isBundled: lumo.isBundled,
   };
 
   return global;

src/js/lumo.js

@@ -143,7 +143,7 @@ type ResourceType =
       modified: number,
     |};
 
-function isBundled(filename: string): boolean {
+export function isBundled(filename: string): boolean {
   if (__DEV__) {
     return fs.existsSync(`./target/${filename}`);
   }

src/test/lumo/lumo/repl_tests.cljs

@@ -208,3 +208,68 @@ Special Form
     (is (= 3 (core/eval (list + 1 2))))
     (is (= 17 (core/eval '(let [a 10] (+ 3 4 a)))))
     (is (= 5 ((eval (eval '+)) 2 3)))))
+
+(deftest ns->bundled-js-path-tests
+  (is (nil? (lumo/ns->bundled-js-path nil)))
+  (is (= "main.js" (lumo/ns->bundled-js-path 'cljs.core)))
+  (is (= "lazy_map/core.js" (lumo/ns->bundled-js-path 'lazy_map.core))))
+
+(deftest stacktrace-function->sym-tests
+  (is (= "cljs$core$seq" (lumo/stacktrace-function->sym "cljs.core.seq")))
+  (is (= "cljs$core$seq" (lumo/stacktrace-function->sym "Object.cljs.core.seq")) "the Object prefix should be trimmed away")
+  (is (= "cljs$core$ffirst" (lumo/stacktrace-function->sym "cljs.core.ffirst")))
+  (is (= "cljs.core.LazySeq.cljs$core$ISeqable$_seq$arity$1"
+         (lumo/stacktrace-function->sym "cljs.core.LazySeq.cljs$core$ISeqable$_seq$arity$1")) "should not convert strings representing protocols")
+  (is (nil? (lumo/stacktrace-function->sym nil))))
+
+(deftest demunge-tests
+  (is (nil? (lumo/demunge-sym "") "demunging an empty string should return nil"))
+
+  (testing "demunging a symbol"
+    (is (= '{:ns cljs.core :sym cljs.core/seq} (lumo/demunge-sym "cljs$core$seq")))
+    (is (= '{:ns cljs.core :sym cljs.core/ffirst} (lumo/demunge-sym "cljs$core$ffirst"))))
+
+  (testing "demunging a protocol"
+    (is (= '{:ns cljs.core :obj cljs.core.LazySeq :sym cljs.core/-seq :protocol cljs.core/ISeqable}
+           (lumo/demunge-sym "cljs.core.LazySeq.cljs$core$ISeqable$_seq$arity$1")))
+    (is (= '{:ns cljs.core :obj Function.cljs.core.apply :sym cljs.core/-invoke :protocol cljs.core/IFn}
+           (lumo/demunge-sym "Function.cljs.core.apply.cljs$core$IFn$_invoke$arity$2")))
+    (is (= '{:ns cljs.core :obj Function.cljs.core.trampoline :sym cljs.core/-invoke :protocol cljs.core/IFn}
+           (lumo/demunge-sym "Function.cljs.core.trampoline.cljs$core$IFn$_invoke$arity$variadic")))))
+
+(when test-util/lumo-env?
+  (deftest patch-embedded-stacktrace-tests
+    (let [st-entry {:file "vm.js" :function "Script.runInThisContext" :line 65 :column 33}]
+      (is (= st-entry (lumo/patch-embedded-stacktrace st-entry)) "it should not touch an entry that is not either <embedded> or evalmachine.<anonymous>"))
+
+    (let [st-entry {:file "<embedded>" :function "Object.cljs.core.first" :line 503 :column 213}
+          expected {:file "cljs/core.js" :function "Object.cljs.core.first" :line 503 :column 213}]
+      (is (= expected (lumo/patch-embedded-stacktrace st-entry))))
+
+    (let [st-entry {:file "evalmachine.<anonymous>" :function "Function.cljs.core.apply.cljs$core$IFn$_invoke$arity$2" :line 333 :column 444}
+          expected {:file "cljs/core.js" :function "Function.cljs.core.apply.cljs$core$IFn$_invoke$arity$2" :line 333 :column 444}]
+      (is (= expected (lumo/patch-embedded-stacktrace st-entry))))
+
+    (let [st-entry {:file "evalmachine.<anonymous>" :function "cljs.core.ffirst" :line 563 :column 448}
+          expected {:file "cljs/core.js" :function "cljs.core.ffirst" :line 563 :column 448}]
+      (is (= expected (lumo/patch-embedded-stacktrace st-entry))))))
+
+(when test-util/lumo-env?
+  (deftest ffirst-stacktrace-smoke-tests
+    (let [st-entries [{:file "evalmachine.<anonymous>" :function "Object.cljs.core.seq" :line 502 :column 410} {:file "evalmachine.<anonymous>" :function "Object.cljs.core.first" :line 503 :column 213} {:file "evalmachine.<anonymous>" :function "cljs.core.ffirst" :line 563 :column 448} {:file "evalmachine.<anonymous>" :function nil :line 1 :column 18} {:file "vm.js" :function "ContextifyScript.Script.runInContext" :line 59 :column 29} {:file "vm.js" :function "Object.runInContext" :line 120 :column 6} {:file "Object.lumoEval" :function nil :line nil :column nil} {:file "Object.lumo.repl.caching_node_eval" :function nil :line nil :column nil} {:file "evalmachine.<anonymous>" :function nil :line 5824 :column 287} {:file "evalmachine.<anonymous>" :function "z" :line 5825 :column 306}]]
+      (= (into [] lumo/stacktrace-patch-xf st-entries)
+         [{:file "cljs/core.js" :function "Object.cljs.core.seq" :line 502 :column 410}
+          {:file "cljs/core.js" :function "Object.cljs.core.first" :line 503 :column 213}
+          {:file "cljs/core.js" :function "cljs.core.ffirst" :line 563 :column 448}
+          {:file "vm.js" :function "ContextifyScript.Script.runInContext" :line 59 :column 29}
+          {:file "vm.js" :function "Object.runInContext" :line 120 :column 6}
+          {:file "Object.lumoEval" :function nil :line nil :column nil}
+          {:file "Object.lumo.repl.caching_node_eval" :function nil :line nil :column nil}]))))
+
+(when test-util/lumo-env?
+  (deftest patch-missing-function-stacktrace-tests
+    (let [st-entry {:file "<embedded>" :function "Object.cljs.core.first" :line 503 :column 213}]
+      (is (= st-entry (lumo/patch-missing-function-stacktrace st-entry)) "it should not touch an entry that has a valid :function key"))
+
+    (let [st-entry {:file "Object.lumo.repl.caching_node_eval" :function nil :line nil :column nil}]
+      (is (= st-entry (lumo/patch-missing-function-stacktrace st-entry)) "it should not touch an entry that does not have bundled source maps"))))