From 622f65cced3fb897b5c0c2c1f64d1621d407a2f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelina=20Ho=C5=82ub?= Date: Mon, 5 Aug 2024 10:40:56 +0200 Subject: [PATCH 1/3] feat(docs): add tutorial for custom emoji --- notebooks/parsing_extensibility.clj | 125 ++++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 6 deletions(-) diff --git a/notebooks/parsing_extensibility.clj b/notebooks/parsing_extensibility.clj index 5bb29602..a06f59eb 100644 --- a/notebooks/parsing_extensibility.clj +++ b/notebooks/parsing_extensibility.clj @@ -2,11 +2,15 @@ (ns parsing-extensibility {:nextjournal.clerk/toc :collapsed :nextjournal.clerk/no-cache true} - (:require [nextjournal.clerk :as clerk] - [nextjournal.markdown :as md] - [nextjournal.markdown.parser :as md.parser] - [edamame.core :as edamame] - [clojure.string :as str])) + (:require + [clojuer.java.io :as io] + [clojure.core.async :as async] + [clojure.string :as str] + [edamame.core :as edamame] + [nextjournal.clerk :as clerk] + [nextjournal.markdown :as md] + [nextjournal.markdown.parser :as md.parser] + [nextjournal.markdown.transform :as md.transform])) ^{:nextjournal.clerk/visibility {:code :hide :result :hide}} (def show-text @@ -31,12 +35,121 @@ (md.parser/tokenize-text-node internal-link-tokenizer {} {:text "some [[set]] of [[wiki]] link"}) - ;; In order to opt-in of the extra tokenization above, we need to configure the document context as follows: (md/parse (update md.parser/empty-doc :text-tokenizers conj internal-link-tokenizer) "some [[set]] of [[wiki]] link") ;; We provide an `internal-link-tokenizer` as well as a `hashtag-tokenizer` as part of the `nextjournal.markdown.parser` namespace. By default, these are not used during parsing and need to be opted-in for like explained above. +;; +;; ### Example #2: custom emoji +;; +;; Another, sligthly more complex example is extending the Hiccup transformer, +;; so that we can add emojis (like :smile:) to the text. +;; +;; Assuming we have multiple emoji packs installed, we need to first build a data +;; structure to hold the metadata about the emoji, like this: +'([:memes [{:name "bonk" + :path "img/emoji/memes/bonk.png"}]] + [:yellow_ball [{:name "yb-sunglasses" + :path "img/emoji/yellow_ball/yb-sunglasses.png"}]]) + +(defonce emoji-dir "resources/public/img/emotes") + +(defn scan-emoji-dir + "Scans the emoji directory and returns a map of emoji info. + The map is a vector of composite map vectors" + [] + (let [root (io/file emoji-dir) + packs (.listFiles root)] + (mapcat (fn [pack] + (when (.isDirectory pack) + (let [pack-name (.getName pack) + files (.listFiles pack) + emoji-names (mapv (fn [file] + (-> file + .getName + (str/split #"\." 2) ; trim the extension naively + first)) + files) + ;; you can remove str/replace call here if it won't cause issues with + ;; your public resources path not being accessible to the client + paths (mapv #(str/replace (.getPath %) #"resources/public" "") files)] + (assoc {} (keyword pack-name) (mapv #(assoc {} :name % :path %2) emoji-names paths))))) + packs))) + +;; cache the emoji. This is a great boost for performance, +;; but you'll need to restart your application when you add new emoji +(defonce emoji (scan-emoji-dir)) + +;; Next, we need to look up our emoji data we just grabbed. +;; Now, depending on whether you want better performance and less risk +;; of conflicts, you might want to require fully qualified emoji names, +;; at the expense of some user inconvenience. +;; This function makes that configurable +(def unqualified-emoji-names? true) + +(defn find-emoji-info + "Looks up the emoji info for an emoji token. + If unqualified names are enabled in the settings, will + sift through all packs." + [emoji-token] + (let [[pack name] (str/split emoji-token #"\." 2)] + (-> + (if unqualified-emoji-names? + (reduce (fn [acc [_ emote-pack]] + (concat acc (filter #(= emoji-token (:name %)) emote-pack))) + [] + emoji) + (filter #(= name (:name %)) ((keyword pack) emoji))) + (first)))) ;; **note**: we discard any duplicates here + +;; Finally, we can create our handler and renderer +(defn emoji-handler [match] + (let [emoji-name (second match) + emote-info (find-emoji-info emoji-name)] + (if emote-info + {:type :emoji + :tag :img + :attrs {:src (:path emote-info) + :alt (:name emote-info) + :style {:max-width "2.5rem"}}} + {:type :text + :text (str ":" emoji-name ":")}))) + +(def ^:private emoji-tokenizer + (md.parser/normalize-tokenizer + {:regex #":([a-zA-Z0-9_\-.]+):" + :handler emoji-handler})) + +(defn emoji-renderer + [ctx node] + (let [params (:attrs node) + src (:path params) + alt (:name params)] + [:img.emote params])) + +;; Finally, we can try to offset the performance hit of the emoji lookup by adding some +;; asynchrony and timeouts. It's up to you to add spinners, error messages, +;; fallback values etc. to your Hiccup template. +(defn parse-message + "Parses message's formatting (extended markdown, see docs) + and returns a rum template" + [message] + (async/go + (try + (let [parsing-chan (async/go (md.parser/parse + ;; add the emoji tokenizer to text tokenizers + (update md.parser/empty-doc :text-tokenizers conj emoji-tokenizer) + (md/tokenize message))) + timeout-chan (async/timeout 2000) + ;; add the renderer to the default renderers map + renderers (assoc md.transform/default-hiccup-renderers :emoji emoji-renderer) + [result port] (async/alts! [parsing-chan timeout-chan])] + (if (= port timeout-chan) + {:error "Parsing timed out"} + (md.transform/->hiccup renderers result))) + (catch Exception e + {:error (str "Error parsing message: " (.getMessage e))})))) ;; ## Read-based tokenization ;; From 58e7070ed04df42fc7977457ac8c9f5deada916d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelina=20Ho=C5=82ub?= Date: Mon, 5 Aug 2024 10:55:07 +0200 Subject: [PATCH 2/3] fix(docs): set image to inline flex --- notebooks/parsing_extensibility.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notebooks/parsing_extensibility.clj b/notebooks/parsing_extensibility.clj index a06f59eb..2d0ea986 100644 --- a/notebooks/parsing_extensibility.clj +++ b/notebooks/parsing_extensibility.clj @@ -121,12 +121,14 @@ {:regex #":([a-zA-Z0-9_\-.]+):" :handler emoji-handler})) +;; Assuming we're using Tailwind CSS. +;; Otherwise, add appropriate styles in `emoji-handler` (defn emoji-renderer [ctx node] (let [params (:attrs node) src (:path params) alt (:name params)] - [:img.emote params])) + [:img.inline-flex params])) ;; Finally, we can try to offset the performance hit of the emoji lookup by adding some ;; asynchrony and timeouts. It's up to you to add spinners, error messages, From f07d3057fd230bfb985bd8c1e67f6b7a48db6b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelina=20Ho=C5=82ub?= Date: Mon, 5 Aug 2024 11:52:35 +0200 Subject: [PATCH 3/3] perf(notebooks): optimize code for emoji tutorial --- notebooks/parsing_extensibility.clj | 33 +++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/notebooks/parsing_extensibility.clj b/notebooks/parsing_extensibility.clj index 2d0ea986..01a4e17e 100644 --- a/notebooks/parsing_extensibility.clj +++ b/notebooks/parsing_extensibility.clj @@ -60,11 +60,11 @@ The map is a vector of composite map vectors" [] (let [root (io/file emoji-dir) - packs (.listFiles root)] + packs (file-seq root)] (mapcat (fn [pack] (when (.isDirectory pack) (let [pack-name (.getName pack) - files (.listFiles pack) + files (file-seq pack) emoji-names (mapv (fn [file] (-> file .getName @@ -77,9 +77,8 @@ (assoc {} (keyword pack-name) (mapv #(assoc {} :name % :path %2) emoji-names paths))))) packs))) -;; cache the emoji. This is a great boost for performance, -;; but you'll need to restart your application when you add new emoji -(defonce emoji (scan-emoji-dir)) +;; avoid re-scanning the directory on every request +(def emoji (delay (scan-emoji-dir))) ;; Next, we need to look up our emoji data we just grabbed. ;; Now, depending on whether you want better performance and less risk @@ -91,17 +90,17 @@ (defn find-emoji-info "Looks up the emoji info for an emoji token. If unqualified names are enabled in the settings, will - sift through all packs." + sift through all packs. Discards duplicates." [emoji-token] - (let [[pack name] (str/split emoji-token #"\." 2)] - (-> - (if unqualified-emoji-names? - (reduce (fn [acc [_ emote-pack]] - (concat acc (filter #(= emoji-token (:name %)) emote-pack))) - [] - emoji) - (filter #(= name (:name %)) ((keyword pack) emoji))) - (first)))) ;; **note**: we discard any duplicates here + (let [[pack name] (str/split emoji-token #"\." 2) + emoji-map @emoji] + (if unqualified-emoji-names? + (->> emoji-map + vals + (apply concat) + (filter #(= emoji-token (:name %))) + first) + (get-in emoji-map [(keyword pack) name])))) ;; Finally, we can create our handler and renderer (defn emoji-handler [match] @@ -125,9 +124,7 @@ ;; Otherwise, add appropriate styles in `emoji-handler` (defn emoji-renderer [ctx node] - (let [params (:attrs node) - src (:path params) - alt (:name params)] + (let [params (:attrs node)] [:img.inline-flex params])) ;; Finally, we can try to offset the performance hit of the emoji lookup by adding some