Merge pull request #10 from puzzmo-com/shiki

Use Shiki code samples and write a blog post saying how to do it

.github/workflows/hugo.yaml

@@ -59,7 +59,17 @@ jobs:
           hugo \
             --gc \
             --minify \
-            --baseURL "${{ steps.pages.outputs.base_url }}/"          
+            --baseURL "${{ steps.pages.outputs.base_url }}/"  
+            
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+          cache: yarn
+
+      - name: Install and run
+        run: yarn install && yarn run syntax
+
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v2
         with:

.gitignore

@@ -1,2 +1,4 @@
 .hugo_build.lock
-.DS_Store
+.DS_Store
+node_modules
+public

.vscode/settings.json

@@ -7,5 +7,6 @@
         "plaintext": false,
         "markdown": false,
         "scminput": false
-    }
+    },
+    "typescript.tsdk": "node_modules/typescript/lib"
 }

README.md

@@ -68,3 +68,5 @@ This post would appear in 3 sections: tech, api and plugins. Which have their ow
 ### This blog
 
 It uses Hugo as a static site generator, it was chosen because it is is simple to install and run locally and shouldn't break over a very long time period (the Artsy blog [I used to write on](https://artsy.github.io/blog/2019/05/03/ortas-best-of/) once or twice a month was Jekyll and required a lot of custom work to get useful features but those eventually started slowing the system down and getting ruby set up is a pain).
+
+It has an optional post-completion hook which you can test via `yarn build && yarn shikify` which uses Shiki for code samples instead of Hugo's defaults.

content/posts/2024/06/23/shiki-hugo/index.md

@@ -0,0 +1,129 @@
++++
+title = 'Using Shiki Syntax Highlighting in Hugo'
+date = 2024-06-23T14:59:00+01:00
+authors = ["orta"]
+tags = ["tech", "api", "graphql", "redwood"]
+theme = "outlook-hayesy-beta"
++++
+
+When I decided on Hugo for this blog, I knew I was gonna have to take a hit on something I felt was very important to me and my writing: fancy tools for syntax highlighting.
+
+I choose Hugo because it should be super easy for folks to contribute (no fancy Node tooling setup etc) - so I have Shiki being applied as an **optional** post build step.
+
+First up, we need to disable the current syntax highlighting for codefences by editing `hugo.toml`:
+
+```toml
+[markup]
+  [markup.highlight]
+    codeFences = false
+```
+
+That means that the hugo process would make a codefenced block look like this HTML:
+
+```html
+<pre><code class="language-toml">[markup]
+  [markup.highlight]
+    codeFences = false
+</code></pre>
+```
+
+Which we can work with! So, the goal will be to edit the built files after Hugo has done its thing to switch the syntax highlighter.So, lets add the Node infra to do this, starting with adding some dependencies:
+
+```ts
+yarn add shiki @types/node node-html-parser
+```
+
+Then create a new script file:
+
+```ts
+import { createHighlighter, bundledLanguages } from "shiki"
+import { readdirSync, readFileSync, writeFileSync } from "fs"
+import { parse } from "node-html-parser"
+
+const posts = "public/posts"
+const files = await readdirSync(posts, { recursive: true, encoding: "utf-8" })
+const indexFiles = files.filter((file) => file.endsWith("index.html") && file.split("/").length > 3)
+
+const highlighter = await createHighlighter({
+  themes: ["nord"],
+  langs: Object.keys(bundledLanguages),
+})
+
+// Find all of the files in the posts directory which are index.html
+for (const file of indexFiles) {
+  // Grab the file, and parse it into a DOM
+  const content = readFileSync(posts + "/" + file, { encoding: "utf-8" })
+  const dom = parse(content)
+
+  // This isn't a particularly smart query implementation,
+  // so lets take the simple route and just grab all of the pre tags
+  const codeBlocks = dom.querySelectorAll("pre")
+
+  for (const codeBlock of codeBlocks) {
+    // We need to look for the code inside it
+    const codeChild = codeBlock.childNodes[0]
+    if (!codeChild) continue
+
+    const codeElement = parse(codeChild.toString())
+
+    // Pull out the language from the original code block
+    let lang = "text"
+    if (codeChild.rawText.startsWith('<code class="language-')) {
+      lang = codeChild.rawText.split("language-")[1].split('"')[0]
+    }
+
+    const code = codeElement.textContent
+    const highlighted = highlighter.codeToHtml(code, {
+      lang: lang || "text",
+      theme: "nord",
+    })
+
+    const newPreElement = parse(highlighted)
+    codeBlock.replaceWith(newPreElement)
+  }
+
+  // Write the new HTML
+  const newContent = dom.toString()
+  writeFileSync(posts + "/" + file, newContent)
+}
+```
+
+_( I saved mine at `scripts/shifify.ts` and use `tsx` to run the file as TypeScript. )_
+
+Next, I changed the CI build process to also run the new script:
+
+```yml
+- name: Build with Hugo
+  env:
+    # For maximum backward compatibility with Hugo modules
+    HUGO_ENVIRONMENT: production
+    HUGO_ENV: production
+  run: |
+    hugo \
+    --gc \
+    --minify \
+    --baseURL "${{ steps.pages.outputs.base_url }}/"  
+
+- name: Setup Node
+  uses: actions/setup-node@v3
+  with:
+    node-version: 20.x
+    cache: yarn
+
+- name: Install and run
+  run: yarn install && yarn tsx scripts/shikify.ts
+
+- name: Upload artifact
+  uses: actions/upload-pages-artifact@v2
+  with:
+    path: ./public
+
+```
+
+And... That's kinda it! So, TLDR:
+
+- Make the default highlighter not do codefences
+- Add a script to parse the output
+- Change CI to run it
+
+Good luck

hugo.toml

@@ -24,7 +24,6 @@ theme = 'paper'
 
 staticDir = ['static']
 
-
 [markup]
   [markup.highlight]
-    style = 'arduino'
+    codeFences = false

package.json

@@ -1,10 +1,21 @@
 {
     "packageManager": "[email protected]",
+    "type": "module",
     "scripts": {
-        "dev": "hugo server -D"
+        "dev": "hugo server -D",
+        "build": "hugo",
+        "syntax": "tsx scripts/shikify.ts"
     },
     "prettier": {
         "semi": false,
         "printWidth": 140
+    },
+    "dependencies": {
+        "@shikijs/twoslash": "^1.9.0",
+        "@types/node": "^20.14.8",
+        "node-html-parser": "^6.1.13",
+        "shiki": "^1.9.0",
+        "tsx": "^4.15.7",
+        "typescript": "^5.5.2"
     }
 }

scripts/shikify.ts

@@ -0,0 +1,55 @@
+// yarn build && yarn syntax
+
+import { createHighlighter, bundledLanguages } from "shiki"
+import { transformerTwoslash } from "@shikijs/twoslash"
+import { readdirSync, readFileSync, writeFileSync } from "fs"
+import { parse } from "node-html-parser"
+
+const posts = "public/posts"
+const files = await readdirSync(posts, { recursive: true, encoding: "utf-8" })
+const indexFiles = files.filter((file) => file.endsWith("index.html") && file.split("/").length > 3)
+
+const highlighter = await createHighlighter({
+  themes: ["solarized-light"],
+  langs: Object.keys(bundledLanguages),
+})
+
+// Find all of the files in the posts directory which are index.html
+
+for (const file of indexFiles) {
+  // Grab the file, and parse it into a DOM
+  const content = readFileSync(posts + "/" + file, { encoding: "utf-8" })
+  const dom = parse(content)
+
+  // This isn't a particularly smart query implementation,
+  // so lets take the simple route and just grab all of the pre tags
+  const codeBlocks = dom.querySelectorAll("pre")
+
+  for (const codeBlock of codeBlocks) {
+    // We need to look for the code inside it
+    const codeChild = codeBlock.childNodes[0]
+    if (!codeChild) continue
+
+    const codeElement = parse(codeChild.toString())
+
+    // Pull out the language from the original code block 
+    let lang = "text"
+    if (codeChild.rawText.startsWith('<code class="language-')) {
+      lang = codeChild.rawText.split("language-")[1].split('"')[0]
+    }
+
+    const code = codeElement.textContent
+    const highlighted = highlighter.codeToHtml(code, {
+      lang: lang || "text",
+      theme: "solarized-light",
+      transformers: [transformerTwoslash({ explicitTrigger: true })],
+    })
+
+    const newPreElement = parse(highlighted)
+    codeBlock.replaceWith(newPreElement)
+  }
+
+  // Write the new HTML
+  const newContent = dom.toString()
+  writeFileSync(posts + "/" + file, newContent)
+}