refactor: extract util splitToken

packages/shikiji-core/src/code-to-tokens-themes.ts

@@ -97,6 +97,7 @@ export function syncThemesTokenization(...themes: ThemedToken[][][]) {
           current[n] = {
             ...token,
             content: token.content.slice(minLength),
+            offset: token.offset + minLength,
           }
         }
       }

packages/shikiji-core/src/utils.ts

@@ -1,5 +1,5 @@
 import type { Element } from 'hast'
-import type { MaybeArray } from './types'
+import type { MaybeArray, ThemedToken } from './types'
 
 export function toArray<T>(x: MaybeArray<T>): T[] {
   return Array.isArray(x) ? x : [x]
@@ -52,6 +52,42 @@ export function addClassToHast(node: Element, className: string | string[]) {
   }
 }
 
+/**
+ * Split a token into multiple tokens by given offsets.
+ *
+ * The offsets are relative to the token, and should be sorted.
+ */
+export function splitToken<
+  T extends Pick<ThemedToken, 'content' | 'offset'>,
+>(
+  token: T,
+  offsets: number[],
+): T[] {
+  let lastOffset = 0
+  const tokens: T[] = []
+
+  for (const offset of offsets) {
+    if (offset > lastOffset) {
+      tokens.push({
+        ...token,
+        content: token.content.slice(lastOffset, offset),
+        offset: token.offset + lastOffset,
+      })
+    }
+    lastOffset = offset
+  }
+
+  if (lastOffset < token.content.length) {
+    tokens.push({
+      ...token,
+      content: token.content.slice(lastOffset),
+      offset: token.offset + lastOffset,
+    })
+  }
+
+  return tokens
+}
+
 export function applyColorReplacements(color: string, replacements?: Record<string, string>): string {
   return replacements?.[color.toLowerCase()] || color
 }

packages/shikiji-twoslash/src/core.ts

@@ -6,7 +6,7 @@ import type { TwoslashExecuteOptions, TwoslashReturn } from 'twoslash'
 import type { ShikijiTransformer } from 'shikiji-core'
 import type { Element, ElementContent, Text } from 'hast'
 
-import { addClassToHast } from 'shikiji-core'
+import { addClassToHast, splitToken } from 'shikiji-core'
 import type { TransformerTwoslashOptions, TwoslashRenderer } from './types'
 
 export * from './types'
@@ -75,22 +75,12 @@ export function createTransformerFactory(
             const breakpointsInToken = breakpoints
               .filter(i => token.offset < i && i < token.offset + token.content.length)
               .map(i => i - token.offset)
+              .sort((a, b) => a - b)
 
             if (!breakpointsInToken.length)
               return token
 
-            breakpointsInToken.push(token.content.length)
-            breakpointsInToken.sort((a, b) => a - b)
-            let lastDelta = 0
-            return breakpointsInToken.map((i) => {
-              const n = {
-                ...token,
-                content: token.content.slice(lastDelta, i),
-                offset: token.offset + lastDelta,
-              }
-              lastDelta = i
-              return n
-            })
+            return splitToken(token, breakpointsInToken)
           })
         })
       },