Fix SyntaxHighlighter delimiter state (#15104)

The syntax highlighter needs to reset the lexer's previous `delimiter_state` after exiting a nested state.

Author: straight-shoota

spec/std/crystal/syntax_highlighter/html_spec.cr

@@ -162,4 +162,8 @@ describe Crystal::SyntaxHighlighter::HTML do
     it_highlights! "%w[foo"
     it_highlights! "%i[foo"
   end
+
+  # fix for https://forum.crystal-lang.org/t/question-about-the-crystal-syntax-highlighter/7283
+  it_highlights %q(/#{l[""]}/
+    "\\n"), %(<span class="s">/</span><span class="i">\#{</span>l[<span class="s">&quot;&quot;</span>]<span class="i">}</span><span class="s">/</span>\n    <span class="s">&quot;\\\\n&quot;</span>)
 end

src/crystal/syntax_highlighter.cr

@@ -84,6 +84,8 @@ abstract class Crystal::SyntaxHighlighter
     space_before = false
 
     while true
+      previous_delimiter_state = lexer.token.delimiter_state
+
       token = lexer.next_token
 
       case token.type
@@ -105,6 +107,7 @@ abstract class Crystal::SyntaxHighlighter
           highlight_token token, last_is_def
         else
           highlight_delimiter_state lexer, token
+          token.delimiter_state = previous_delimiter_state
         end
       when .string_array_start?, .symbol_array_start?
         highlight_string_array lexer, token