Escape hash in code/codeblocks (#504)

* Escape hash in code/codeblocks

* format

* pin test accelerarte

Author: mishig25

.github/workflows/accelerate_doc.yml

@@ -23,6 +23,7 @@ jobs:
         with:
           repository: 'huggingface/accelerate'
           path: accelerate
+          ref: 16eb6d76bf987c7d8d877ce5152f2e29878eab37
 
       - name: Loading cache.
         uses: actions/cache@v2

kit/preprocessors/hashInCode.js

@@ -0,0 +1,17 @@
+import { replaceAsync } from "./utils.js";
+
+// Preprocessor that converts `#` char inside code/codeblocks into its html5 entity `#`
+// otherwise, an extra new space was being rendered
+export const hashInCodePreprocess = {
+	markup: async ({ content }) => {
+		const REGEX_CODE_BLOCK = /```.*?```/gs;
+		const REGEX_INLINE_CODE = /`.*?`/gs;
+		content = await replaceAsync(content, REGEX_CODE_BLOCK, async (codeContent) => {
+			return codeContent.replaceAll("#", "#");
+		});
+		content = await replaceAsync(content, REGEX_INLINE_CODE, async (codeContent) => {
+			return codeContent.replaceAll("#", "#");
+		});
+		return { code: content };
+	},
+};

kit/preprocessors/index.js

@@ -4,3 +4,4 @@ export { inferenceSnippetPreprocess } from "./inferenceSnippet.js";
 export { tokenizersLangPreprocess } from "./tokenizersLang.js";
 export { mdsvexPreprocess } from "./mdsvex/index.js";
 export { hfOptionsPreprocess } from "./hfOptions.js";
+export { hashInCodePreprocess } from "./hashInCode.js";

kit/preprocessors/mdsvex/index.js

@@ -6,6 +6,7 @@ import htmlTags from "html-tags";
 import { readdir } from "fs/promises";
 import path from "path";
 import cheerio from "cheerio";
+import { renderSvelteChars } from "../utils.js";
 
 /**
  * inside `<code>` html elements, we need to replace `&amp;` with `&`
@@ -43,15 +44,6 @@ export const mdsvexPreprocess = {
 	},
 };
 
-/**
- * Render escaped characters like `<`, `{`.
- * used for Doc
- * @param {string} code
- */
-function renderSvelteChars(code) {
-	return code.replace(/&amp;lcub;/g, "{").replace(/&amp;lt;/g, "<");
-}
-
 /**
  * Latex support in mdsvex
  * @param {string} content

kit/preprocessors/utils.js

@@ -24,12 +24,15 @@ export async function replaceAsync(string, searchValue, replacer) {
 }
 
 /**
- * Render escaped characters like `<`, `{`.
+ * Render escaped characters like `<`, `{`, '#'.
  * used for Doc
  * @param {string} code
  */
 export function renderSvelteChars(code) {
-	return code.replace(/&amp;lcub;/g, "{").replace(/&amp;lt;/g, "<");
+	return code
+		.replace(/&amp;lcub;/g, "{")
+		.replace(/&amp;lt;/g, "<")
+		.replace(/&amp;num;/g, "#");
 }
 
 /**

kit/svelte.config.js

@@ -8,13 +8,15 @@ import {
 	inferenceSnippetPreprocess,
 	tokenizersLangPreprocess,
 	hfOptionsPreprocess,
+	hashInCodePreprocess,
 } from "./preprocessors/index.js";
 
 /** @type {import('@sveltejs/kit').Config} */
 const config = {
 	// Consult https://kit.svelte.dev/docs/integrations#preprocessors
 	// for more information about preprocessors
 	preprocess: [
+		hashInCodePreprocess,
 		docstringPreprocess,
 		frameworkcontentPreprocess,
 		inferenceSnippetPreprocess,