Skip to content

tools/content: Support surveying unimplemented KaTeX features #1600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 3, 2025
Merged

tools/content: Support surveying unimplemented KaTeX features #1600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5e686f0
content [nfc]: Make MathNode a sealed class
rajveermalviya Jun 17, 2025
88ebff2
content [nfc]: Make `KatexHtmlParseError` private
rajveermalviya Jun 17, 2025
b11f758
content: Allow KaTeX parser to report failure reasons
rajveermalviya Jun 17, 2025
4b90a9a
tools/content: Support surveying unimplemented KaTeX features
rajveermalviya Jun 17, 2025
80dcd47
tools/content: Add a flag to control verbosity of KaTeX check result
rajveermalviya Jun 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions lib/model/content.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,11 +341,13 @@ class CodeBlockSpanNode extends ContentNode {
}
}

abstract class MathNode extends ContentNode {
sealed class MathNode extends ContentNode {
const MathNode({
super.debugHtmlNode,
required this.texSource,
required this.nodes,
this.debugHardFailReason,
this.debugSoftFailReason,
});

final String texSource;
Expand All @@ -357,6 +359,9 @@ abstract class MathNode extends ContentNode {
/// fallback instead.
final List<KatexNode>? nodes;

final KatexParserHardFailReason? debugHardFailReason;
final KatexParserSoftFailReason? debugSoftFailReason;

@override
void debugFillProperties(DiagnosticPropertiesBuilder properties) {
super.debugFillProperties(properties);
Expand Down Expand Up @@ -411,6 +416,8 @@ class MathBlockNode extends MathNode implements BlockContentNode {
super.debugHtmlNode,
required super.texSource,
required super.nodes,
super.debugHardFailReason,
super.debugSoftFailReason,
});
}

Expand Down Expand Up @@ -880,6 +887,8 @@ class MathInlineNode extends MathNode implements InlineContentNode {
super.debugHtmlNode,
required super.texSource,
required super.nodes,
super.debugHardFailReason,
super.debugSoftFailReason,
});
}

Expand Down Expand Up @@ -921,7 +930,9 @@ class _ZulipInlineContentParser {
return MathInlineNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: debugHtmlNode);
debugHtmlNode: debugHtmlNode,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null);
}

UserMentionNode? parseUserMention(dom.Element element) {
Expand Down Expand Up @@ -1628,7 +1639,9 @@ class _ZulipContentParser {
result.add(MathBlockNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: kDebugMode ? firstChild : null));
debugHtmlNode: kDebugMode ? firstChild : null,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null));
} else {
result.add(UnimplementedBlockContentNode(htmlNode: firstChild));
}
Expand Down Expand Up @@ -1664,7 +1677,9 @@ class _ZulipContentParser {
result.add(MathBlockNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: debugHtmlNode));
debugHtmlNode: debugHtmlNode,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null));
continue;
}
}
Expand Down
90 changes: 72 additions & 18 deletions lib/model/katex.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,40 @@ import 'binding.dart';
import 'content.dart';
import 'settings.dart';

/// The failure reason in case the KaTeX parser encountered a
/// `_KatexHtmlParseError` exception.
///
/// Generally this means that parser encountered an unexpected HTML structure,
/// an unsupported HTML node, or an unexpected inline CSS style or CSS class on
/// a specific node.
class KatexParserHardFailReason {
const KatexParserHardFailReason({
required this.error,
required this.stackTrace,
});

final String error;
final StackTrace stackTrace;
}

/// The failure reason in case the KaTeX parser found an unsupported
/// CSS class or unsupported inline CSS style property.
class KatexParserSoftFailReason {
const KatexParserSoftFailReason({
this.unsupportedCssClasses = const [],
this.unsupportedInlineCssProperties = const [],
});

final List<String> unsupportedCssClasses;
final List<String> unsupportedInlineCssProperties;
}

class MathParserResult {
const MathParserResult({
required this.texSource,
required this.nodes,
this.hardFailReason,
this.softFailReason,
});

final String texSource;
Expand All @@ -23,6 +53,9 @@ class MathParserResult {
/// CSS style, indicating that the widget should render the [texSource] as a
/// fallback instead.
final List<KatexNode>? nodes;

final KatexParserHardFailReason? hardFailReason;
final KatexParserSoftFailReason? softFailReason;
}

/// Parses the HTML spans containing KaTeX HTML tree.
Expand Down Expand Up @@ -88,21 +121,33 @@ MathParserResult? parseMath(dom.Element element, { required bool block }) {
final flagForceRenderKatex =
globalSettings.getBool(BoolGlobalSetting.forceRenderKatex);

KatexParserHardFailReason? hardFailReason;
KatexParserSoftFailReason? softFailReason;
List<KatexNode>? nodes;
if (flagRenderKatex) {
final parser = _KatexParser();
try {
nodes = parser.parseKatexHtml(katexHtmlElement);
} on KatexHtmlParseError catch (e, st) {
} on _KatexHtmlParseError catch (e, st) {
assert(debugLog('$e\n$st'));
hardFailReason = KatexParserHardFailReason(
error: e.message ?? 'unknown',
stackTrace: st);
}

if (parser.hasError && !flagForceRenderKatex) {
nodes = null;
softFailReason = KatexParserSoftFailReason(
unsupportedCssClasses: parser.unsupportedCssClasses,
unsupportedInlineCssProperties: parser.unsupportedInlineCssProperties);
}
}

return MathParserResult(nodes: nodes, texSource: texSource);
return MathParserResult(
nodes: nodes,
texSource: texSource,
hardFailReason: hardFailReason,
softFailReason: softFailReason);
} else {
return null;
}
Expand All @@ -112,6 +157,9 @@ class _KatexParser {
bool get hasError => _hasError;
bool _hasError = false;

final unsupportedCssClasses = <String>[];
final unsupportedInlineCssProperties = <String>[];

List<KatexNode> parseKatexHtml(dom.Element element) {
assert(element.localName == 'span');
assert(element.className == 'katex-html');
Expand All @@ -123,7 +171,10 @@ class _KatexParser {
if (node case dom.Element(localName: 'span')) {
return _parseSpan(node);
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError(
node is dom.Element
? 'unsupported html node: ${node.localName}'
: 'unsupported html node');
}
}));
}
Expand Down Expand Up @@ -303,48 +354,48 @@ class _KatexParser {
case 'fontsize-ensurer':
// .sizing,
// .fontsize-ensurer { ... }
if (index + 2 > spanClasses.length) throw KatexHtmlParseError();
if (index + 2 > spanClasses.length) throw _KatexHtmlParseError();
final resetSizeClass = spanClasses[index++];
final sizeClass = spanClasses[index++];

final resetSizeClassSuffix = _resetSizeClassRegExp.firstMatch(resetSizeClass)?.group(1);
if (resetSizeClassSuffix == null) throw KatexHtmlParseError();
if (resetSizeClassSuffix == null) throw _KatexHtmlParseError();
final sizeClassSuffix = _sizeClassRegExp.firstMatch(sizeClass)?.group(1);
if (sizeClassSuffix == null) throw KatexHtmlParseError();
if (sizeClassSuffix == null) throw _KatexHtmlParseError();

const sizes = <double>[0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.2, 1.44, 1.728, 2.074, 2.488];

final resetSizeIdx = int.parse(resetSizeClassSuffix, radix: 10);
final sizeIdx = int.parse(sizeClassSuffix, radix: 10);

// These indexes start at 1.
if (resetSizeIdx > sizes.length) throw KatexHtmlParseError();
if (sizeIdx > sizes.length) throw KatexHtmlParseError();
if (resetSizeIdx > sizes.length) throw _KatexHtmlParseError();
if (sizeIdx > sizes.length) throw _KatexHtmlParseError();
fontSizeEm = sizes[sizeIdx - 1] / sizes[resetSizeIdx - 1];

case 'delimsizing':
// .delimsizing { ... }
if (index + 1 > spanClasses.length) throw KatexHtmlParseError();
if (index + 1 > spanClasses.length) throw _KatexHtmlParseError();
fontFamily = switch (spanClasses[index++]) {
'size1' => 'KaTeX_Size1',
'size2' => 'KaTeX_Size2',
'size3' => 'KaTeX_Size3',
'size4' => 'KaTeX_Size4',
'mult' =>
// TODO handle nested spans with `.delim-size{1,4}` class.
throw KatexHtmlParseError(),
_ => throw KatexHtmlParseError(),
throw _KatexHtmlParseError(),
_ => throw _KatexHtmlParseError(),
};

// TODO handle .nulldelimiter and .delimcenter .

case 'op-symbol':
// .op-symbol { ... }
if (index + 1 > spanClasses.length) throw KatexHtmlParseError();
if (index + 1 > spanClasses.length) throw _KatexHtmlParseError();
fontFamily = switch (spanClasses[index++]) {
'small-op' => 'KaTeX_Size1',
'large-op' => 'KaTeX_Size2',
_ => throw KatexHtmlParseError(),
_ => throw _KatexHtmlParseError(),
};

// TODO handle more classes from katex.scss
Expand Down Expand Up @@ -374,6 +425,7 @@ class _KatexParser {

default:
assert(debugLog('KaTeX: Unsupported CSS class: $spanClass'));
unsupportedCssClasses.add(spanClass);
_hasError = true;
}
}
Expand All @@ -392,7 +444,7 @@ class _KatexParser {
} else {
spans = _parseChildSpans(element.nodes);
}
if (text == null && spans == null) throw KatexHtmlParseError();
if (text == null && spans == null) throw _KatexHtmlParseError();

return KatexSpanNode(
styles: inlineStyles != null
Expand Down Expand Up @@ -427,17 +479,18 @@ class _KatexParser {
// TODO handle more CSS properties
assert(debugLog('KaTeX: Unsupported CSS expression:'
' ${expression.toDebugString()}'));
unsupportedInlineCssProperties.add(property);
_hasError = true;
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError();
}
}

return KatexSpanStyles(
heightEm: heightEm,
);
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError();
}
}
return null;
Expand Down Expand Up @@ -540,9 +593,10 @@ class KatexSpanStyles {
}
}

class KatexHtmlParseError extends Error {
class _KatexHtmlParseError extends Error {
final String? message;
KatexHtmlParseError([this.message]);

_KatexHtmlParseError([this.message]);

@override
String toString() {
Expand Down
15 changes: 14 additions & 1 deletion tools/content/check-features
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ The steps are:
file.
This wraps around tools/content/unimplemented_features_test.dart.

katex-check
Check for unimplemented KaTeX features. This requires the corpus
directory \`CORPUS_DIR\` to contain at least one corpus file.
This wraps around tools/content/unimplemented_katex_test.dart.

Options:

--config <FILE>
Expand All @@ -50,7 +55,7 @@ opt_verbose=
opt_steps=()
while (( $# )); do
case "$1" in
fetch|check) opt_steps+=("$1"); shift;;
fetch|check|katex-check) opt_steps+=("$1"); shift;;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's update the usage message too (the --help output), to keep it in sync.

--config) shift; opt_zuliprc="$1"; shift;;
--verbose) opt_verbose=1; shift;;
--help) usage; exit 0;;
Expand Down Expand Up @@ -98,11 +103,19 @@ run_check() {
|| return 1
}

run_katex_check() {
flutter test tools/content/unimplemented_katex_test.dart \
--dart-define=corpusDir="$opt_corpus_dir" \
--dart-define=verbose="$opt_verbose" \
|| return 1
}

for step in "${opt_steps[@]}"; do
echo "Running ${step}"
case "${step}" in
fetch) run_fetch ;;
check) run_check ;;
katex-check) run_katex_check ;;
*) echo >&2 "Internal error: unknown step ${step}" ;;
esac
done
Loading