Skip to content

tools/content: Support surveying unimplemented KaTeX features #1600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 3, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions lib/model/content.dart
Original file line number Diff line number Diff line change
Expand Up @@ -341,11 +341,13 @@ class CodeBlockSpanNode extends ContentNode {
}
}

abstract class MathNode extends ContentNode {
sealed class MathNode extends ContentNode {
const MathNode({
super.debugHtmlNode,
required this.texSource,
required this.nodes,
this.debugHardFailReason,
this.debugSoftFailReason,
});

final String texSource;
Expand All @@ -357,6 +359,9 @@ abstract class MathNode extends ContentNode {
/// fallback instead.
final List<KatexNode>? nodes;

final KatexParserHardFailReason? debugHardFailReason;
final KatexParserSoftFailReason? debugSoftFailReason;

@override
void debugFillProperties(DiagnosticPropertiesBuilder properties) {
super.debugFillProperties(properties);
Expand Down Expand Up @@ -411,6 +416,8 @@ class MathBlockNode extends MathNode implements BlockContentNode {
super.debugHtmlNode,
required super.texSource,
required super.nodes,
super.debugHardFailReason,
super.debugSoftFailReason,
});
}

Expand Down Expand Up @@ -880,6 +887,8 @@ class MathInlineNode extends MathNode implements InlineContentNode {
super.debugHtmlNode,
required super.texSource,
required super.nodes,
super.debugHardFailReason,
super.debugSoftFailReason,
});
}

Expand Down Expand Up @@ -921,7 +930,9 @@ class _ZulipInlineContentParser {
return MathInlineNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: debugHtmlNode);
debugHtmlNode: debugHtmlNode,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null);
}

UserMentionNode? parseUserMention(dom.Element element) {
Expand Down Expand Up @@ -1628,7 +1639,9 @@ class _ZulipContentParser {
result.add(MathBlockNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: kDebugMode ? firstChild : null));
debugHtmlNode: kDebugMode ? firstChild : null,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null));
} else {
result.add(UnimplementedBlockContentNode(htmlNode: firstChild));
}
Expand Down Expand Up @@ -1664,7 +1677,9 @@ class _ZulipContentParser {
result.add(MathBlockNode(
texSource: parsed.texSource,
nodes: parsed.nodes,
debugHtmlNode: debugHtmlNode));
debugHtmlNode: debugHtmlNode,
debugHardFailReason: kDebugMode ? parsed.hardFailReason : null,
debugSoftFailReason: kDebugMode ? parsed.softFailReason : null));
continue;
}
}
Expand Down
90 changes: 72 additions & 18 deletions lib/model/katex.dart
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,40 @@ import 'binding.dart';
import 'content.dart';
import 'settings.dart';

/// The failure reason in case the KaTeX parser encountered a
/// `_KatexHtmlParseError` exception.
///
/// Generally this means that parser encountered an unexpected HTML structure,
/// an unsupported HTML node, or an unexpected inline CSS style or CSS class on
/// a specific node.
class KatexParserHardFailReason {
const KatexParserHardFailReason({
required this.error,
required this.stackTrace,
});

final String error;
final StackTrace stackTrace;
}

/// The failure reason in case the KaTeX parser found an unsupported
/// CSS class or unsupported inline CSS style property.
class KatexParserSoftFailReason {
const KatexParserSoftFailReason({
this.unsupportedCssClasses = const [],
this.unsupportedInlineCssProperties = const [],
});

final List<String> unsupportedCssClasses;
final List<String> unsupportedInlineCssProperties;
}

class MathParserResult {
const MathParserResult({
required this.texSource,
required this.nodes,
this.hardFailReason,
this.softFailReason,
});

final String texSource;
Expand All @@ -23,6 +53,9 @@ class MathParserResult {
/// CSS style, indicating that the widget should render the [texSource] as a
/// fallback instead.
final List<KatexNode>? nodes;

final KatexParserHardFailReason? hardFailReason;
final KatexParserSoftFailReason? softFailReason;
}

/// Parses the HTML spans containing KaTeX HTML tree.
Expand Down Expand Up @@ -88,21 +121,33 @@ MathParserResult? parseMath(dom.Element element, { required bool block }) {
final flagForceRenderKatex =
globalSettings.getBool(BoolGlobalSetting.forceRenderKatex);

KatexParserHardFailReason? hardFailReason;
KatexParserSoftFailReason? softFailReason;
List<KatexNode>? nodes;
if (flagRenderKatex) {
final parser = _KatexParser();
try {
nodes = parser.parseKatexHtml(katexHtmlElement);
} on KatexHtmlParseError catch (e, st) {
} on _KatexHtmlParseError catch (e, st) {
assert(debugLog('$e\n$st'));
hardFailReason = KatexParserHardFailReason(
error: e.message ?? 'unknown',
stackTrace: st);
}

if (parser.hasError && !flagForceRenderKatex) {
nodes = null;
softFailReason = KatexParserSoftFailReason(
unsupportedCssClasses: parser.unsupportedCssClasses,
unsupportedInlineCssProperties: parser.unsupportedInlineCssProperties);
}
}

return MathParserResult(nodes: nodes, texSource: texSource);
return MathParserResult(
nodes: nodes,
texSource: texSource,
hardFailReason: hardFailReason,
softFailReason: softFailReason);
} else {
return null;
}
Expand All @@ -112,6 +157,9 @@ class _KatexParser {
bool get hasError => _hasError;
bool _hasError = false;

final unsupportedCssClasses = <String>[];
final unsupportedInlineCssProperties = <String>[];

List<KatexNode> parseKatexHtml(dom.Element element) {
assert(element.localName == 'span');
assert(element.className == 'katex-html');
Expand All @@ -123,7 +171,10 @@ class _KatexParser {
if (node case dom.Element(localName: 'span')) {
return _parseSpan(node);
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError(
node is dom.Element
? 'unsupported html node: ${node.localName}'
: 'unsupported html node');
}
}));
}
Expand Down Expand Up @@ -303,48 +354,48 @@ class _KatexParser {
case 'fontsize-ensurer':
// .sizing,
// .fontsize-ensurer { ... }
if (index + 2 > spanClasses.length) throw KatexHtmlParseError();
if (index + 2 > spanClasses.length) throw _KatexHtmlParseError();
final resetSizeClass = spanClasses[index++];
final sizeClass = spanClasses[index++];

final resetSizeClassSuffix = _resetSizeClassRegExp.firstMatch(resetSizeClass)?.group(1);
if (resetSizeClassSuffix == null) throw KatexHtmlParseError();
if (resetSizeClassSuffix == null) throw _KatexHtmlParseError();
final sizeClassSuffix = _sizeClassRegExp.firstMatch(sizeClass)?.group(1);
if (sizeClassSuffix == null) throw KatexHtmlParseError();
if (sizeClassSuffix == null) throw _KatexHtmlParseError();

const sizes = <double>[0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.2, 1.44, 1.728, 2.074, 2.488];

final resetSizeIdx = int.parse(resetSizeClassSuffix, radix: 10);
final sizeIdx = int.parse(sizeClassSuffix, radix: 10);

// These indexes start at 1.
if (resetSizeIdx > sizes.length) throw KatexHtmlParseError();
if (sizeIdx > sizes.length) throw KatexHtmlParseError();
if (resetSizeIdx > sizes.length) throw _KatexHtmlParseError();
if (sizeIdx > sizes.length) throw _KatexHtmlParseError();
fontSizeEm = sizes[sizeIdx - 1] / sizes[resetSizeIdx - 1];

case 'delimsizing':
// .delimsizing { ... }
if (index + 1 > spanClasses.length) throw KatexHtmlParseError();
if (index + 1 > spanClasses.length) throw _KatexHtmlParseError();
fontFamily = switch (spanClasses[index++]) {
'size1' => 'KaTeX_Size1',
'size2' => 'KaTeX_Size2',
'size3' => 'KaTeX_Size3',
'size4' => 'KaTeX_Size4',
'mult' =>
// TODO handle nested spans with `.delim-size{1,4}` class.
throw KatexHtmlParseError(),
_ => throw KatexHtmlParseError(),
throw _KatexHtmlParseError(),
_ => throw _KatexHtmlParseError(),
};

// TODO handle .nulldelimiter and .delimcenter .

case 'op-symbol':
// .op-symbol { ... }
if (index + 1 > spanClasses.length) throw KatexHtmlParseError();
if (index + 1 > spanClasses.length) throw _KatexHtmlParseError();
fontFamily = switch (spanClasses[index++]) {
'small-op' => 'KaTeX_Size1',
'large-op' => 'KaTeX_Size2',
_ => throw KatexHtmlParseError(),
_ => throw _KatexHtmlParseError(),
};

// TODO handle more classes from katex.scss
Expand Down Expand Up @@ -374,6 +425,7 @@ class _KatexParser {

default:
assert(debugLog('KaTeX: Unsupported CSS class: $spanClass'));
unsupportedCssClasses.add(spanClass);
_hasError = true;
}
}
Expand All @@ -392,7 +444,7 @@ class _KatexParser {
} else {
spans = _parseChildSpans(element.nodes);
}
if (text == null && spans == null) throw KatexHtmlParseError();
if (text == null && spans == null) throw _KatexHtmlParseError();

return KatexSpanNode(
styles: inlineStyles != null
Expand Down Expand Up @@ -427,17 +479,18 @@ class _KatexParser {
// TODO handle more CSS properties
assert(debugLog('KaTeX: Unsupported CSS expression:'
' ${expression.toDebugString()}'));
unsupportedInlineCssProperties.add(property);
_hasError = true;
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError();
}
}

return KatexSpanStyles(
heightEm: heightEm,
);
} else {
throw KatexHtmlParseError();
throw _KatexHtmlParseError();
}
}
return null;
Expand Down Expand Up @@ -540,9 +593,10 @@ class KatexSpanStyles {
}
}

class KatexHtmlParseError extends Error {
class _KatexHtmlParseError extends Error {
final String? message;
KatexHtmlParseError([this.message]);

_KatexHtmlParseError([this.message]);

@override
String toString() {
Expand Down
15 changes: 14 additions & 1 deletion tools/content/check-features
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ The steps are:
file.
This wraps around tools/content/unimplemented_features_test.dart.

katex-check
Check for unimplemented KaTeX features. This requires the corpus
directory \`CORPUS_DIR\` to contain at least one corpus file.
This wraps around tools/content/unimplemented_katex_test.dart.

Options:

--config <FILE>
Expand All @@ -50,7 +55,7 @@ opt_verbose=
opt_steps=()
while (( $# )); do
case "$1" in
fetch|check) opt_steps+=("$1"); shift;;
fetch|check|katex-check) opt_steps+=("$1"); shift;;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's update the usage message too (the --help output), to keep it in sync.

--config) shift; opt_zuliprc="$1"; shift;;
--verbose) opt_verbose=1; shift;;
--help) usage; exit 0;;
Expand Down Expand Up @@ -98,11 +103,19 @@ run_check() {
|| return 1
}

run_katex_check() {
flutter test tools/content/unimplemented_katex_test.dart \
--dart-define=corpusDir="$opt_corpus_dir" \
--dart-define=verbose="$opt_verbose" \
|| return 1
}

for step in "${opt_steps[@]}"; do
echo "Running ${step}"
case "${step}" in
fetch) run_fetch ;;
check) run_check ;;
katex-check) run_katex_check ;;
*) echo >&2 "Internal error: unknown step ${step}" ;;
esac
done
Loading