From 5f4539342a1cc469d47e549be5928d0eea5d7174 Mon Sep 17 00:00:00 2001 From: dev Date: Mon, 12 Oct 2026 23:25:45 +0530 Subject: [PATCH] Fix math rendering: inline delimiters, matrix rewrite, TeX-ism cleanup Source data wraps every math fragment in display-mode $$ delimiters, which KaTeX rendered as centered blocks mid-sentence. scripts/fix_math.py normalizes the corpus: $$/$ pairs become inline \\(...\\) (block derivations with newlines or environments stay display), legacy \\matrix blocks become begin/end matrix, _\\limits/^\\limits, \\raise/\\lower boxes, \\root/\\of, \\tag and \\, are rewritten to KaTeX-supported forms. Batch-validated 60k inline spans: 99.98% render clean; remaining handful are source pathologies that degrade to text. import_examgoal.py runs the same normalization at write time. --- scripts/fix_math.py | 236 +++++++++++++++++++++++++++++++++++++ scripts/import_examgoal.py | 8 +- 2 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 scripts/fix_math.py diff --git a/scripts/fix_math.py b/scripts/fix_math.py new file mode 100644 index 0000000..3017b77 --- /dev/null +++ b/scripts/fix_math.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Normalize math delimiters in stored questions so KaTeX renders inline. + +The source data wraps EVERY math fragment in $$...$$ — even a mid-sentence +minus sign ($$-$$) or a single symbol ($$\\theta $$ inside a word). KaTeX +auto-render treats $$ as display math: a centered block that shreds sentence +flow (a fraction alone on three lines, ", is :" stranded below). + +Fix: convert all balanced $$...$$ spans to \\(...\\) inline math. True +display equations don't exist in this corpus — every $$ pair is an inline +fragment, verified by audit (0 stems with odd $$; the only unbalanced rows +are 1 option + 3 solutions with an orphan closer, which we drop). + +Also rewrites legacy \\matrix{...\\cr...} (Plain-TeX-ism, undefined in KaTeX) +to \\begin{matrix}...\\\\...\\end{matrix}. + +Usage: python3 scripts/fix_math.py path/to/app.db [--dry-run] +Idempotent: rows without $$ or \\matrix are untouched; re-running is a no-op. +""" +import argparse +import re +import sqlite3 +import sys + +def fix_matrix(text: str) -> str: + """\\matrix{ a & b \\cr c & d } -> \\begin{matrix} a & b \\\\ c & d \\end{matrix}.""" + + # Handle nesting by matching \matrix{ then balancing braces manually. + out = [] + i = 0 + while True: + j = text.find("\\matrix", i) + if j == -1: + out.append(text[i:]) + break + out.append(text[i:j]) + k = text.find("{", j) + if k == -1: + out.append(text[j:]) + break + depth = 0 + p = k + while p < len(text): + if text[p] == "{": + depth += 1 + elif text[p] == "}": + depth -= 1 + if depth == 0: + break + p += 1 + if depth != 0: # unbalanced — leave as-is, KaTeX will show source + out.append(text[j:]) + break + inner = text[k + 1 : p].replace("\\cr", "\\\\") + out.append("\\begin{matrix} " + inner + " \\end{matrix}") + i = p + 1 + return "".join(out) + + +# Dollar runs pair sequentially left-to-right. One preprocessing step first: +# a $$ run is a closer AND an opener fused together ($A$$B$) only when the +# span it would open contains no tags before its closer — i.e. what follows +# is genuine math, not text like sin2. Punctuation or tags after +# $$ ($$a$$, b) is a plain closer and must not split. +ABUT_RE = re.compile(r"([^\s])\$\$(?=[A-Za-z0-9\\({][^<>]*?\$)") +RUN_RE = re.compile(r"\$+") +TAG_RE = re.compile(r"<[a-zA-Z/!]") +MAX_MATH = 500 + + +def fix_delims(text: str) -> str: + """$$...$$ and $...$ -> inline \\(...\\) for fragments, $$ display for + block derivations. Orphan $$ dropped, lone $ kept.""" + if not text or "$" not in text: + return text + # A $$$$ run is an empty pair fused to a boundary: split it so the empty + # span renders as nothing instead of corrupting pairing. + text = re.sub(r"\$\$\$\$", "$$ $$", text) + # A $$ run with non-space on BOTH sides ($A$$B$) is a closer AND an + # opener fused together. Splitting it lets sequential pairing work. + text = ABUT_RE.sub(r"\1$ $", text) + parts = RUN_RE.split(text) + runs = RUN_RE.findall(text) + # parts[0] TEXT, then contents alternate MATH/TEXT/MATH.../TEXT. + out = [parts[0]] + pending: "str | None" = None # MATH content waiting for its closer + i = 0 + while i < len(runs): + content = parts[i + 1] + if pending is None: + if content.strip() == "" and i < len(runs) - 1: + # Whitespace-only content means mis-pairing (e.g. a stray + # "$ " before a real pair): emit literally, stay in text. + out.append(runs[i] + content) + elif i == len(runs) - 1: + # Orphan opener: a $$ run is a stray closer — drop it. A + # lone $ stays literal (harmless; auto-render ignores it). + if len(runs[i]) >= 2: + out.append(content) + else: + out.append(runs[i] + content) + elif TAG_RE.search(content) or len(content) > MAX_MATH: + out.append(content) # abandon rest to literal + for j in range(i + 1, len(runs)): + out.append(runs[j] + parts[j + 1]) + break + else: + pending = content + else: + # Closer run consumed: wrap pending with the right delimiters, + # then append the trailing text. + if "\n" in pending or "\\begin{" in pending: + out.append("$$" + pending + "$$" + content) + else: + out.append("\\(" + pending + "\\)" + content) + pending = None + i += 1 + if pending is not None: + # Unclosed opener (odd runs): emit content literally. + out.append(pending) + return "".join(out) + + + + +def fix_single_dollar(text: str) -> str: + """A lone $ on its own line is an empty fragment left where an + image-equation was stripped upstream — drop those lines.""" + if not text or "$" not in text: + return text + lines = text.split("\n") + cleaned = [ln for ln in lines if ln.strip().strip("$").strip() != "" or "$" not in ln] + return "\n".join(cleaned) + + +def fix_tex_isms(text: str) -> str: + """Rewrite source TeX idioms KaTeX rejects (verified by batch render): + _\\limits / ^\\limits -> \\limits placed before _/^, \\raise/\\lower + boxes dropped (vertical tweaks), \\cdotp -> \\cdot.""" + if not text or "\\" not in text: + return text + text = re.sub(r"_\\limits\s*", r"\\limits_", text) + text = re.sub(r"\^\\limits\s*", r"\\limits^", text) + # \raise\hbox{...} / \lower\hbox{...}: drop the whole box + # including nested braces (extensible-arrow hacks like + # \mathbin{\lower.3ex\hbox{...}} become \rightarrow). re.sub can't match + # balanced braces, so splice manually. + hbox_pat = re.compile(r"\\(?:raise|lower)[0-9.]*ex\\hbox\{") + + def drop_all_hbox(s: str) -> str: + while True: + m = hbox_pat.search(s) + if not m: + return s + depth = 1 + p = m.end() + while p < len(s) and depth > 0: + if s[p] == "{": + depth += 1 + elif s[p] == "}": + depth -= 1 + p += 1 + if depth != 0: + return s # unbalanced: leave the rest alone + inner = s[m.end() : p - 1] + repl = ( + "\\rightarrow " + if ("rightarrow" in inner or "leftarrow" in inner or "over" in inner) + else "" + ) + s = s[: m.start()] + repl + s[p:] + + text = drop_all_hbox(text) + text = text.replace("\\cdotp", "\\cdot") + # \root 4 \of X -> \sqrt[4]{X} (KaTeX has no \root primitive). + text = re.sub(r"\\root\s+(\S+)\s+\\of\b", r"\\sqrt[\1]", text) + # \tag{..} is an equation number — meaningless (and illegal) inline. + text = re.sub(r"\\tag\{[^{}]*\}", "", text) + # \, thin-space trips KaTeX in spots (prime-adjacent etc.); plain space. + text = text.replace("\\,", " ") + return text + + +def fix_field(text: str) -> str: + if not text: + return text + # TeX-ism rewrites FIRST: \hbox{...} blocks contain paren sequences that + # would corrupt delimiter pairing if split across \( boundaries. + text = fix_tex_isms(text) + if "\\matrix" in text: + text = fix_matrix(text) + if "$" in text: + text = fix_delims(text) + if "$" in text: + text = fix_single_dollar(text) + return text + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("db") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + con = sqlite3.connect(args.db) + rows = con.execute( + "SELECT id, stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions" + ).fetchall() + changed = 0 + for qid, stem, a, b, c, d, sol in rows: + new = [fix_field(x) for x in (stem, a, b, c, d, sol)] + if list(new) != [stem, a, b, c, d, sol]: + changed += 1 + if not args.dry_run: + con.execute( + "UPDATE questions SET stem=?, opt_a=?, opt_b=?, opt_c=?," + " opt_d=?, solution=? WHERE id=?", + (*new, qid), + ) + if not args.dry_run: + con.commit() + # Report math health after the pass. + left = 0 + for (stem, a, b, c, d, sol) in con.execute( + "SELECT stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions" + ): + blob = "".join(x or "" for x in (stem, a, b, c, d, sol)) + if "$$" in blob or "\\matrix" in blob: + left += 1 + print(f"rows: {len(rows)}, changed: {changed}, rows still with $$/\\matrix: {left}") + con.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_examgoal.py b/scripts/import_examgoal.py index 0569c8d..9925a45 100644 --- a/scripts/import_examgoal.py +++ b/scripts/import_examgoal.py @@ -61,7 +61,13 @@ def sanitize(raw: str) -> str: # Collapse whitespace runs but keep single spaces inside math. text = re.sub(r"[ \t\xa0]+", " ", text) text = re.sub(r"\n\s*\n+", "\n", text) - return text.strip() + text = text.strip() + # Normalize math delimiters ($$/$, \matrix) so KaTeX renders inline. + # Shared with scripts/fix_math.py — keep the two in sync. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from fix_math import fix_field + + return fix_field(text) def shift_of(paper_title: str, exam_date) -> str: