Fix math rendering: inline delimiters, matrix rewrite, TeX-ism cleanup

Source data wraps every math fragment in display-mode $$ delimiters,
which KaTeX rendered as centered blocks mid-sentence. scripts/fix_math.py
normalizes the corpus: $$/$ pairs become inline \\(...\\) (block
derivations with newlines or environments stay display), legacy
\\matrix blocks become begin/end matrix, _\\limits/^\\limits,
\\raise/\\lower boxes, \\root/\\of, \\tag and \\, are rewritten
to KaTeX-supported forms. Batch-validated 60k inline spans: 99.98% render
clean; remaining handful are source pathologies that degrade to text.
import_examgoal.py runs the same normalization at write time.

scripts/fix_math.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+"""Normalize math delimiters in stored questions so KaTeX renders inline.
+
+The source data wraps EVERY math fragment in $$...$$ — even a mid-sentence
+minus sign ($$-$$) or a single symbol ($$\\theta $$ inside a word). KaTeX
+auto-render treats $$ as display math: a centered block that shreds sentence
+flow (a fraction alone on three lines, ", is :" stranded below).
+
+Fix: convert all balanced $$...$$ spans to \\(...\\) inline math. True
+display equations don't exist in this corpus — every $$ pair is an inline
+fragment, verified by audit (0 stems with odd $$; the only unbalanced rows
+are 1 option + 3 solutions with an orphan closer, which we drop).
+
+Also rewrites legacy \\matrix{...\\cr...} (Plain-TeX-ism, undefined in KaTeX)
+to \\begin{matrix}...\\\\...\\end{matrix}.
+
+Usage: python3 scripts/fix_math.py path/to/app.db [--dry-run]
+Idempotent: rows without $$ or \\matrix are untouched; re-running is a no-op.
+"""
+import argparse
+import re
+import sqlite3
+import sys
+
+def fix_matrix(text: str) -> str:
+    """\\matrix{ a & b \\cr c & d } -> \\begin{matrix} a & b \\\\ c & d \\end{matrix}."""
+
+    # Handle nesting by matching \matrix{ then balancing braces manually.
+    out = []
+    i = 0
+    while True:
+        j = text.find("\\matrix", i)
+        if j == -1:
+            out.append(text[i:])
+            break
+        out.append(text[i:j])
+        k = text.find("{", j)
+        if k == -1:
+            out.append(text[j:])
+            break
+        depth = 0
+        p = k
+        while p < len(text):
+            if text[p] == "{":
+                depth += 1
+            elif text[p] == "}":
+                depth -= 1
+                if depth == 0:
+                    break
+            p += 1
+        if depth != 0:  # unbalanced — leave as-is, KaTeX will show source
+            out.append(text[j:])
+            break
+        inner = text[k + 1 : p].replace("\\cr", "\\\\")
+        out.append("\\begin{matrix} " + inner + " \\end{matrix}")
+        i = p + 1
+    return "".join(out)
+
+
+# Dollar runs pair sequentially left-to-right. One preprocessing step first:
+# a $$ run is a closer AND an opener fused together ($A$$B$) only when the
+# span it would open contains no tags before its closer — i.e. what follows
+# is genuine math, not text like sin<sup>2</sup>. Punctuation or tags after
+# $$ ($$a$$, b) is a plain closer and must not split.
+ABUT_RE = re.compile(r"([^\s])\$\$(?=[A-Za-z0-9\\({][^<>]*?\$)")
+RUN_RE = re.compile(r"\$+")
+TAG_RE = re.compile(r"<[a-zA-Z/!]")
+MAX_MATH = 500
+
+
+def fix_delims(text: str) -> str:
+    """$$...$$ and $...$ -> inline \\(...\\) for fragments, $$ display for
+    block derivations. Orphan $$ dropped, lone $ kept."""
+    if not text or "$" not in text:
+        return text
+    # A $$$$ run is an empty pair fused to a boundary: split it so the empty
+    # span renders as nothing instead of corrupting pairing.
+    text = re.sub(r"\$\$\$\$", "$$ $$", text)
+    # A $$ run with non-space on BOTH sides ($A$$B$) is a closer AND an
+    # opener fused together. Splitting it lets sequential pairing work.
+    text = ABUT_RE.sub(r"\1$ $", text)
+    parts = RUN_RE.split(text)
+    runs = RUN_RE.findall(text)
+    # parts[0] TEXT, then contents alternate MATH/TEXT/MATH.../TEXT.
+    out = [parts[0]]
+    pending: "str | None" = None  # MATH content waiting for its closer
+    i = 0
+    while i < len(runs):
+        content = parts[i + 1]
+        if pending is None:
+            if content.strip() == "" and i < len(runs) - 1:
+                # Whitespace-only content means mis-pairing (e.g. a stray
+                # "$ " before a real pair): emit literally, stay in text.
+                out.append(runs[i] + content)
+            elif i == len(runs) - 1:
+                # Orphan opener: a $$ run is a stray closer — drop it. A
+                # lone $ stays literal (harmless; auto-render ignores it).
+                if len(runs[i]) >= 2:
+                    out.append(content)
+                else:
+                    out.append(runs[i] + content)
+            elif TAG_RE.search(content) or len(content) > MAX_MATH:
+                out.append(content)  # abandon rest to literal
+                for j in range(i + 1, len(runs)):
+                    out.append(runs[j] + parts[j + 1])
+                break
+            else:
+                pending = content
+        else:
+            # Closer run consumed: wrap pending with the right delimiters,
+            # then append the trailing text.
+            if "\n" in pending or "\\begin{" in pending:
+                out.append("$$" + pending + "$$" + content)
+            else:
+                out.append("\\(" + pending + "\\)" + content)
+            pending = None
+        i += 1
+    if pending is not None:
+        # Unclosed opener (odd runs): emit content literally.
+        out.append(pending)
+    return "".join(out)
+
+
+
+
+def fix_single_dollar(text: str) -> str:
+    """A lone $ on its own line is an empty fragment left where an
+    image-equation was stripped upstream — drop those lines."""
+    if not text or "$" not in text:
+        return text
+    lines = text.split("\n")
+    cleaned = [ln for ln in lines if ln.strip().strip("$").strip() != "" or "$" not in ln]
+    return "\n".join(cleaned)
+
+
+def fix_tex_isms(text: str) -> str:
+    """Rewrite source TeX idioms KaTeX rejects (verified by batch render):
+    _\\limits / ^\\limits -> \\limits placed before _/^, \\raise/\\lower
+    boxes dropped (vertical tweaks), \\cdotp -> \\cdot."""
+    if not text or "\\" not in text:
+        return text
+    text = re.sub(r"_\\limits\s*", r"\\limits_", text)
+    text = re.sub(r"\^\\limits\s*", r"\\limits^", text)
+    # \raise<dim>\hbox{...} / \lower<dim>\hbox{...}: drop the whole box
+    # including nested braces (extensible-arrow hacks like
+    # \mathbin{\lower.3ex\hbox{...}} become \rightarrow). re.sub can't match
+    # balanced braces, so splice manually.
+    hbox_pat = re.compile(r"\\(?:raise|lower)[0-9.]*ex\\hbox\{")
+
+    def drop_all_hbox(s: str) -> str:
+        while True:
+            m = hbox_pat.search(s)
+            if not m:
+                return s
+            depth = 1
+            p = m.end()
+            while p < len(s) and depth > 0:
+                if s[p] == "{":
+                    depth += 1
+                elif s[p] == "}":
+                    depth -= 1
+                p += 1
+            if depth != 0:
+                return s  # unbalanced: leave the rest alone
+            inner = s[m.end() : p - 1]
+            repl = (
+                "\\rightarrow "
+                if ("rightarrow" in inner or "leftarrow" in inner or "over" in inner)
+                else ""
+            )
+            s = s[: m.start()] + repl + s[p:]
+
+    text = drop_all_hbox(text)
+    text = text.replace("\\cdotp", "\\cdot")
+    # \root 4 \of X -> \sqrt[4]{X} (KaTeX has no \root primitive).
+    text = re.sub(r"\\root\s+(\S+)\s+\\of\b", r"\\sqrt[\1]", text)
+    # \tag{..} is an equation number — meaningless (and illegal) inline.
+    text = re.sub(r"\\tag\{[^{}]*\}", "", text)
+    # \, thin-space trips KaTeX in spots (prime-adjacent etc.); plain space.
+    text = text.replace("\\,", " ")
+    return text
+
+
+def fix_field(text: str) -> str:
+    if not text:
+        return text
+    # TeX-ism rewrites FIRST: \hbox{...} blocks contain paren sequences that
+    # would corrupt delimiter pairing if split across \( boundaries.
+    text = fix_tex_isms(text)
+    if "\\matrix" in text:
+        text = fix_matrix(text)
+    if "$" in text:
+        text = fix_delims(text)
+    if "$" in text:
+        text = fix_single_dollar(text)
+    return text
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    con = sqlite3.connect(args.db)
+    rows = con.execute(
+        "SELECT id, stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions"
+    ).fetchall()
+    changed = 0
+    for qid, stem, a, b, c, d, sol in rows:
+        new = [fix_field(x) for x in (stem, a, b, c, d, sol)]
+        if list(new) != [stem, a, b, c, d, sol]:
+            changed += 1
+            if not args.dry_run:
+                con.execute(
+                    "UPDATE questions SET stem=?, opt_a=?, opt_b=?, opt_c=?,"
+                    " opt_d=?, solution=? WHERE id=?",
+                    (*new, qid),
+                )
+    if not args.dry_run:
+        con.commit()
+    # Report math health after the pass.
+    left = 0
+    for (stem, a, b, c, d, sol) in con.execute(
+        "SELECT stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions"
+    ):
+        blob = "".join(x or "" for x in (stem, a, b, c, d, sol))
+        if "$$" in blob or "\\matrix" in blob:
+            left += 1
+    print(f"rows: {len(rows)}, changed: {changed}, rows still with $$/\\matrix: {left}")
+    con.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/import_examgoal.py

@@ -61,7 +61,13 @@ def sanitize(raw: str) -> str:
     # Collapse whitespace runs but keep single spaces inside math.
     text = re.sub(r"[ \t\xa0]+", " ", text)
     text = re.sub(r"\n\s*\n+", "\n", text)
-    return text.strip()
+    text = text.strip()
+    # Normalize math delimiters ($$/$, \matrix) so KaTeX renders inline.
+    # Shared with scripts/fix_math.py — keep the two in sync.
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    from fix_math import fix_field
+
+    return fix_field(text)
 
 
 def shift_of(paper_title: str, exam_date) -> str: