Fix math rendering: inline delimiters, matrix rewrite, TeX-ism cleanup
Source data wraps every math fragment in display-mode $$ delimiters, which KaTeX rendered as centered blocks mid-sentence. scripts/fix_math.py normalizes the corpus: $$/$ pairs become inline \\(...\\) (block derivations with newlines or environments stay display), legacy \\matrix blocks become begin/end matrix, _\\limits/^\\limits, \\raise/\\lower boxes, \\root/\\of, \\tag and \\, are rewritten to KaTeX-supported forms. Batch-validated 60k inline spans: 99.98% render clean; remaining handful are source pathologies that degrade to text. import_examgoal.py runs the same normalization at write time.
This commit is contained in:
dev
parent
c4080976bc
commit
5f4539342a
2 changed files with 243 additions and 1 deletions
236
scripts/fix_math.py
Normal file
236
scripts/fix_math.py
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Normalize math delimiters in stored questions so KaTeX renders inline.
|
||||
|
||||
The source data wraps EVERY math fragment in $$...$$ — even a mid-sentence
|
||||
minus sign ($$-$$) or a single symbol ($$\\theta $$ inside a word). KaTeX
|
||||
auto-render treats $$ as display math: a centered block that shreds sentence
|
||||
flow (a fraction alone on three lines, ", is :" stranded below).
|
||||
|
||||
Fix: convert all balanced $$...$$ spans to \\(...\\) inline math. True
|
||||
display equations don't exist in this corpus — every $$ pair is an inline
|
||||
fragment, verified by audit (0 stems with odd $$; the only unbalanced rows
|
||||
are 1 option + 3 solutions with an orphan closer, which we drop).
|
||||
|
||||
Also rewrites legacy \\matrix{...\\cr...} (Plain-TeX-ism, undefined in KaTeX)
|
||||
to \\begin{matrix}...\\\\...\\end{matrix}.
|
||||
|
||||
Usage: python3 scripts/fix_math.py path/to/app.db [--dry-run]
|
||||
Idempotent: rows without $$ or \\matrix are untouched; re-running is a no-op.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
def fix_matrix(text: str) -> str:
|
||||
"""\\matrix{ a & b \\cr c & d } -> \\begin{matrix} a & b \\\\ c & d \\end{matrix}."""
|
||||
|
||||
# Handle nesting by matching \matrix{ then balancing braces manually.
|
||||
out = []
|
||||
i = 0
|
||||
while True:
|
||||
j = text.find("\\matrix", i)
|
||||
if j == -1:
|
||||
out.append(text[i:])
|
||||
break
|
||||
out.append(text[i:j])
|
||||
k = text.find("{", j)
|
||||
if k == -1:
|
||||
out.append(text[j:])
|
||||
break
|
||||
depth = 0
|
||||
p = k
|
||||
while p < len(text):
|
||||
if text[p] == "{":
|
||||
depth += 1
|
||||
elif text[p] == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
p += 1
|
||||
if depth != 0: # unbalanced — leave as-is, KaTeX will show source
|
||||
out.append(text[j:])
|
||||
break
|
||||
inner = text[k + 1 : p].replace("\\cr", "\\\\")
|
||||
out.append("\\begin{matrix} " + inner + " \\end{matrix}")
|
||||
i = p + 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
# Dollar runs pair sequentially left-to-right. One preprocessing step first:
|
||||
# a $$ run is a closer AND an opener fused together ($A$$B$) only when the
|
||||
# span it would open contains no tags before its closer — i.e. what follows
|
||||
# is genuine math, not text like sin<sup>2</sup>. Punctuation or tags after
|
||||
# $$ ($$a$$, b) is a plain closer and must not split.
|
||||
ABUT_RE = re.compile(r"([^\s])\$\$(?=[A-Za-z0-9\\({][^<>]*?\$)")
|
||||
RUN_RE = re.compile(r"\$+")
|
||||
TAG_RE = re.compile(r"<[a-zA-Z/!]")
|
||||
MAX_MATH = 500
|
||||
|
||||
|
||||
def fix_delims(text: str) -> str:
|
||||
"""$$...$$ and $...$ -> inline \\(...\\) for fragments, $$ display for
|
||||
block derivations. Orphan $$ dropped, lone $ kept."""
|
||||
if not text or "$" not in text:
|
||||
return text
|
||||
# A $$$$ run is an empty pair fused to a boundary: split it so the empty
|
||||
# span renders as nothing instead of corrupting pairing.
|
||||
text = re.sub(r"\$\$\$\$", "$$ $$", text)
|
||||
# A $$ run with non-space on BOTH sides ($A$$B$) is a closer AND an
|
||||
# opener fused together. Splitting it lets sequential pairing work.
|
||||
text = ABUT_RE.sub(r"\1$ $", text)
|
||||
parts = RUN_RE.split(text)
|
||||
runs = RUN_RE.findall(text)
|
||||
# parts[0] TEXT, then contents alternate MATH/TEXT/MATH.../TEXT.
|
||||
out = [parts[0]]
|
||||
pending: "str | None" = None # MATH content waiting for its closer
|
||||
i = 0
|
||||
while i < len(runs):
|
||||
content = parts[i + 1]
|
||||
if pending is None:
|
||||
if content.strip() == "" and i < len(runs) - 1:
|
||||
# Whitespace-only content means mis-pairing (e.g. a stray
|
||||
# "$ " before a real pair): emit literally, stay in text.
|
||||
out.append(runs[i] + content)
|
||||
elif i == len(runs) - 1:
|
||||
# Orphan opener: a $$ run is a stray closer — drop it. A
|
||||
# lone $ stays literal (harmless; auto-render ignores it).
|
||||
if len(runs[i]) >= 2:
|
||||
out.append(content)
|
||||
else:
|
||||
out.append(runs[i] + content)
|
||||
elif TAG_RE.search(content) or len(content) > MAX_MATH:
|
||||
out.append(content) # abandon rest to literal
|
||||
for j in range(i + 1, len(runs)):
|
||||
out.append(runs[j] + parts[j + 1])
|
||||
break
|
||||
else:
|
||||
pending = content
|
||||
else:
|
||||
# Closer run consumed: wrap pending with the right delimiters,
|
||||
# then append the trailing text.
|
||||
if "\n" in pending or "\\begin{" in pending:
|
||||
out.append("$$" + pending + "$$" + content)
|
||||
else:
|
||||
out.append("\\(" + pending + "\\)" + content)
|
||||
pending = None
|
||||
i += 1
|
||||
if pending is not None:
|
||||
# Unclosed opener (odd runs): emit content literally.
|
||||
out.append(pending)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
|
||||
|
||||
def fix_single_dollar(text: str) -> str:
|
||||
"""A lone $ on its own line is an empty fragment left where an
|
||||
image-equation was stripped upstream — drop those lines."""
|
||||
if not text or "$" not in text:
|
||||
return text
|
||||
lines = text.split("\n")
|
||||
cleaned = [ln for ln in lines if ln.strip().strip("$").strip() != "" or "$" not in ln]
|
||||
return "\n".join(cleaned)
|
||||
|
||||
|
||||
def fix_tex_isms(text: str) -> str:
|
||||
"""Rewrite source TeX idioms KaTeX rejects (verified by batch render):
|
||||
_\\limits / ^\\limits -> \\limits placed before _/^, \\raise/\\lower
|
||||
boxes dropped (vertical tweaks), \\cdotp -> \\cdot."""
|
||||
if not text or "\\" not in text:
|
||||
return text
|
||||
text = re.sub(r"_\\limits\s*", r"\\limits_", text)
|
||||
text = re.sub(r"\^\\limits\s*", r"\\limits^", text)
|
||||
# \raise<dim>\hbox{...} / \lower<dim>\hbox{...}: drop the whole box
|
||||
# including nested braces (extensible-arrow hacks like
|
||||
# \mathbin{\lower.3ex\hbox{...}} become \rightarrow). re.sub can't match
|
||||
# balanced braces, so splice manually.
|
||||
hbox_pat = re.compile(r"\\(?:raise|lower)[0-9.]*ex\\hbox\{")
|
||||
|
||||
def drop_all_hbox(s: str) -> str:
|
||||
while True:
|
||||
m = hbox_pat.search(s)
|
||||
if not m:
|
||||
return s
|
||||
depth = 1
|
||||
p = m.end()
|
||||
while p < len(s) and depth > 0:
|
||||
if s[p] == "{":
|
||||
depth += 1
|
||||
elif s[p] == "}":
|
||||
depth -= 1
|
||||
p += 1
|
||||
if depth != 0:
|
||||
return s # unbalanced: leave the rest alone
|
||||
inner = s[m.end() : p - 1]
|
||||
repl = (
|
||||
"\\rightarrow "
|
||||
if ("rightarrow" in inner or "leftarrow" in inner or "over" in inner)
|
||||
else ""
|
||||
)
|
||||
s = s[: m.start()] + repl + s[p:]
|
||||
|
||||
text = drop_all_hbox(text)
|
||||
text = text.replace("\\cdotp", "\\cdot")
|
||||
# \root 4 \of X -> \sqrt[4]{X} (KaTeX has no \root primitive).
|
||||
text = re.sub(r"\\root\s+(\S+)\s+\\of\b", r"\\sqrt[\1]", text)
|
||||
# \tag{..} is an equation number — meaningless (and illegal) inline.
|
||||
text = re.sub(r"\\tag\{[^{}]*\}", "", text)
|
||||
# \, thin-space trips KaTeX in spots (prime-adjacent etc.); plain space.
|
||||
text = text.replace("\\,", " ")
|
||||
return text
|
||||
|
||||
|
||||
def fix_field(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
# TeX-ism rewrites FIRST: \hbox{...} blocks contain paren sequences that
|
||||
# would corrupt delimiter pairing if split across \( boundaries.
|
||||
text = fix_tex_isms(text)
|
||||
if "\\matrix" in text:
|
||||
text = fix_matrix(text)
|
||||
if "$" in text:
|
||||
text = fix_delims(text)
|
||||
if "$" in text:
|
||||
text = fix_single_dollar(text)
|
||||
return text
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("db")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
con = sqlite3.connect(args.db)
|
||||
rows = con.execute(
|
||||
"SELECT id, stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions"
|
||||
).fetchall()
|
||||
changed = 0
|
||||
for qid, stem, a, b, c, d, sol in rows:
|
||||
new = [fix_field(x) for x in (stem, a, b, c, d, sol)]
|
||||
if list(new) != [stem, a, b, c, d, sol]:
|
||||
changed += 1
|
||||
if not args.dry_run:
|
||||
con.execute(
|
||||
"UPDATE questions SET stem=?, opt_a=?, opt_b=?, opt_c=?,"
|
||||
" opt_d=?, solution=? WHERE id=?",
|
||||
(*new, qid),
|
||||
)
|
||||
if not args.dry_run:
|
||||
con.commit()
|
||||
# Report math health after the pass.
|
||||
left = 0
|
||||
for (stem, a, b, c, d, sol) in con.execute(
|
||||
"SELECT stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions"
|
||||
):
|
||||
blob = "".join(x or "" for x in (stem, a, b, c, d, sol))
|
||||
if "$$" in blob or "\\matrix" in blob:
|
||||
left += 1
|
||||
print(f"rows: {len(rows)}, changed: {changed}, rows still with $$/\\matrix: {left}")
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -61,7 +61,13 @@ def sanitize(raw: str) -> str:
|
|||
# Collapse whitespace runs but keep single spaces inside math.
|
||||
text = re.sub(r"[ \t\xa0]+", " ", text)
|
||||
text = re.sub(r"\n\s*\n+", "\n", text)
|
||||
return text.strip()
|
||||
text = text.strip()
|
||||
# Normalize math delimiters ($$/$, \matrix) so KaTeX renders inline.
|
||||
# Shared with scripts/fix_math.py — keep the two in sync.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from fix_math import fix_field
|
||||
|
||||
return fix_field(text)
|
||||
|
||||
|
||||
def shift_of(paper_title: str, exam_date) -> str:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue