Diagram pipeline: import 1,856 image MCQs with local figures, sanitized img allowlist, WebP shrink

.gitignore

@@ -3,3 +3,6 @@
 whaleprep.db*
 *.pid
 *.log
+
+# Question figures: 200MB of WebP/SVG, deployed by rsync not git
+/static/qimg/

scripts/import_with_images.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Import the image-bearing MCQs that scripts/import_examgoal.py skipped.
+
+Downloads each question figure from the examgoal CDN into static/qimg/
+and rewrites <img src> to the local path, so the bank stays self-contained.
+
+Usage: python3 scripts/import_with_images.py DB_PATH [--pkl /tmp/db.pkl]
+Safe to re-run: dedupes on exact stem match, skips already-downloaded files.
+"""
+import argparse, hashlib, html, pickle, re, sqlite3, sys, urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "scripts"))
+sys.path.insert(0, str(ROOT / "scripts" / "examgoal_model"))
+from question import Question
+from chapter import Chapter
+from fix_math import fix_field
+
+ALLOWED = {"br", "p", "sup", "sub", "b", "strong", "i", "em", "u",
+           "ol", "ul", "li", "table", "tbody", "thead", "tr", "th", "td",
+           "span", "h3"}
+SUBJECT_FIX = {"mathematics": "Maths", "physics": "Physics", "chemistry": "Chemistry"}
+QIMG = ROOT / "static" / "qimg"
+
+class FixUnpickler(pickle.Unpickler):
+    def find_class(self, module, name):
+        if name == "Question":
+            return Question
+        if name == "Chapter":
+            return Chapter
+        return super().find_class(module, name)
+
+def title_case(slug: str) -> str:
+    return " ".join(w.capitalize() for w in slug.replace("-", " ").split())
+
+def shift_of(paper_title, exam_date) -> str:
+    t = str(paper_title or "")
+    low = t.lower()
+    if "evening" in low: return "S2"
+    if "morning" in low: return "S1"
+    return "S1"
+
+def url_hash(u: str) -> str:
+    return hashlib.sha1(u.encode()).hexdigest()[:16]
+
+def fetch(url: str, dest: Path, tries=3) -> bool:
+    if dest.exists() and dest.stat().st_size > 0:
+        return True
+    for _ in range(tries):
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
+            with urllib.request.urlopen(req, timeout=20) as r:
+                b = r.read()
+            if b:
+                dest.write_bytes(b)
+                return True
+        except Exception:
+            pass
+    return False
+
+def rewrite_images(qid: str, raw: str):
+    """Download each <img src>, rewrite to /static/qimg/<name>.png."""
+    ok_all = True
+    def repl(m):
+        nonlocal ok_all
+        url = html.unescape(m.group(1))
+        name = f"{qid[:24]}-{url_hash(url)}.png"
+        if fetch(url, QIMG / name):
+            return f'<img src="/static/qimg/{name}">'
+        ok_all = False
+        return ""
+    out = re.sub(r'<img[^>]*src="([^"]+)"[^>]*>', repl, raw)
+    return out, ok_all
+
+def sanitize_img(qid: str, raw: str):
+    """sanitize() + surviving img tags with local src only."""
+    if not raw:
+        return "", True
+    text = html.unescape(str(raw))
+    # unwrap <picture>: keep the <img> fallback inside, drop <source> variants
+    def unwrap_pic(m):
+        inner = m.group(0)
+        imgs = re.findall(r"<img[^>]*>", inner)
+        return imgs[0] if imgs else ""
+    text = re.sub(r"<picture.*?</picture>", unwrap_pic, text, flags=re.S | re.I)
+    text = re.sub(r"<style.*?</style>", "", text, flags=re.S | re.I)
+    text = re.sub(r"<colgroup.*?</colgroup>", "", text, flags=re.S | re.I)
+    text = re.sub(r"</?(?:col|colgroup)[^>]*>", "", text, flags=re.I)
+    text, ok = rewrite_images(qid, text)
+    def tag(m):
+        whole, closing, name = m.group(0), m.group(1), m.group(2).lower()
+        if name == "img":
+            # keep ONLY if src is already a local rewritten path
+            if 'src="/static/qimg/' in whole:
+                return whole
+            return ""
+        if name in ALLOWED:
+            return f"<{closing}{name}>"
+        return ""
+    text = re.sub(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)((?:\s[^>]*)?)>", tag, text)
+    text = re.sub(r"[ \t\xa0]+", " ", text)
+    text = re.sub(r"\n\s*\n+", "\n", text)
+    text = text.strip()
+    return fix_field(text), ok
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("db")
+    ap.add_argument("--pkl", default="/tmp/db.pkl")
+    args = ap.parse_args()
+
+    print(f"loading {args.pkl} ...", flush=True)
+    with open(args.pkl, "rb") as f:
+        data = FixUnpickler(f).load()
+    QIMG.mkdir(parents=True, exist_ok=True)
+
+    con = sqlite3.connect(args.db)
+    cur = con.cursor()
+    existing = {r[0] for r in cur.execute("SELECT stem FROM questions")}
+    print(f"existing stems: {len(existing)}", flush=True)
+
+    def ensure_ids(subject, chapter, subtopic):
+        cur.execute("SELECT id FROM chapters WHERE subject=? AND name=?", (subject, chapter))
+        row = cur.fetchone()
+        if row:
+            ch_id = row[0]
+        else:
+            cur.execute("INSERT INTO chapters(subject,name) VALUES(?,?)", (subject, chapter))
+            ch_id = cur.lastrowid
+        cur.execute("SELECT id FROM subtopics WHERE chapter_id=? AND name=?", (ch_id, subtopic))
+        row = cur.fetchone()
+        if row:
+            return ch_id, row[0]
+        cur.execute("INSERT INTO subtopics(chapter_id,name) VALUES(?,?)", (ch_id, subtopic))
+        return ch_id, cur.lastrowid
+
+    imported = skipped = dl_fail = opt_img = 0
+    diff_map = {"easy": "Easy", "medium": "Medium", "hard": "Hard"}
+    for _key, chap in data.items():
+        for q in chap.question_dict.values():
+            d = q.__dict__
+            if d.get("type") != "mcq":
+                continue
+            co = d.get("correct_options") or []
+            if len(co) != 1 or co[0] not in ("A", "B", "C", "D"):
+                continue
+            stem_raw = d.get("question") or ""
+            opts_raw = [o for o in (d.get("options") or []) if isinstance(o, dict)]
+            opt_txt = " ".join((o.get("content") or "") for o in opts_raw)
+            if "<img" not in stem_raw and "<img" not in opt_txt:
+                continue
+            if len([o for o in opts_raw if (o.get("content") or "").strip()]) != 4:
+                skipped += 1
+                continue
+            qid = str(d.get("question_id") or f"q{d.get('year','x')}")
+            stem, ok = sanitize_img(qid, stem_raw)
+            stem_has_img = "<img" in stem_raw
+            if not ok or (stem_has_img and "<img" not in stem):
+                dl_fail += 1
+                continue
+            # Options may carry images (organic structures, graphs): download
+            # and rewrite them exactly like stem figures.
+            opts = []
+            bad = False
+            for o in opts_raw[:4]:
+                t, ook = sanitize_img(qid, o.get("content") or "")
+                if not t or (("<img" in (o.get("content") or "")) and not ook):
+                    bad = True
+                    break
+                opts.append(t)
+            if bad or len(opts) != 4:
+                skipped += 1
+                continue
+            solution, _ok = sanitize_img(qid, d.get("explanation") or "")
+            if stem in existing:
+                skipped += 1
+                continue
+            subject = SUBJECT_FIX.get(str(getattr(chap, "parent_subject", "")).lower(),
+                                      str(getattr(chap, "parent_subject", "")).title())
+            chapter = title_case(str(d.get("chpaterGroup") or d.get("chapter") or "General"))
+            subtopic = title_case(str(d.get("topic") or d.get("chapter") or "General"))
+            ch_id, st_id = ensure_ids(subject, chapter, subtopic)
+            diff = diff_map.get(str(d.get("difficulty", "medium")).lower(), "Medium")
+            try:
+                year = int(d.get("year") or 0)
+            except (TypeError, ValueError):
+                year = 0
+            shift = shift_of(d.get("paperTitle", ""), d.get("examDate"))
+            cur.execute(
+                "INSERT INTO questions(subject,chapter_id,subtopic_id,qtype,"
+                "stem,opt_a,opt_b,opt_c,opt_d,answer,difficulty,year,shift,solution)"
+                " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                (subject, ch_id, st_id, "MCQ", stem, opts[0], opts[1], opts[2], opts[3],
+                 co[0], diff, year, shift, solution))
+            existing.add(stem)
+            imported += 1
+            if imported % 200 == 0:
+                print(f"  ... {imported} imported", flush=True)
+    con.commit()
+    n_img = len(list(QIMG.glob("*.png")))
+    print(f"imported {imported} image MCQs (skipped {skipped}, dl-fail {dl_fail}, "
+          f"opt-image {opt_img}); static/qimg holds {n_img} files", flush=True)
+    con.close()
+
+if __name__ == "__main__":
+    main()

scripts/shrink_qimg.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Convert static/qimg/*.png to resized WebP and update DB references.
+
+Question figures only need ~900px width for the exam card; WebP q80
+cuts the folder to about a fifth of its PNG size.
+
+Usage: python3 scripts/shrink_qimg.py DB_PATH
+"""
+import re
+import sqlite3
+import sys
+from pathlib import Path
+from PIL import Image
+
+ROOT = Path(__file__).resolve().parent.parent
+QIMG = ROOT / "static" / "qimg"
+
+def main():
+    db = sqlite3.connect(sys.argv[1])
+    cur = db.cursor()
+    converted = 0
+    for png in sorted(QIMG.glob("*.png")):
+        webp = png.with_suffix(".webp")
+        if webp.exists():
+            continue
+        try:
+            im = Image.open(png)
+            if im.mode in ("P", "RGBA", "LA"):
+                im = im.convert("RGB")
+            w, h = im.size
+            if w > 900:
+                im = im.resize((900, int(h * 900 / w)))
+            im.save(webp, "WEBP", quality=80)
+            png.unlink()
+            converted += 1
+        except Exception as e:
+            print(f"skip {png.name}: {e}")
+            continue
+    print(f"converted {converted} pngs to webp")
+    # rewrite DB references
+    n = 0
+    for (qid, stem, a, b, c, d, sol) in cur.execute(
+            "SELECT id, stem, opt_a, opt_b, opt_c, opt_d, solution FROM questions"):
+        vals = [stem, a, b, c, d, sol]
+        changed = False
+        for i, v in enumerate(vals):
+            if v and "/static/qimg/" in v and ".png" in v:
+                nv = re.sub(r"(/static/qimg/[A-Za-z0-9_-]+)\.png", r"\1.webp", v)
+                if nv != v:
+                    vals[i] = nv
+                    changed = True
+        if changed:
+            cur.execute("UPDATE questions SET stem=?, opt_a=?, opt_b=?, opt_c=?, opt_d=?, solution=? WHERE id=?",
+                        (vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], qid))
+            n += 1
+    db.commit()
+    print(f"updated {n} question rows")
+    db.close()
+
+if __name__ == "__main__":
+    main()

src/views.rs

@@ -25,8 +25,7 @@ pub fn rich(s: &str) -> String {
     const ALLOWED: &[&str] = &[
         "br", "p", "sup", "sub", "b", "strong", "i", "em", "u", "ol", "ul",
         "li", "table", "tbody", "thead", "tr", "th", "td", "span",
-    ];
+    ];    // A `<` only starts a tag when followed by a letter, `/`, `!` or `?`
-    // A `<` only starts a tag when followed by a letter, `/`, `!` or `?`
     // (i.e. `<p>`, `</p>`, `<!--`, `<?`). A `<` followed by a space, digit,
     // `=` or end-of-string is literal math text (`\(a < 0\)`) and must be
     // escaped, not parsed as a tag — otherwise the sanitizer eats the `<`
@@ -34,6 +33,18 @@ pub fn rich(s: &str) -> String {
     fn is_tag_start(after_lt: &str) -> bool {
         matches!(after_lt.chars().next(), Some('a'..='z' | 'A'..='Z' | '/' | '!' | '?'))
     }
+    // img is allowed only when its src points at our local question-image
+    // directory; remote URLs are dropped to keep the bank self-contained
+    // and close the sanitizer to outside hosts.
+    fn safe_img(inner: &str) -> bool {
+        let src_ok = inner
+            .split("src=\"")
+            .nth(1)
+            .and_then(|rest| rest.split('"').next())
+            .map(|src| src.starts_with("/static/qimg/"))
+            .unwrap_or(false);
+        src_ok && !inner.contains("on")
+    }
     let mut o = String::with_capacity(s.len());
     let mut rest = s;
     while let Some(idx) = rest.find('<') {
@@ -58,17 +69,26 @@ pub fn rich(s: &str) -> String {
                 None => (false, inner.split_whitespace().next().unwrap_or("")),
             };
             let name = name.trim_end_matches('/').to_ascii_lowercase();
-            let has_attrs = inner.contains(char::is_whitespace) && !inner.ends_with('/');
+            if name == "img" && !closing && safe_img(inner) {
-            if !has_attrs && ALLOWED.contains(&name.as_str()) {
+                // keep the sanitized local image tag verbatim
                 o.push('<');
-                if closing {
+                o.push_str(inner);
-                    o.push('/');
-                }
-                o.push_str(&name);
-                if !closing && inner.ends_with('/') {
-                    o.push('/');
-                }
                 o.push('>');
+            } else if name == "img" {
+                // remote or unsafe img: drop the tag, keep nothing
+            } else {
+                let has_attrs = inner.contains(char::is_whitespace) && !inner.ends_with('/');
+                if !has_attrs && ALLOWED.contains(&name.as_str()) {
+                    o.push('<');
+                    if closing {
+                        o.push('/');
+                    }
+                    o.push_str(&name);
+                    if !closing && inner.ends_with('/') {
+                        o.push('/');
+                    }
+                    o.push('>');
+                }
             }
             rest = &after[end + 1..];
         } else {
@@ -176,7 +196,18 @@ fn nav(ctx: &Ctx, user: Option<&str>) -> String {
 
 fn foot(ctx: &Ctx) -> String {
     format!(
-        r#"<footer><span>WhalePrep · JEE Main PYQ practice</span><span><a href="{b}/terms">terms</a> · <a href="{b}/privacy">privacy</a></span></footer></body></html>"#,
+        r#"<footer><span>WhalePrep · JEE Main PYQ practice</span><span><a href="{b}/terms">terms</a> · <a href="{b}/privacy">privacy</a></span></footer>
+<script>
+(function () {{
+  // Tap a question figure to enlarge it on small screens (ZenithPrep .qimg.zoom).
+  if (!matchMedia('(max-width: 720px)').matches) return;
+  document.addEventListener('click', function (e) {{
+    var img = e.target.closest('.qcard img, .stem img');
+    if (img) img.classList.toggle('zoom');
+  }});
+}})();
+</script>
+</body></html>"#,
         b = ctx.base
     )
 }
@@ -870,7 +901,7 @@ pub fn attempt(ctx: &Ctx, user: &str, attempt_id: &str, questions: &[Question],
                         .enumerate()
                         .map(|(k, _letter)| {
                             format!(
-                                "<p>({n}) {t}</p>",
+                                "<p class=\"opt-readout\">({n}) {t}</p>",
                                 n = k + 1,
                                 t = rich(&q.opts[k]),
                             )
@@ -1491,4 +1522,24 @@ mod rich_tests {
         let out = rich("a < b and c <");
         assert_eq!(out, "a &lt; b and c &lt;");
     }
+
+    #[test]
+    fn local_question_image_survives() {
+        let out = rich(r#"see figure <img src="/static/qimg/abc-123.png"> below"#);
+        assert!(out.contains(r#"<img src="/static/qimg/abc-123.png">"#), "local img must pass: {out}");
+    }
+
+    #[test]
+    fn remote_image_dropped() {
+        let out = rich(r#"x <img src="https://evil.example/x.png"> y"#);
+        assert!(!out.contains("<img"), "remote img must be dropped: {out}");
+        assert!(out.contains("x") && out.contains("y"), "text around it stays: {out}");
+    }
+
+    #[test]
+    fn img_with_event_handler_dropped() {
+        let out = rich(r#"<img src="/static/qimg/a.png" onerror="alert(1)">"#);
+        assert!(!out.contains("onerror"), "handlers must go: {out}");
+        assert!(!out.contains("<img"), "unsafe img must go entirely: {out}");
+    }
 }

static/style.css

@@ -1145,6 +1145,32 @@ footer {
   line-height: 1.7;
 }
 
+/* Question figures: local downloads in static/qimg (ZenithPrep .qimg style) */
+.qcard img,
+.stem img {
+  display: block;
+  max-width: 100%;
+  height: auto;
+  margin: 12px auto;
+  background: #ffffff;
+  border: 1px solid #e5e7eb;
+  border-radius: 8px;
+  padding: 8px;
+}
+
+@media (max-width: 720px) {
+  .qcard img,
+  .stem img {
+    cursor: zoom-in;
+  }
+  .qcard img.zoom,
+  .stem img.zoom {
+    max-width: none;
+    width: 220%;
+    cursor: zoom-out;
+  }
+}
+
 .qcard .katex {
   color: #111827;
 }
@@ -1212,6 +1238,29 @@ footer {
   line-height: 1.5;
 }
 
+/* Images inside an option (organic structures, graphs): shrink to fit the pill */
+.opt-text img {
+  display: inline-block;
+  max-height: 120px;
+  max-width: 100%;
+  height: auto;
+  vertical-align: middle;
+  background: #ffffff;
+  border: 1px solid #e5e7eb;
+  border-radius: 6px;
+  padding: 4px;
+  margin: 2px 0;
+}
+
+/* Same for option text read out inside the white question card */
+.qcard .opt-readout img {
+  display: inline-block;
+  max-height: 140px;
+  max-width: 100%;
+  vertical-align: middle;
+  margin: 4px 4px 4px 0;
+}
+
 .opt p {
   display: inline;
   margin: 0;