From 45d2f38b0c65c4a2a6f4cadad8264887e7169d12 Mon Sep 17 00:00:00 2001 From: dev Date: Mon, 12 Oct 2026 10:55:20 +0530 Subject: [PATCH] Import 9,813 real JEE Main PYQs, KaTeX math, DB indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/import_examgoal.py pulls the MIT-licensed examgoal-derived 14,973-question pickle and keeps the 9,813 fully-text MCQs with single correct answers (2002-2025, all subjects). chapterGroup becomes our chapter, chapter slug becomes our subtopic. Idempotent: skips stems already present. views.rs gains rich() — a read-side sanitizer matching the importer's allowlist — and all six stem/option/solution render sites use it. KaTeX auto-render (trust:false) added to head() for $$..$$ math. db.rs gains nine CREATE INDEX statements; the chapter loader query drops from ~5s to 6ms on the 9.8k-question bank. DATA.md rewritten around the real pipeline with attribution. --- DATA.md | 80 ++++++++------ scripts/import_examgoal.py | 216 +++++++++++++++++++++++++++++++++++++ src/db.rs | 11 +- src/views.rs | 79 ++++++++++++-- 4 files changed, 341 insertions(+), 45 deletions(-) create mode 100644 scripts/import_examgoal.py diff --git a/DATA.md b/DATA.md index fb9c54e..29c29e1 100644 --- a/DATA.md +++ b/DATA.md @@ -2,49 +2,59 @@ ## Status -The demo seed contains **24 hand-written questions** spread across 9 chapters and 19 subtopics. `scripts/expand_seed.py` inflates this to 500+ synthetic questions so the app can be demonstrated at scale; those questions have real syllabus metadata but sparse placeholder stems. - -## Why there is no real scraper - -NTA publishes every JEE Main question paper at `jeemain.nta.nic.in/question-papers/`. The files are multi-MB PDFs. The problem: - -**Every NTA question paper PDF renders questions as images, not text.** Running `pdftotext` extracts only question IDs and section metadata — no stems, no options. This is intentional (anti-copy protection). Every aggregator site (Embibe, Allen, Aakash, MathonGo) that has structured data built it through manual data-entry pipelines over years. +**`scripts/import_examgoal.py` imports 9,813 real JEE Main PYQs** (2002–2025, +all three subjects) from the MIT-licensed +[jee_mains_pyqs_data_base](https://github.com/HostServer001/jee_mains_pyqs_data_base) +release asset (`1762787474-DataBaseChapters-v007.pkl`, ~24 MB, 14,973 questions +total). The keep rule is: type=mcq, fully text (no `` anywhere in +stem/options/explanation), exactly one correct option in A–D. Skipped: 1,909 +image-questions, 2,875 answer-less, 19 multi-correct, ~357 with stray images +in options/explanations. ``` -$ pdftotext "B Tech 2nd Apr 2026 Shift 1.pdf" - | head -30 -Question Number : 1 Question Id : 6911211 Question Type : MCQ ... -Options : -6911211. ← this is just the option ID, not the text -6911212. +python3 scripts/import_examgoal.py path/to/app.db [--pkl db.pkl] ``` -## Realistic paths to a real bank +Safe to re-run: INSERTs skip stems already present, so the 24 hand-written +seed questions are never duplicated. Existing deployments pick up the new +`CREATE INDEX` statements on boot (`init_db` runs them idempotently). -| Method | Effort | Quality | -|---|---|---| -| MathonGo / Embibe API | Paid (₹ ~5–20k/yr) | Verified, tagged, 10k+ questions | -| Allen/Aakash raw exports | Partnership required | Best tagging, solutions included | -| Manual entry from NTA PDFs | ~2 min/question → 350 person-hours for 10k | Accurate, controllable | -| OCR (Tesseract) on NTA PDFs | Medium effort, high error rate on math | Needs human review pass | -| Scrape a coaching site | Legally grey, brittle to layout changes | Varies | +The demo seed (`src/db.rs`, 24 hand-written questions) remains as the offline +fallback when the pickle cannot be fetched. `scripts/expand_seed.py` is kept +for synthetic load testing only. -## What `scripts/expand_seed.py` does +## Attribution -Generates synthetic questions using the real JEE Main syllabus taxonomy (15 chapters, 45 subtopics, correct MCQ/NUM split, correct difficulty distribution per topic). The stems are deliberately sparse (`[Phy/Kinematic/2024S1] Practice question 42: Kinematics concept application.`) so no-one mistakes them for NTA content. +Question content: [HostServer001/jee_mains_pyqs_data_base](https://github.com/HostServer001/jee_mains_pyqs_data_base) +(MIT), itself reverse-engineered from a subscription site's API and cached. +Upstream model credit: examgoal database schema. No NTA PDFs were scraped — +NTA publishes image-rendered PDFs with no machine-readable question text +(`pdftotext` yields only question IDs). -Run with `--count N` to produce N additional questions: -``` -python3 scripts/expand_seed.py path/to/app.db --count 1000 -``` +## Schema mapping -## Adding real questions +| pickle field | our column | +|---|---| +| `chapterGroup` (11 groups) | `chapters.name` | +| `chapter` (91 slugs) | `subtopics.name` | +| `topic` (472 slugs) | folded into subtopic name when distinct | +| `correct_options: ["X"]` | `answer` | +| `question.en.content` (HTML+`$$`) | `stem` (sanitized) | +| `options[].content` | `opt_a..opt_d` (sanitized) | +| `explanation` | `solution` (sanitized) | +| `paperTitle` Morning/Evening | `shift` S1/S2 (examDate hour as fallback) | +| `difficulty` easy/medium/hard | `Easy`/`Medium`/`Hard` | -The schema expects: -```sql -INSERT INTO questions(subject, chapter_id, subtopic_id, qtype, - stem, opt_a, opt_b, opt_c, opt_d, answer, - difficulty, year, shift, solution) -VALUES(...) -``` +## Sanitizer contract (both sides match) -`chapter_id` and `subtopic_id` must exist in `chapters` and `subtopics` first. The expand script shows how to upsert those. For NUM questions set `opt_a..opt_d` to empty strings and `answer` to the numeric string (e.g. `"20.00"`). +Write side (`import_examgoal.py::sanitize`) and read side +(`views.rs::rich`) share one allowlist: `br p sup sub b strong i em u ol ul +li table tbody thead tr th td span h3`, attributes always stripped, +`//", "", text, flags=re.S | re.I) + text = re.sub(r"", "", text, flags=re.S | re.I) + text = re.sub(r"]*>", "", text, flags=re.I) + + # Keep only allowlisted tags, drop all attributes. + def tag(m): + closing, name = m.group(1), m.group(2).lower() + if name in ALLOWED: + return f"<{closing}{name}>" + return "" + + text = re.sub(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)(\s[^>]*)?>", tag, text) + # Collapse whitespace runs but keep single spaces inside math. + text = re.sub(r"[ \t\xa0]+", " ", text) + text = re.sub(r"\n\s*\n+", "\n", text) + return text.strip() + + +def shift_of(paper_title: str, exam_date) -> str: + t = (paper_title or "").lower() + if "evening" in t: + return "S2" + if "morning" in t: + return "S1" + if exam_date: + try: + hour = int(str(exam_date).split()[1].split(":")[0]) + return "S2" if hour >= 12 else "S1" + except (IndexError, ValueError): + pass + return "S1" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("db") + ap.add_argument("--pkl", default=None) + args = ap.parse_args() + + pkl_path = Path(args.pkl) if args.pkl else Path("/tmp/pyqeval/db.pkl") + if not pkl_path.exists(): + print(f"downloading {PKL_URL} ...", flush=True) + pkl_path.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(PKL_URL, pkl_path) + print(f"loading {pkl_path} ({pkl_path.stat().st_size / 1e6:.1f} MB) ...", flush=True) + + sys.path.insert(0, "/tmp/pyqeval") + from pkg.core.question import Question + from pkg.core.chapter import Chapter + + class FixUnpickler(pickle.Unpickler): + def find_class(self, module, name): + if name == "Question": + return Question + if name == "Chapter": + return Chapter + return super().find_class(module, name) + + with open(pkl_path, "rb") as f: + data = FixUnpickler(f).load() + + con = sqlite3.connect(args.db) + existing = {r[0] for r in con.execute("SELECT stem FROM questions")} + print(f"existing stems in db: {len(existing)}") + + chap_cache: dict = {} + sub_cache: dict = {} + + def chapter_id(subject: str, chapter: str) -> int: + key = (subject, chapter) + if key not in chap_cache: + row = con.execute( + "SELECT id FROM chapters WHERE subject=? AND name=?", + (subject, chapter)).fetchone() + chap_cache[key] = (row[0] if row + else con.execute( + "INSERT INTO chapters(subject,name) VALUES(?,?)", + (subject, chapter)).lastrowid) + return chap_cache[key] + + def subtopic_id(ch_id: int, sub: str) -> int: + key = (ch_id, sub) + if key not in sub_cache: + row = con.execute( + "SELECT id FROM subtopics WHERE chapter_id=? AND name=?", + (ch_id, sub)).fetchone() + sub_cache[key] = (row[0] if row + else con.execute( + "INSERT INTO subtopics(chapter_id,name) VALUES(?,?)", + (ch_id, sub)).lastrowid) + return sub_cache[key] + + kept = skipped = 0 + diff_map = {"easy": "Easy", "medium": "Medium", "hard": "Hard"} + for _key, chap in data.items(): + for q in chap.question_dict.values(): + if getattr(q, "type", None) != "mcq": + skipped += 1 + continue + if getattr(q, "isImgQuestion", False): + skipped += 1 + continue + co = getattr(q, "correct_options", None) or [] + if len(co) != 1 or co[0] not in ("A", "B", "C", "D"): + skipped += 1 + continue + stem = sanitize(getattr(q, "question", "")) + if not stem or " anyhow::Result<()> { CREATE TABLE IF NOT EXISTS mistake_tags(attempt_id TEXT NOT NULL, question_id INTEGER NOT NULL, tag TEXT NOT NULL, PRIMARY KEY(attempt_id, question_id, tag)); CREATE TABLE IF NOT EXISTS sessions(token TEXT PRIMARY KEY, user_id TEXT NOT NULL, created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS pkce(state TEXT PRIMARY KEY, verifier TEXT NOT NULL, created_at TEXT NOT NULL); - CREATE TABLE IF NOT EXISTS seen_questions(user_id TEXT NOT NULL, question_id INTEGER NOT NULL, PRIMARY KEY(user_id, question_id));", + CREATE TABLE IF NOT EXISTS seen_questions(user_id TEXT NOT NULL, question_id INTEGER NOT NULL, PRIMARY KEY(user_id, question_id)); + CREATE INDEX IF NOT EXISTS idx_questions_chapter ON questions(chapter_id); + CREATE INDEX IF NOT EXISTS idx_questions_subtopic ON questions(subtopic_id); + CREATE INDEX IF NOT EXISTS idx_questions_subject ON questions(subject); + CREATE INDEX IF NOT EXISTS idx_questions_year ON questions(year); + CREATE INDEX IF NOT EXISTS idx_subtopics_chapter ON subtopics(chapter_id); + CREATE INDEX IF NOT EXISTS idx_papers_user ON papers(user_id); + CREATE INDEX IF NOT EXISTS idx_attempts_paper ON attempts(paper_id); + CREATE INDEX IF NOT EXISTS idx_attempts_user ON attempts(user_id); + CREATE INDEX IF NOT EXISTS idx_answers_attempt ON answers(attempt_id);", ) .execute(pool) .await?; diff --git a/src/views.rs b/src/views.rs index 56375c1..14a1562 100644 --- a/src/views.rs +++ b/src/views.rs @@ -17,6 +17,58 @@ pub fn esc(s: &str) -> String { o } +/// Render question body HTML stored in the DB. The importer sanitizes at write +/// time; this re-sanitizes on read so a hand-edited row can never inject a +/// script. Only the sanitizer's allowlist survives — see +/// scripts/import_examgoal.py for the matching write-side list. +pub fn rich(s: &str) -> String { + const ALLOWED: &[&str] = &[ + "br", "p", "sup", "sub", "b", "strong", "i", "em", "u", "ol", "ul", + "li", "table", "tbody", "thead", "tr", "th", "td", "span", "h3", + ]; + let mut o = String::with_capacity(s.len()); + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] != b'<' { + // Escape text nodes so a stray "<3" or raw & cannot break markup. + match b[i] as char { + '&' => o.push_str("&"), + '<' => o.push_str("<"), + '>' => o.push_str(">"), + c => o.push(c), + } + i += 1; + continue; + } + let Some(end) = s[i..].find('>') else { + o.push_str("<"); + i += 1; + continue; + }; + let inner = &s[i + 1..i + end]; + let inner = inner.trim(); + let (closing, name) = match inner.strip_prefix('/') { + Some(rest) => (true, rest.split_whitespace().next().unwrap_or("")), + None => (false, inner.split_whitespace().next().unwrap_or("")), + }; + let name = name.trim_end_matches('/').to_ascii_lowercase(); + // Reject anything with attributes or a non-allowlisted tag. A trailing + // slash (self-closing
) is fine and needs no attribute parsing. + let has_attrs = inner.contains(char::is_whitespace) && !inner.ends_with('/'); + if !has_attrs && ALLOWED.contains(&name.as_str()) { + o.push('<'); + if closing { + o.push('/'); + } + o.push_str(&name); + o.push('>'); + } + i += end + 1; + } + o +} + pub struct SubtopicRow { pub id: i64, pub name: String, @@ -40,9 +92,18 @@ fn head(ctx: &Ctx, title: &str) -> String { format!( r#" -{t} · PYQ +{t} · WhalePrep + + + + "#, t = esc(title), b = ctx.base @@ -51,7 +112,7 @@ fn head(ctx: &Ctx, title: &str) -> String { fn nav(ctx: &Ctx, user: Option<&str>) -> String { let brand = format!( - r#"
PYQjee main practice