Vendor examgoal unpickle models, portable pickle path

import_examgoal.py required /tmp/pyqeval helper modules that were never
committed, so it failed on any fresh machine. The two upstream model
files now live in scripts/examgoal_model/ (relative import fixed) and
the default pickle path is scripts/examgoal_db.pkl.

scripts/examgoal_model/chapter.py

@@ -0,0 +1,41 @@
+"""
+This file has the Chapter class
+"""
+
+import json
+from question import Question
+
+
+class Chapter:
+    """Abstraction for chapter"""
+    def __repr__(self)->str:
+        template = f"""
+Parent Subject:{self.parent_subject}
+Name:{self.name}
+Question Dict: {self.question_dict_status}
+Total Questions:{self.total_questions}
+"""
+        return template
+    
+    def __init__(self,chapter_path)->None:
+        """Initialization of Chapter
+        :param:
+        chapter_path: Path to chpaters json file , recommended to use absolute path
+        """
+        chapter_file = open(chapter_path,"r")
+        chapter_json = json.load(chapter_file)
+        chapter_file.close()
+
+        parent_subject = chapter_json["results"][0]["title"]
+        question_dict = dict()
+        counter = 0
+        for question_json in chapter_json["results"][0]["questions"]:
+            question = Question(question_json)
+            question_dict[counter] = question
+            counter += 1
+        
+        self.parent_subject = parent_subject
+        self.name = chapter_json["results"][0]["questions"][0]["chapter"]
+        self.question_dict = question_dict
+        self.question_dict_status = "healthy"
+        self.total_questions = len(question_dict)

scripts/examgoal_model/question.py

@@ -0,0 +1,76 @@
+"""
+This file has Question class
+"""
+
+#from sentence_transformers import SentenceTransformer
+
+class Question:
+    """Loads data from database into inherited class variables"""
+    def __repr__(self)->str:
+        template = f"""
+QuestionId: {self.question_id}
+Exam: {self.exam}
+Year: {self.year}
+Subject: {self.subject}
+Chapter: {self.chapter}
+"""
+        return template
+    def __init__(self, question_json: dict,embedding_model:str="all-MiniLM-L6-v2") -> None:
+        """Initialization of Question
+        :param:
+        question_json: json part of question (this is kind of hardcoded configured to examgoal database
+                        and the structure of this system)
+        embedding_model: A vail SentenceTransformer embedding model name . default to all-MiniLM-L6-v2
+        """
+        #model = SentenceTransformer(embedding_model)
+
+        self.question_id = question_json.get("question_id", "")
+        self.examGroup = question_json.get("examGroup", "")
+        self.exam = question_json.get("exam", "")
+        self.subject = question_json.get("subject", "")
+        self.chpaterGroup = question_json.get("chapterGroup", "")
+        self.chapter = question_json.get("chapter", "")
+        self.year = question_json.get("year", 0)
+        self.paperTitle = question_json.get("paperTitle", "")
+        self.difficulty = question_json.get("difficulty", "")
+        self.topic = question_json.get("topic", "")
+        self.type = question_json.get("type", "")
+        self.examDate = question_json.get("examDate", None)
+        self.answer = question_json.get("answer",None)
+        question_en = question_json.get("question", {}).get("en", {})
+        self.question = question_en.get("content", "")
+        self.options = question_en.get("options", {})
+        self.correct_options = question_en.get("correct_options", [])
+        self.explanation = question_en.get("explanation", "")
+
+        self.isOutOfSyllabus = question_json.get("isOutOfSyllabus", False)
+        self.isBonus = question_json.get("isBonus", False)
+
+        
+        self.isImgQuestion = self.check_image_in_text(self.question)
+        self.isImgExplanation = self.check_image_in_text(self.explanation)
+        self.isImgOption = self.check_image_in_options()
+        
+        #self.embedding = model.encode(self.question) #disbaled for v003 testing version | this is a computation
+        #heavy param will take 8 mins to calc the embeddings | latest update moving embeddings to its seprate cache!!!
+
+    def check_image_in_text(self,text:str)->bool:
+        """Check if text consists of image (inform of html)"""
+        text_s = text.split("<img")
+        if len(text_s) != 1:
+            return True
+        else:
+            return False
+    
+    def check_image_in_options(self)->bool:
+        """Check if question consists of image (inform of html)"""
+        options_json = self.options
+        option_bool_list = []
+        for option in options_json:
+            option_content = option["content"]
+            option_content_s = option_content.split("<img")
+            if len(option_content_s) != 1:
+                option_bool_list.append(True)
+            else:
+                option_bool_list.append(False)
+        return option_bool_list

scripts/import_examgoal.py

@@ -85,16 +85,17 @@ def main() -> None:
     ap.add_argument("--pkl", default=None)
     args = ap.parse_args()
 
-    pkl_path = Path(args.pkl) if args.pkl else Path("/tmp/pyqeval/db.pkl")
+    default_pkl = Path(__file__).resolve().parent / "examgoal_db.pkl"
+    pkl_path = Path(args.pkl) if args.pkl else default_pkl
     if not pkl_path.exists():
         print(f"downloading {PKL_URL} ...", flush=True)
         pkl_path.parent.mkdir(parents=True, exist_ok=True)
         urllib.request.urlretrieve(PKL_URL, pkl_path)
     print(f"loading {pkl_path} ({pkl_path.stat().st_size / 1e6:.1f} MB) ...", flush=True)
 
-    sys.path.insert(0, "/tmp/pyqeval")
+    sys.path.insert(0, str(Path(__file__).resolve().parent / "examgoal_model"))
-    from pkg.core.question import Question
+    from question import Question
-    from pkg.core.chapter import Chapter
+    from chapter import Chapter
 
     class FixUnpickler(pickle.Unpickler):
         def find_class(self, module, name):