analyzer + other-stream courses

app/services/classifier.py

@@ -22,7 +22,7 @@ def norm_title_key(title: str) -> str:
 
 # Short, hand-written synonyms (collapsed course name -> aliases). Cheap to extend.
 ALIASES: dict[str, list[str]] = {
-    "mathematicsfornaturalsciences": ["calculus", "natural math", "math natural"],
+    "mathematicsfornaturalsciences": ["ሒሳብ", "ሂሳብ", "calculus", "natural math", "math natural"],
     "appliedmathematicsi": ["applied math", "applied maths", "applied mathematics"],
     "mathematicsforsocialsciences": ["social math", "math social", "maths social"],
     "generalphysics": ["physics", "phys", "mechanics", "ፊዚክስ"],
@@ -31,18 +31,18 @@ ALIASES: dict[str, list[str]] = {
     "communicativeenglishlanguageskillsi": ["english", "eng", "flen", "comm english", "እንግሊዝኛ"],
     "communicativeenglishlanguageskillsii": ["english", "eng", "flen", "comm english", "እንግሊዝኛ"],
     "generalpsychology": ["psychology", "psych", "psys", "ሳይኮሎጂ"],
-    "criticalthinking": ["logic", "loct", "logic and critical thinking"],
+    "criticalthinking": ["አመክንዮ", "logic", "loct", "logic and critical thinking"],
     "geographyofethiopiaandthehorn": ["geography", "geo", "gees", "ጂኦግራፊ"],
     "historyofethiopiaandthehorn": ["history", "hist", "ታሪክ"],
-    "socialanthropology": ["anthropology", "anth"],
-    "entrepreneurship": ["entrepreneur", "entre", "mgmt"],
+    "socialanthropology": ["አንትሮፖሎጂ", "anthropology", "anth"],
+    "entrepreneurship": ["ስራ ፈጠራ", "ሥራ ፈጠራ", "entrepreneur", "entre", "mgmt"],
     "introductiontoemergingtechnologies": ["emerging", "emte", "emerging technology"],
     "computerprogramming": ["programming", "coding", "eceg", "python", "c++"],
-    "moralandciviceducation": ["civic", "civics", "moral", "mcie", "ethics"],
-    "physicalfitness": ["fitness", "sports", "spsc"],
-    "introductiontoeconomics": ["economics", "econ"],
+    "moralandciviceducation": ["ሲቪክስ", "ሥነ ምግባር", "civic", "civics", "moral", "mcie", "ethics"],
+    "physicalfitness": ["ስፖርት", "fitness", "sports", "spsc"],
+    "introductiontoeconomics": ["ኢኮኖሚክስ", "ምጣኔ ሀብት", "economics", "econ"],
     "globalaffairs": ["global", "global trends"],
-    "introductiontolawandtheethiopianlegalsystem": ["law", "legal"],
+    "introductiontolawandtheethiopianlegalsystem": ["ህግ", "ሕግ", "law", "legal"],
     "freshmanguideandgeneralresources": [
         "cut point", "cutt point", "cutpoint", "cutoff", "cut off", "placement", "department selection",
         "quantitative reasoning", "verbal reasoning", "logical reasoning", "aptitude", "reasoning", "verbal",
@@ -66,20 +66,24 @@ def _ratio(a: str, b: str) -> float:
     return difflib.SequenceMatcher(None, a, b).ratio()
 
 
-def _alias_hit(alias: str, ws: Sequence[str]) -> bool:
+def _alias_hit(alias: str, ws: Sequence[str], fuzzy_ok: bool = True) -> int:
+    """0 = no match, 1 = prefix match, 2 = exact match, 3 = close typo of a whole word."""
     parts = words(alias)
     if not parts:
-        return False
+        return 0
     if len(parts) > 1:
-        return all(any(w == p for w in ws) for p in parts)
+        return 2 if all(any(w == p for w in ws) for p in parts) else 0
     p = parts[0]
+    best = 0
     for w in ws:
         base = re.sub(r"\d+$", "", w)
-        if w == p or base == p or (len(p) >= 4 and w.startswith(p)):
-            return True
-        if len(p) >= 6 and len(w) >= 6 and _ratio(w, p) >= 0.85:  # "pysychology" -> psychology
-            return True
-    return False
+        if w == p or base == p:
+            return 2
+        if len(p) >= 4 and w.startswith(p):
+            best = max(best, 1)
+        elif fuzzy_ok and len(p) >= 6 and len(w) >= 6 and w[0] == p[0] and _ratio(w, p) >= 0.7:  # "psyscho" -> psychology
+            best = max(best, 3)
+    return best
 
 
 def _level(ws: Sequence[str]) -> Optional[int]:
@@ -113,8 +117,11 @@ def rank_courses(courses: Sequence, sources: Sequence[tuple[str, str]]) -> list[
                 score += 8 * weight
             best_alias = 0
             for alias in ALIASES.get(cname_key, ()):
-                if _alias_hit(alias, ws):
-                    best_alias = max(best_alias, 8 if len(alias.split()) > 1 else 5)
+                # the catch-all bucket only matches exact words ('reading' must never look like 'reasoning')
+                strength = _alias_hit(alias, ws, fuzzy_ok=cname_key != "freshmanguideandgeneralresources")
+                if strength:
+                    multi = len(alias.split()) > 1
+                    best_alias = max(best_alias, 8 if multi else (5 if strength == 1 else 6))
             score += best_alias * weight
             hits = sum(1 for nw in sig_words if any(_hit(w, nw) for w in ws))
             score += min(hits * 2.0 * weight, 4.0 if weight < 1 else 8.0)
@@ -142,27 +149,43 @@ def decide(ranked: Sequence[tuple[float, object]]) -> tuple[Optional[object], bo
 
 
 # ---------------------------------------------------------------- kind
-_EXAM_STRONG = ("exam", "midterm", "mid exam", "quiz", "past paper", "question paper", "ፈተና")
-_EXAM_WEAK = ("final", "test", "question", "previous")  # "Geography Module Final" is NOT an exam
-_SLIDE = ("slide", "lecture", "ppt", "presentation", "chapter")
-_NOTE = ("note", "handout", "summary", "worksheet", "assignment", "tutorial", "module", "ማስታወሻ")
-_BOOK = ("textbook", "book", "edition", "isbn", "volume")
+_EXAM_STRONG = {"exam", "exams", "examination", "midterm", "midexam", "quiz", "quizzes", "ፈተና", "ፈተናዎች", "ፈተናው"}
+_EXAM_PHRASES = ("past paper", "model exam", "question paper", "mid exam", "final exam")
+_EXAM_WEAK = {"final", "test", "tests", "question", "questions", "previous", "mid", "ጥያቄ", "ጥያቄዎች"}  # "Module Final" is NOT an exam
+_SLIDE = {"slide", "slides", "ppt", "pptx", "presentation", "lecture", "lectures", "ስላይድ", "ስላይዶች", "ፕሬዘንቴሽን"}
+_NOTE = {"note", "notes", "handout", "handouts", "summary", "summaries", "worksheet", "assignment", "tutorial", "module",
+         "modules", "lab", "ማስታወሻ", "ሞጁል", "ሞዱል"}
+_BOOK = {"textbook", "book", "books", "edition", "isbn", "volume", "መጽሐፍ", "መፅሐፍ", "መጽሀፍ", "መፅሀፍ"}
 _EXAM_TEXT = ("time allowed", "answer all questions", "choose the best answer", "instruction:", "id no", "name:")
 
 
+def _kind_hit(tokens: Sequence[str], vocab: set) -> bool:
+    """A whole word of the file name is a keyword, or a close typo of a longer one ('exm', 'quize', 'slid')."""
+    for t in tokens:
+        if t in vocab:
+            return True
+        if len(t) >= 3 and t.isascii():
+            for k in vocab:
+                if k.isascii() and len(k) >= 4 and t[0] == k[0] and abs(len(t) - len(k)) <= 2 and _ratio(t, k) >= 0.8:
+                    return True
+    return False
+
+
 def detect_kind(name: str, caption: str, text: str, pages: Optional[int], size: Optional[int], ext: str) -> tuple[str, str]:
-    label = f" {(name or '').lower()} {(caption or '').lower()} ".replace("_", " ").replace("-", " ")
+    label = f"{name or ''} {caption or ''}".lower()
+    tokens = words(label)
+    spaced = " " + " ".join(tokens) + " "
     if ext in (".ppt", ".pptx"):
         return "slide", "presentation file"
-    if any(k in label for k in _EXAM_STRONG):
+    if _kind_hit(tokens, _EXAM_STRONG) or any(f" {p} " in spaced for p in _EXAM_PHRASES):
         return "exam", "name says exam"
-    if any(k in label for k in _SLIDE):
-        return "slide", "name says slides/chapter"
-    if any(k in label for k in _BOOK):
+    if _kind_hit(tokens, _BOOK):
         return "book", "name says book"
-    if any(k in label for k in _NOTE):
-        return "note", "name says module/notes"  # module/notes beat the ambiguous word "final"
-    if any(k in label for k in _EXAM_WEAK) or re.search(r"\bmid\b", label):
+    if _kind_hit(tokens, _NOTE):
+        return "note", "name says module/notes"  # beats the ambiguous words "final" and "chapter"
+    if _kind_hit(tokens, _SLIDE):
+        return "slide", "name says slides/lecture"
+    if _kind_hit(tokens, _EXAM_WEAK):
         return "exam", "name hints at an exam"
     if sum(1 for k in _EXAM_TEXT if k in (text or "").lower()) >= 2:
         return "exam", "content looks like an exam"

tests/test_logic.py

@@ -634,6 +634,229 @@ def test_analytics_insights_and_screens():
         assert render(empty)
 
 
+REAL_NAMES = [
+ [
+  "psyscho 2014 mid exam collection.pdf",
+  "General Psychology",
+  "exam"
+ ],
+ [
+  "Physics mid exam 2014.pdf",
+  "General Physics",
+  "exam"
+ ],
+ [
+  "General Physics Final Exam.pdf",
+  "General Physics",
+  "exam"
+ ],
+ [
+  "Math 1003 mid.pdf",
+  "Mathematics for Natural Sciences",
+  "exam"
+ ],
+ [
+  "Chem Exam 2013 EC.pdf",
+  "General Chemistry",
+  "exam"
+ ],
+ [
+  "Anthropology Exam Questions.pdf",
+  "Social Anthropology",
+  "exam"
+ ],
+ [
+  "Logic and critical thinking questions.pdf",
+  "Critical Thinking",
+  "exam"
+ ],
+ [
+  "ኢኮኖሚክስ ጥያቄ.pdf",
+  "Introduction to Economics",
+  "exam"
+ ],
+ [
+  "ፊዚክስ ፈተና 2014.pdf",
+  "General Physics",
+  "exam"
+ ],
+ [
+  "Civics model exam.pdf",
+  "Moral and Civic Education",
+  "exam"
+ ],
+ [
+  "Past paper English 2013.pdf",
+  null,
+  "exam"
+ ],
+ [
+  "geography quiz 1.pdf",
+  "Geography of Ethiopia and the Horn",
+  "exam"
+ ],
+ [
+  "Psychology test 1 answer key.pdf",
+  "General Psychology",
+  "exam"
+ ],
+ [
+  "Mid-term exam for Entrepreneurship.pdf",
+  "Entrepreneurship",
+  "exam"
+ ],
+ [
+  "Psychology note.pdf",
+  "General Psychology",
+  "note"
+ ],
+ [
+  "Chapter 1-3 summary psychology.pdf",
+  "General Psychology",
+  "note"
+ ],
+ [
+  "ፊዚክስ ማስታወሻ.pdf",
+  "General Physics",
+  "note"
+ ],
+ [
+  "Economics handout.pdf",
+  "Introduction to Economics",
+  "note"
+ ],
+ [
+  "Entrepreneurship chapter 2 notes.pdf",
+  "Entrepreneurship",
+  "note"
+ ],
+ [
+  "Anthropology module.pdf",
+  "Social Anthropology",
+  "note"
+ ],
+ [
+  "Physics short note for mid.pdf",
+  "General Physics",
+  "note"
+ ],
+ [
+  "Lecture 3 physics.pdf",
+  "General Physics",
+  "slide"
+ ],
+ [
+  "Chapter 1 Entrepreneurship.pptx",
+  "Entrepreneurship",
+  "slide"
+ ],
+ [
+  "ስላይድ ኢኮኖሚክስ.pdf",
+  "Introduction to Economics",
+  "slide"
+ ],
+ [
+  "General Chemistry textbook.pdf",
+  "General Chemistry",
+  "book"
+ ],
+ [
+  "ባዮሎጂ መጽሐፍ.pdf",
+  "General Biology",
+  "book"
+ ],
+ [
+  "document.pdf",
+  null,
+  "note"
+ ],
+ [
+  "IMG_2034.pdf",
+  null,
+  "note"
+ ],
+ [
+  "1698765432.pdf",
+  null,
+  "note"
+ ],
+ [
+  "PSYCH_1007_Module(final).pdf",
+  "General Psychology",
+  "note"
+ ],
+ [
+  "Psys1007 worksheet 2.pdf",
+  "General Psychology",
+  "note"
+ ],
+ [
+  "physics lab report.pdf",
+  "General Physics",
+  "note"
+ ],
+ [
+  "economics assignment 1.pdf",
+  "Introduction to Economics",
+  "note"
+ ],
+ [
+  "Exm 2014 geography.pdf",
+  "Geography of Ethiopia and the Horn",
+  "exam"
+ ],
+ [
+  "Psychology Exam collection (2013-2015).pdf",
+  "General Psychology",
+  "exam"
+ ],
+ [
+  "philosophy notes.pdf",
+  null,
+  "note"
+ ],
+ [
+  "geometry worked examples.pdf",
+  null,
+  "note"
+ ],
+ [
+  "notebook.pdf",
+  null,
+  "note"
+ ],
+ [
+  "finalize registration.pdf",
+  null,
+  "note"
+ ],
+ [
+  "midnight reading.pdf",
+  null,
+  "note"
+ ],
+ [
+  "latest news.pdf",
+  null,
+  "note"
+ ]
+]
+
+
+def test_realistic_student_file_names():
+    """Amharic names, typos, mid/final/collection, chapter..., plus words that must NOT be misread."""
+    import os
+
+    wrong = []
+    for name, want_course, want_kind in REAL_NAMES:
+        best, confident, _ = C.decide(C.rank_courses(COURSES, [(name, "name")]))
+        course = best.name if (best and confident) else None
+        kind, _why = C.detect_kind(name, "", "", None, None, os.path.splitext(name)[1].lower())
+        if (course, kind) != (want_course, want_kind):
+            wrong.append((name, course, kind))
+    assert not wrong, wrong
+
+
 if __name__ == "__main__":
     for name, fn in list(globals().items()):
         if name.startswith("test_"):