add advanced file name detector

app/bot/handlers/admin.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 from aiogram import F, Router
 from aiogram.filters import BaseFilter, Command, CommandObject
-from aiogram.types import Message, ReactionTypeEmoji
+from aiogram.types import CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, Message, ReactionTypeEmoji
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -19,6 +19,7 @@ from app.db.repos import stats as stats_repo
 from app.core import season as clock
 from app.db.repos import users as users_repo
 from app.services import library as library_service
+from app.services import smart_ingest
 from app.services import season as season_service
 from app.services import xp as xp_service
 from app.services.ingest import KIND_ALIASES, clean_title, parse_caption
@@ -27,12 +28,13 @@ log = logging.getLogger(__name__)
 
 
 class IsAdmin(BaseFilter):
-    async def __call__(self, message: Message) -> bool:
-        return bool(message.from_user and message.from_user.id in get_settings().admin_id_set)
+    async def __call__(self, event: Message | CallbackQuery) -> bool:
+        return bool(event.from_user and event.from_user.id in get_settings().admin_id_set)
 
 
 router = Router()
 router.message.filter(IsAdmin())
+router.callback_query.filter(IsAdmin())
 
 HELP = (
     "<b>Admin tools</b>\n"
@@ -48,10 +50,13 @@ HELP = (
     "/closeweek &lt;2026W41&gt; : close a finished week now\n"
     "/give &lt;user_id&gt; &lt;xp&gt; : reward a real contribution\n"
     "/trust &lt;user_id&gt; &lt;delta&gt; : lower/raise a trust score (below 70 = no prizes)\n\n"
-    "<b>Bulk upload (easiest):</b>\n"
-    "<code>/upload 3 exam</code> then send or FORWARD many files (e.g. from your channel).\n"
-    "👍 = saved. Send /done when finished.\n\n"
-    "<b>Single file:</b> send it with caption\n"
+    "<b>Smart upload (default):</b> just send or FORWARD files. The bot finds the course, type, "
+    "title and skips duplicates. If unsure it asks you (tap a button).\n"
+    "/recent : last 15 files · /review : files waiting for you\n"
+    "/move &lt;id&gt; &lt;course_id&gt; [kind] · /rename &lt;id&gt; &lt;title&gt; · /del &lt;id&gt;\n"
+    "/mergecourses : merge courses shared by both streams (run once, then /seedcourses)\n\n"
+    "<b>Force a course:</b> <code>/upload 3 exam</code>, then send files, /done when finished.\n"
+    "<b>Or</b> send a file with caption\n"
     "<code>&lt;course_id&gt; &lt;slide|exam|note|book&gt; [title]</code>\n"
     "e.g. <code>3 exam Midterm 2017</code>"
 )
@@ -150,6 +155,7 @@ async def add_course(message: Message, command: CommandObject, session: AsyncSes
 async def seed_courses(message: Message, session: AsyncSession) -> None:
     added = skipped = 0
     for stream, year, semester, code, name in FRESHMAN_COURSES:
+        stream = None if stream == "all" else stream
         if await repo.find_course(session, name, stream, year):
             skipped += 1
             continue
@@ -190,20 +196,20 @@ async def stats(message: Message, session: AsyncSession) -> None:
     )
 
 
-def _extract(message: Message) -> Optional[tuple[str, str, str, Optional[str]]]:
-    """-> (file_type, file_id, file_unique_id, file_name)"""
+def _extract(message: Message) -> Optional[tuple[str, str, str, Optional[str], Optional[int]]]:
+    """-> (file_type, file_id, file_unique_id, file_name, size)"""
     if message.document:
         d = message.document
-        return "document", d.file_id, d.file_unique_id, d.file_name
+        return "document", d.file_id, d.file_unique_id, d.file_name, d.file_size
     if message.video:
         v = message.video
-        return "video", v.file_id, v.file_unique_id, v.file_name
+        return "video", v.file_id, v.file_unique_id, v.file_name, v.file_size
     if message.audio:
         a = message.audio
-        return "audio", a.file_id, a.file_unique_id, a.file_name or a.title
+        return "audio", a.file_id, a.file_unique_id, a.file_name or a.title, a.file_size
     if message.photo:
         p = message.photo[-1]
-        return "photo", p.file_id, p.file_unique_id, None
+        return "photo", p.file_id, p.file_unique_id, None, p.file_size
     return None
 
 
@@ -224,9 +230,9 @@ async def upload_on(message: Message, command: CommandObject, session: AsyncSess
         return
     _upload_mode[message.from_user.id] = {"course_id": course.id, "kind": kind, "saved": 0, "dupes": 0}
     await message.answer(
-        f"📥 Upload mode <b>ON</b>: {course.name} · {kind}\n"
-        "Send or forward files now (many at once is fine). 👍 = saved.\n"
-        "Change course: send /upload again. Finish: /done"
+        f"📥 Forced mode <b>ON</b>: {course.name} · {kind}\n"
+        "Every file goes to this course. 👍 = saved. Finish: /done\n"
+        "(Normally you do NOT need this: just send files and the bot sorts them.)"
     )
 
 
@@ -234,7 +240,7 @@ async def upload_on(message: Message, command: CommandObject, session: AsyncSess
 async def upload_off(message: Message) -> None:
     mode = _upload_mode.pop(message.from_user.id, None)
     if not mode:
-        await message.answer("Upload mode was not on.")
+        await message.answer("Forced mode was not on.")
         return
     await message.answer(f"✅ Done. Saved <b>{mode['saved']}</b> files, skipped {mode['dupes']} duplicates.")
 
@@ -242,62 +248,180 @@ async def upload_off(message: Message) -> None:
 async def _ack(message: Message) -> None:
     try:
         await message.react([ReactionTypeEmoji(emoji="👍")])
-    except Exception:  # reactions unavailable in this chat/client: stay quiet
+    except Exception:  # reactions unavailable: stay quiet
         pass
 
 
+def _line(o: smart_ingest.Outcome) -> str:
+    m = o.material
+    course = f"{o.course.code + ' ' if o.course and o.course.code else ''}{o.course.name}" if o.course else "❓ unsorted"
+    text = f"✅ <b>#{m.id}</b> · {course} · <b>{o.kind}</b>\n“{o.title}”"
+    if m.pages:
+        text += f" · {m.pages} p"
+    for note in o.notes:
+        text += f"\n⚠️ {note}"
+    return text
+
+
+def _kb_review(o: smart_ingest.Outcome) -> InlineKeyboardMarkup:
+    mid = o.material.id
+    rows = [
+        [InlineKeyboardButton(text=f"📘 {(c.code + ' ') if c.code else ''}{c.name}"[:60], callback_data=f"rv:c:{mid}:{c.id}")]
+        for c in o.suggestions
+    ]
+    rows.append([InlineKeyboardButton(text="🗑 Delete", callback_data=f"rv:x:{mid}")])
+    return InlineKeyboardMarkup(inline_keyboard=rows)
+
+
 @router.message(F.chat.type == "private", F.document | F.video | F.audio | F.photo)
 async def ingest(message: Message, user: User, session: AsyncSession) -> None:
-    parsed = parse_caption(message.caption)  # explicit caption wins
+    parsed = parse_caption(message.caption)  # explicit "<id> <kind> [title]" caption wins
     mode = _upload_mode.get(user.id)
-    if parsed:
-        course_id, kind, title = parsed
-    elif mode:
-        course_id, kind, title = mode["course_id"], mode["kind"], None
-    else:
+    forced_course = forced_kind = forced_title = None
+    if parsed or mode:
+        course_id, forced_kind, forced_title = parsed if parsed else (mode["course_id"], mode["kind"], None)
+        forced_course = await repo.get_course(session, course_id)
+        if not forced_course:
+            await message.answer(f"❗ No course with id {course_id}. See /courses")
+            return
+    file_type, file_id, unique_id, file_name, size = _extract(message)
+    out = await smart_ingest.ingest_file(
+        message.bot, session,
+        uploader_id=user.id, file_type=file_type, file_id=file_id, unique_id=unique_id,
+        file_name=file_name, size=size, caption=message.caption,
+        forced_course=forced_course, forced_kind=forced_kind, forced_title=forced_title,
+        backup_channel=get_settings().library_channel_id, message=message,
+    )
+    quiet = bool(mode and not parsed)  # forced bulk mode: one 👍 per file instead of a message
+
+    if out.status == "duplicate":
+        if quiet:
+            mode["dupes"] += 1
+            await _ack(message)
+        else:
+            d = out.duplicate_of
+            await message.answer(f"♻️ Skipped: {out.why}. Already saved as <b>#{d.id}</b> “{d.title}”.")
+    elif out.status == "saved":
+        if quiet:
+            mode["saved"] += 1
+            await _ack(message)
+        else:
+            await message.answer(_line(out))
+    elif out.status == "near_duplicate":
+        d = out.duplicate_of
+        mid = out.material.id
         await message.answer(
-            "❗ Use <code>/upload &lt;course_id&gt; &lt;kind&gt;</code> for bulk upload, or send the file with caption\n"
-            "<code>&lt;course_id&gt; &lt;" + "|".join(KINDS) + "&gt; [title]</code>\nSee /courses for ids."
+            f"🔍 <b>#{mid}</b> “{out.title}” has almost the same text as <b>#{d.id}</b> “{d.title}”.\nKeep both?",
+            reply_markup=InlineKeyboardMarkup(inline_keyboard=[[
+                InlineKeyboardButton(text="✅ Keep both", callback_data=f"rv:k:{mid}"),
+                InlineKeyboardButton(text="🗑 Delete new", callback_data=f"rv:x:{mid}"),
+            ]]),
         )
+    else:  # review
+        hint = "Pick the course:" if out.suggestions else f"No idea which course. Use <code>/move {out.material.id} &lt;course_id&gt;</code>"
+        await message.answer(
+            f"🤔 <b>#{out.material.id}</b> “{out.title}” · {out.kind}\nNot sure about the course. {hint}",
+            reply_markup=_kb_review(out) if out.suggestions else None,
+        )
+
+
+# ---------- review buttons ----------
+@router.callback_query(F.data.startswith("rv:"))
+async def review_button(call: CallbackQuery, session: AsyncSession) -> None:
+    parts = call.data.split(":")
+    action, material_id = parts[1], int(parts[2])
+    material = await repo.get_material(session, material_id)
+    if not material:
+        await call.answer("Already gone.", show_alert=True)
         return
-    course = await repo.get_course(session, course_id)
-    if not course:
-        await message.answer(f"❗ No course with id {course_id}. See /courses")
+    if action == "x":
+        await repo.delete_material(session, material_id)
+        session.sync_session.info["wrote"] = True
+        text = f"🗑 Deleted #{material_id}."
+    elif action == "k":
+        material.status = "ok"
+        text = f"✅ Kept #{material_id} “{material.title}”."
+    else:  # "c": assign course
+        course = await repo.get_course(session, int(parts[3]))
+        if not course:
+            await call.answer("No such course.", show_alert=True)
+            return
+        material.course_id, material.status = course.id, "ok"
+        text = f"✅ #{material_id} → {course.name}"
+    library_service.invalidate_courses()
+    await call.answer()
+    if call.message:
+        try:
+            await call.message.edit_text(text)
+        except Exception:
+            await call.message.answer(text)
+
+
+# ---------- corrections ----------
+@router.message(Command("recent"))
+async def recent(message: Message, session: AsyncSession) -> None:
+    items = await repo.recent_materials(session, 15)
+    if not items:
+        await message.answer("No files yet.")
         return
-    file_type, file_id, unique_id, file_name = _extract(message)
-    existing = await repo.find_by_unique_id(session, unique_id)
-    if existing:
-        if mode and not parsed:
-            mode["dupes"] += 1
-            await _ack(message)
-        else:
-            await message.answer(f"⚠️ Already saved as file #{existing.id}: {existing.title}")
+    courses = await repo.get_courses_map(session, {m.course_id for m in items if m.course_id})
+    lines = []
+    for m in items:
+        c = courses.get(m.course_id)
+        flag = "" if m.status == "ok" else " 🔸review"
+        lines.append(f"<b>#{m.id}</b> {c.name if c else '❓'} · {m.kind} · {m.title[:40]}{flag}")
+    await message.answer("\n".join(lines))
+
+
+@router.message(Command("review"))
+async def review_list(message: Message, session: AsyncSession) -> None:
+    items = await repo.review_materials(session)
+    if not items:
+        await message.answer("Nothing waiting for review 🎉")
         return
+    await message.answer("\n".join(f"<b>#{m.id}</b> {m.kind} · {m.title[:50]}" for m in items) + "\n\nFix with /move &lt;id&gt; &lt;course_id&gt; or /del &lt;id&gt;")
 
-    backup_chat = backup_msg = None
-    channel = get_settings().library_channel_id
-    if channel:
-        try:
-            copied = await message.copy_to(channel)
-            backup_chat, backup_msg = channel, copied.message_id
-        except Exception as exc:  # backup is optional; never block the upload
-            log.warning("Library backup copy failed: %s", exc)
-
-    material = await repo.add_material(
-        session,
-        course_id=course.id,
-        kind=kind,
-        title=(title or clean_title(file_name, message.caption, f"{course.name} {kind}"))[:250],
-        file_id=file_id,
-        file_type=file_type,
-        file_unique_id=unique_id,
-        uploader_id=user.id,
-        library_chat_id=backup_chat,
-        library_message_id=backup_msg,
-    )
-    if mode and not parsed:
-        mode["saved"] += 1
-        await _ack(message)
-    else:
-        note = "" if backup_msg else " (no backup copy)"
-        await message.answer(f"✅ Saved #{material.id}: {material.title}\n📘 {course.name} · {kind}{note}")
+
+@router.message(Command("move"))
+async def move(message: Message, command: CommandObject, session: AsyncSession) -> None:
+    parts = (command.args or "").split()
+    if len(parts) not in (2, 3) or not parts[0].isdigit() or not parts[1].isdigit():
+        await message.answer("Usage: <code>/move &lt;file_id&gt; &lt;course_id&gt; [slide|exam|note|book]</code>")
+        return
+    material, course = await repo.get_material(session, int(parts[0])), await repo.get_course(session, int(parts[1]))
+    if not material or not course:
+        await message.answer("File or course not found.")
+        return
+    material.course_id, material.status = course.id, "ok"
+    if len(parts) == 3 and KIND_ALIASES.get(parts[2].lower()):
+        material.kind = KIND_ALIASES[parts[2].lower()]
+    await message.answer(f"✅ #{material.id} → {course.name} · {material.kind}")
+
+
+@router.message(Command("rename"))
+async def rename(message: Message, command: CommandObject, session: AsyncSession) -> None:
+    parts = (command.args or "").split(maxsplit=1)
+    material = await repo.get_material(session, int(parts[0])) if len(parts) == 2 and parts[0].isdigit() else None
+    if not material:
+        await message.answer("Usage: <code>/rename &lt;file_id&gt; &lt;new title&gt;</code>")
+        return
+    material.title = parts[1].strip()[:250]
+    await message.answer(f"✅ #{material.id} renamed: “{material.title}”")
+
+
+@router.message(Command("del"))
+async def delete_cmd(message: Message, command: CommandObject, session: AsyncSession) -> None:
+    arg = (command.args or "").strip()
+    if not arg.isdigit() or not await repo.delete_material(session, int(arg)):
+        await message.answer("Usage: <code>/del &lt;file_id&gt;</code> (must exist)")
+        return
+    session.sync_session.info["wrote"] = True
+    await message.answer(f"🗑 Deleted #{arg}.")
+
+
+@router.message(Command("mergecourses"))
+async def merge_courses(message: Message, session: AsyncSession) -> None:
+    n = await repo.merge_duplicate_courses(session)
+    session.sync_session.info["wrote"] = True
+    library_service.invalidate_courses()
+    await message.answer(f"✅ Merged {n} duplicate courses. Now run /seedcourses to add any missing ones, then /courses.")

app/bot/handlers/library.py

@@ -86,7 +86,7 @@ async def on_materials(call: CallbackQuery, user: User, session: AsyncSession) -
 @router.callback_query(F.data.startswith("lib:f:"))
 async def on_file(call: CallbackQuery, user: User, session: AsyncSession) -> None:
     material = await repo.get_material(session, int(call.data.split(":")[2]))
-    if not material or not material.file_id:
+    if not material or not material.file_id or material.status != "ok":
         await call.answer(t(user.lang, "file_missing"), show_alert=True)
         return
     course = await repo.get_course(session, material.course_id) if material.course_id else None

app/core/seed_courses.py

@@ -1,52 +1,37 @@
 """Year-1 (freshman) course catalog used by /seedcourses.
 
 Sources (checked 14 Oct 2026):
-  * NATURAL stream, courses WITH a code: official DBU course catalog (dbu.edu.et, College of
-    Engineering, Year I). Codes are DBU's own.
-  * NATURAL stream, General Biology / General Chemistry: common national natural-science
-    freshman list (Jimma / Debark University). DBU code not verified -> shown without a code.
-  * SOCIAL stream: national common social-science freshman list (University of Gondar listing).
-    No DBU-specific source found -> shown without codes. Please verify against your registrar list.
+  * Courses WITH a code: official DBU course catalog (dbu.edu.et, College of Engineering, Year I).
+  * General Biology / General Chemistry: national natural-science freshman list (Jimma / Debark).
+  * Social-science-only courses: national common list (University of Gondar). No DBU source found.
+  * Courses that natural AND social students both take are stored ONCE (stream "all").
+    A file for "English" therefore never needs to be copied into two courses.
 
-Tuple: (stream, year, semester, code or None, name)
+Tuple: (stream natural|social|all, year, semester, code or None, name)
 """
 
 FRESHMAN_COURSES: list[tuple[str, int, int, str | None, str]] = [
-    # ---- NATURAL, semester I (DBU catalog) ----
-    ("natural", 1, 1, "FLEn1001", "Communicative English Language Skills I"),
+    # ---- semester I ----
+    ("all", 1, 1, "FLEn1001", "Communicative English Language Skills I"),
     ("natural", 1, 1, "Math1003", "Mathematics for Natural Sciences"),
-    ("natural", 1, 1, "LoCT1005", "Critical Thinking"),
-    ("natural", 1, 1, "Psys1007", "General Psychology"),
+    ("all", 1, 1, "LoCT1005", "Critical Thinking"),
+    ("all", 1, 1, "Psys1007", "General Psychology"),
     ("natural", 1, 1, "Phys1009", "General Physics"),
-    ("natural", 1, 1, "SpSc1011", "Physical Fitness"),
-    ("natural", 1, 1, "GeES1013", "Geography of Ethiopia and the Horn"),
-    # ---- NATURAL, semester II (DBU catalog) ----
-    ("natural", 1, 2, "FLEn1002", "Communicative English Language Skills II"),
+    ("all", 1, 1, "SpSc1011", "Physical Fitness"),
+    ("all", 1, 1, "GeES1013", "Geography of Ethiopia and the Horn"),
+    ("social", 1, 1, None, "Introduction to Economics"),
+    ("social", 1, 1, None, "Mathematics for Social Sciences"),
+    # ---- semester II ----
+    ("all", 1, 2, "FLEn1002", "Communicative English Language Skills II"),
     ("natural", 1, 2, "Math1004", "Applied Mathematics I"),
-    ("natural", 1, 2, "Anth1006", "Social Anthropology"),
-    ("natural", 1, 2, "MGMT1008", "Entrepreneurship"),
-    ("natural", 1, 2, "EmTe1010", "Introduction to Emerging Technologies"),
+    ("all", 1, 2, "Anth1006", "Social Anthropology"),
+    ("all", 1, 2, "MGMT1008", "Entrepreneurship"),
+    ("all", 1, 2, "EmTe1010", "Introduction to Emerging Technologies"),
     ("natural", 1, 2, "ECEg1012", "Computer Programming"),
-    ("natural", 1, 2, "MCiE1014", "Moral and Civic Education"),
-    ("natural", 1, 2, "Hist1016", "History of Ethiopia and the Horn"),
-    # ---- NATURAL science (non-engineering), semester II: code not verified for DBU ----
+    ("all", 1, 2, "MCiE1014", "Moral and Civic Education"),
+    ("all", 1, 2, "Hist1016", "History of Ethiopia and the Horn"),
     ("natural", 1, 2, None, "General Biology"),
     ("natural", 1, 2, None, "General Chemistry"),
-    # ---- SOCIAL, semester I (national common list; codes not verified for DBU) ----
-    ("social", 1, 1, None, "Communicative English Language Skills I"),
-    ("social", 1, 1, None, "General Psychology"),
-    ("social", 1, 1, None, "Geography of Ethiopia and the Horn"),
-    ("social", 1, 1, None, "Introduction to Economics"),
-    ("social", 1, 1, None, "Logic and Critical Thinking"),
-    ("social", 1, 1, None, "Mathematics for Social Sciences"),
-    ("social", 1, 1, None, "Physical Fitness"),
-    # ---- SOCIAL, semester II ----
-    ("social", 1, 2, None, "Communicative English Language Skills II"),
-    ("social", 1, 2, None, "Entrepreneurship"),
     ("social", 1, 2, None, "Global Affairs"),
     ("social", 1, 2, None, "Introduction to Law and the Ethiopian Legal System"),
-    ("social", 1, 2, None, "Social Anthropology"),
-    ("social", 1, 2, None, "History of Ethiopia and the Horn"),
-    ("social", 1, 2, None, "Introduction to Emerging Technologies"),
-    ("social", 1, 2, None, "Moral and Civic Education"),
 ]

app/db/migrate.py

@@ -27,6 +27,13 @@ POSTGRES_UPGRADES = [
     "ALTER TABLE users ADD COLUMN IF NOT EXISTS day_key VARCHAR(10)",
     "ALTER TABLE users ADD COLUMN IF NOT EXISTS day_xp INTEGER NOT NULL DEFAULT 0",
     "ALTER TABLE users ADD COLUMN IF NOT EXISTS trust INTEGER NOT NULL DEFAULT 100",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64)",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS text_sig BIGINT",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS pages INTEGER",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS size INTEGER",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS source_name VARCHAR(256)",
+    "ALTER TABLE materials ADD COLUMN IF NOT EXISTS status VARCHAR(8) NOT NULL DEFAULT 'ok'",
+    "CREATE INDEX IF NOT EXISTS ix_materials_content_hash ON materials (content_hash)",
     "ALTER TABLE referrals ADD COLUMN IF NOT EXISTS qualified BOOLEAN NOT NULL DEFAULT FALSE",
 ]
 

app/db/models/material.py

@@ -23,6 +23,12 @@ class Material(Base):
     file_id: Mapped[Optional[str]] = mapped_column(String(512))
     file_type: Mapped[str] = mapped_column(String(16), default="document", server_default="document")
     file_unique_id: Mapped[Optional[str]] = mapped_column(String(128), unique=True)
+    content_hash: Mapped[Optional[str]] = mapped_column(String(64), index=True)  # sha256 of the bytes
+    text_sig: Mapped[Optional[int]] = mapped_column(BigInteger)  # SimHash of the text (near-duplicates)
+    pages: Mapped[Optional[int]] = mapped_column(Integer)
+    size: Mapped[Optional[int]] = mapped_column(Integer)
+    source_name: Mapped[Optional[str]] = mapped_column(String(256))  # original file name
+    status: Mapped[str] = mapped_column(String(8), default="ok", server_default="ok")  # ok | review
     downloads: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
     uploader_id: Mapped[Optional[int]] = mapped_column(BigInteger)
     library_chat_id: Mapped[Optional[int]] = mapped_column(BigInteger)

app/db/repos/library.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from sqlalchemy import and_, func, or_, select
+from sqlalchemy import and_, delete, func, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import Course, Material
@@ -78,7 +78,7 @@ async def add_course(
 async def kind_counts(session: AsyncSession, course_id: int) -> dict[str, int]:
     res = await session.execute(
         select(Material.kind, func.count())
-        .where(Material.course_id == course_id)
+        .where(Material.course_id == course_id, Material.status == "ok")
         .group_by(Material.kind)
     )
     return {kind: int(n) for kind, n in res.all()}
@@ -88,7 +88,7 @@ async def count_materials(session: AsyncSession, course_id: int, kind: str) -> i
     res = await session.execute(
         select(func.count())
         .select_from(Material)
-        .where(Material.course_id == course_id, Material.kind == kind)
+        .where(Material.course_id == course_id, Material.kind == kind, Material.status == "ok")
     )
     return int(res.scalar_one())
 
@@ -98,7 +98,7 @@ async def list_materials(
 ) -> list[Material]:
     res = await session.execute(
         select(Material)
-        .where(Material.course_id == course_id, Material.kind == kind)
+        .where(Material.course_id == course_id, Material.kind == kind, Material.status == "ok")
         .order_by(Material.downloads.desc(), Material.id.desc())
         .offset(offset)
         .limit(limit)
@@ -125,7 +125,7 @@ async def add_material(session: AsyncSession, **fields) -> Material:
 async def top_materials(session: AsyncSession, limit: int) -> list[Material]:
     res = await session.execute(
         select(Material)
-        .where(Material.file_id.is_not(None))
+        .where(Material.file_id.is_not(None), Material.status == "ok")
         .order_by(Material.downloads.desc(), Material.id.desc())
         .limit(limit)
     )
@@ -153,7 +153,7 @@ async def search_materials(session: AsyncSession, tokens: list[str], limit: int)
     res = await session.execute(
         select(Material)
         .outerjoin(Course, Material.course_id == Course.id)
-        .where(cond, Material.file_id.is_not(None))
+        .where(cond, Material.file_id.is_not(None), Material.status == "ok")
         .order_by(Material.downloads.desc(), Material.id.desc())
         .limit(limit)
     )
@@ -163,7 +163,7 @@ async def search_materials(session: AsyncSession, tokens: list[str], limit: int)
 async def file_counts(session: AsyncSession) -> dict[int, int]:
     res = await session.execute(
         select(Material.course_id, func.count())
-        .where(Material.file_id.is_not(None), Material.course_id.is_not(None))
+        .where(Material.file_id.is_not(None), Material.status == "ok", Material.course_id.is_not(None))
         .group_by(Material.course_id)
     )
     return {int(cid): int(n) for cid, n in res.all()}
@@ -174,8 +174,77 @@ async def materials_for_courses(session: AsyncSession, course_ids: list[int], li
         return []
     res = await session.execute(
         select(Material)
-        .where(Material.course_id.in_(course_ids), Material.file_id.is_not(None))
+        .where(Material.course_id.in_(course_ids), Material.file_id.is_not(None), Material.status == "ok")
         .order_by(Material.downloads.desc(), Material.id.desc())
         .limit(limit)
     )
     return list(res.scalars())
+
+
+# ----- smart-ingest helpers -----
+async def find_by_hash(session: AsyncSession, content_hash: str) -> Optional[Material]:
+    res = await session.execute(select(Material).where(Material.content_hash == content_hash).limit(1))
+    return res.scalar_one_or_none()
+
+
+async def all_sigs(session: AsyncSession) -> list[tuple[int, int, str]]:
+    res = await session.execute(
+        select(Material.id, Material.text_sig, Material.title).where(Material.text_sig.is_not(None))
+    )
+    return [(int(i), int(sig), title) for i, sig, title in res.all()]
+
+
+async def titles_in_course(session: AsyncSession, course_id: int, kind: str) -> list[str]:
+    res = await session.execute(
+        select(Material.title).where(Material.course_id == course_id, Material.kind == kind)
+    )
+    return list(res.scalars())
+
+
+async def recent_materials(session: AsyncSession, limit: int = 15) -> list[Material]:
+    res = await session.execute(select(Material).order_by(Material.id.desc()).limit(limit))
+    return list(res.scalars())
+
+
+async def review_materials(session: AsyncSession, limit: int = 30) -> list[Material]:
+    res = await session.execute(
+        select(Material).where(Material.status == "review").order_by(Material.id).limit(limit)
+    )
+    return list(res.scalars())
+
+
+async def delete_material(session: AsyncSession, material_id: int) -> bool:
+    res = await session.execute(delete(Material).where(Material.id == material_id))
+    return bool(res.rowcount)
+
+
+async def merge_duplicate_courses(session: AsyncSession) -> int:
+    """Courses that natural and social students both take were seeded twice (once per stream).
+    Merge each pair into ONE course visible to everybody and move its files. Safe to run twice."""
+    import re
+
+    def key(name: str) -> str:
+        k = re.sub(r"[^a-z0-9]+", "", name.lower())
+        return {"logicandcriticalthinking": "criticalthinking"}.get(k, k)
+
+    groups: dict[str, list[Course]] = {}
+    for course in await all_courses(session):
+        groups.setdefault(key(course.name), []).append(course)
+    merged = 0
+    for members in groups.values():
+        if len(members) < 2:
+            continue
+        keep = sorted(members, key=lambda c: (c.code is None, c.id))[0]
+        if len({m.stream for m in members}) > 1:
+            keep.stream = None
+        for other in members:
+            if other.id == keep.id:
+                continue
+            await session.execute(
+                update(Material).where(Material.course_id == other.id).values(course_id=keep.id)
+                .execution_options(synchronize_session=False)
+            )
+            await session.execute(delete(Course).where(Course.id == other.id))
+            merged += 1
+    await session.flush()
+    return merged

app/services/classifier.py

@@ -0,0 +1,251 @@
+"""Smart file understanding: which course, which kind, a clean title, duplicate fingerprints.
+Pure functions (no I/O) so everything here is unit-tested."""
+import hashlib
+import re
+from typing import Optional, Sequence
+
+from app.services.fuzzy import words
+
+# ---------------------------------------------------------------- helpers
+_STOP = {"for", "of", "the", "and", "to", "in", "on", "a", "an"}
+_GENERIC = {"general", "introduction", "skills", "language", "sciences", "science", "education", "technologies", "communicative"}
+
+
+def collapse(text: str) -> str:
+    return re.sub(r"[^a-z0-9\u1200-\u137f]+", "", (text or "").lower())
+
+
+def norm_title_key(title: str) -> str:
+    return collapse(title)
+
+
+# Short, hand-written synonyms (collapsed course name -> aliases). Cheap to extend.
+ALIASES: dict[str, list[str]] = {
+    "mathematicsfornaturalsciences": ["calculus", "natural math", "math natural"],
+    "appliedmathematicsi": ["applied math", "applied maths", "applied mathematics"],
+    "mathematicsforsocialsciences": ["social math", "math social", "maths social"],
+    "generalphysics": ["physics", "phys", "mechanics", "ፊዚክስ"],
+    "generalchemistry": ["chemistry", "chem", "ኬሚስትሪ"],
+    "generalbiology": ["biology", "bio", "ባዮሎጂ"],
+    "communicativeenglishlanguageskillsi": ["english", "eng", "flen", "comm english", "እንግሊዝኛ"],
+    "communicativeenglishlanguageskillsii": ["english", "eng", "flen", "comm english", "እንግሊዝኛ"],
+    "generalpsychology": ["psychology", "psych", "psys", "ሳይኮሎጂ"],
+    "criticalthinking": ["logic", "loct", "logic and critical thinking"],
+    "geographyofethiopiaandthehorn": ["geography", "geo", "gees", "ጂኦግራፊ"],
+    "historyofethiopiaandthehorn": ["history", "hist", "ታሪክ"],
+    "socialanthropology": ["anthropology", "anth"],
+    "entrepreneurship": ["entrepreneur", "entre", "mgmt"],
+    "introductiontoemergingtechnologies": ["emerging", "emte", "emerging technology"],
+    "computerprogramming": ["programming", "coding", "eceg", "python", "c++"],
+    "moralandciviceducation": ["civic", "civics", "moral", "mcie", "ethics"],
+    "physicalfitness": ["fitness", "sports", "spsc"],
+    "introductiontoeconomics": ["economics", "econ"],
+    "globalaffairs": ["global", "global trends"],
+    "introductiontolawandtheethiopianlegalsystem": ["law", "legal"],
+}
+
+
+def _hit(word: str, target: str) -> bool:
+    """word (from a file name) refers to target (a course-name word)."""
+    if word == target:
+        return True
+    if len(word) >= 4 and (target.startswith(word) or word.startswith(target)):
+        return True
+    return len(word) >= 5 and len(target) >= 5 and _ratio(word, target) >= 0.85
+
+
+def _ratio(a: str, b: str) -> float:
+    import difflib
+
+    return difflib.SequenceMatcher(None, a, b).ratio()
+
+
+def _alias_hit(alias: str, ws: Sequence[str]) -> bool:
+    parts = words(alias)
+    if not parts:
+        return False
+    if len(parts) > 1:
+        return all(any(w == p for w in ws) for p in parts)
+    p = parts[0]
+    for w in ws:
+        base = re.sub(r"\d+$", "", w)
+        if w == p or base == p or (len(p) >= 4 and w.startswith(p)):
+            return True
+    return False
+
+
+def _level(ws: Sequence[str]) -> Optional[int]:
+    if any(w in ("ii", "2") for w in ws) or any(re.fullmatch(r"[a-z]*(1002|ii)", w) for w in ws):
+        return 2
+    if any(w in ("i", "1") for w in ws):
+        return 1
+    return None
+
+
+# ---------------------------------------------------------------- course matching
+def rank_courses(courses: Sequence, sources: Sequence[tuple[str, str]]) -> list[tuple[float, object]]:
+    """sources: [(text, 'name'|'text')]. 'name' = file name/caption (strong), 'text' = content (weak).
+    Returns [(score, course)] best first, only courses with score > 0."""
+    prepared = []
+    for text, kind in sources:
+        if text and text.strip():
+            prepared.append((collapse(text), words(text), 1.0 if kind == "name" else 0.5))
+    if not prepared:
+        return []
+    out = []
+    for c in courses:
+        cname_key = collapse(c.name)
+        code_key = collapse(c.code) if c.code else ""
+        sig_words = [w for w in words(c.name) if len(w) >= 4 and w not in _STOP and w not in _GENERIC]
+        score = 0.0
+        for compact, ws, weight in prepared:
+            if code_key and code_key in compact:
+                score += 10 * weight
+            if cname_key and cname_key in compact:
+                score += 8 * weight
+            for alias in ALIASES.get(cname_key, ()):
+                if _alias_hit(alias, ws):
+                    score += 5 * weight
+                    break
+            hits = sum(1 for nw in sig_words if any(_hit(w, nw) for w in ws))
+            score += min(hits * 2.0 * weight, 4.0 if weight < 1 else 8.0)
+        # "English I" vs "English II": use the level written in the file name
+        name_words = words(c.name)
+        if name_words and name_words[-1] in ("i", "ii") and score > 0:
+            lvl = _level([w for _, ws, wt in prepared if wt == 1.0 for w in ws])
+            if lvl:
+                is_two = name_words[-1] == "ii"
+                score += 3 if (lvl == 2) == is_two else -3
+        if score > 0:
+            out.append((score, c))
+    out.sort(key=lambda item: (-item[0], item[1].id))
+    return out
+
+
+def decide(ranked: Sequence[tuple[float, object]]) -> tuple[Optional[object], bool, list]:
+    """-> (best course or None, confident?, top-3 suggestions)."""
+    if not ranked:
+        return None, False, []
+    top = ranked[0][0]
+    second = ranked[1][0] if len(ranked) > 1 else 0.0
+    confident = top >= 6 and (top - second >= 3 or second <= 0)
+    return ranked[0][1], confident, [c for _, c in ranked[:3]]
+
+
+# ---------------------------------------------------------------- kind
+_EXAM = ("exam", "midterm", "mid exam", "mid-exam", "final", "quiz", "test", "question", "past paper", "previous", "ፈተና")
+_SLIDE = ("slide", "lecture", "ppt", "presentation", "chapter")
+_NOTE = ("note", "handout", "summary", "worksheet", "assignment", "tutorial", "module", "ማስታወሻ")
+_BOOK = ("textbook", "book", "edition", "isbn", "volume")
+_EXAM_TEXT = ("time allowed", "answer all questions", "choose the best answer", "instruction:", "id no", "name:")
+
+
+def detect_kind(name: str, caption: str, text: str, pages: Optional[int], size: Optional[int], ext: str) -> tuple[str, str]:
+    label = f" {(name or '').lower()} {(caption or '').lower()} ".replace("_", " ").replace("-", " ")
+    if ext in (".ppt", ".pptx"):
+        return "slide", "presentation file"
+    if any(k in label for k in _EXAM):
+        return "exam", "name says exam"
+    if any(k in label for k in _SLIDE):
+        return "slide", "name says slides/chapter"
+    if any(k in label for k in _BOOK):
+        return "book", "name says book"
+    if sum(1 for k in _EXAM_TEXT if k in (text or "").lower()) >= 2:
+        return "exam", "content looks like an exam"
+    if any(k in label for k in _NOTE):
+        return "note" if not pages or pages < 150 else "book", "name says module/notes"
+    if pages and pages >= 200:
+        return "book", f"{pages} pages"
+    if size and size > 30 * 1024 * 1024:
+        return "book", "very large file"
+    return "note", "default"
+
+
+# ---------------------------------------------------------------- title
+_JUNK = re.compile(r"(https?://\S+|t\.me/\S+|@\w+|www\.\S+|\(\d+\)|\bcopy\b|\bscanned\b)", re.I)
+
+
+def detect_year(*texts: str) -> Optional[str]:
+    for t in texts:
+        m = re.search(r"\b(20[0-3]\d)\b", t or "")
+        if m:
+            return m.group(1)
+    return None
+
+
+def exam_stage(*texts: str) -> Optional[str]:
+    blob = " ".join(texts).lower()
+    if "mid" in blob:
+        return "Mid"
+    if "final" in blob:
+        return "Final"
+    if "quiz" in blob:
+        return "Quiz"
+    return None
+
+
+def clean_title(file_name: Optional[str], caption: Optional[str], course_name: str, kind: str, year: Optional[str], stage: Optional[str]) -> str:
+    base = re.sub(r"\.[A-Za-z0-9]{2,5}$", "", file_name or "")
+    base = _JUNK.sub(" ", base)  # junk first: "@some_channel" must go before underscores become spaces
+    base = base.replace("_", " ").replace("-", " ").replace(".", " ")
+    parts = []
+    for tok in base.split():
+        if re.fullmatch(r"\d{8,}", tok) or (len(tok) >= 12 and re.search(r"\d", tok) and re.search(r"[a-z]", tok, re.I)):
+            continue  # phone-number / hash-like junk
+        parts.append(tok)
+    title = " ".join(parts).strip()
+    if not title and caption:
+        title = _JUNK.sub(" ", caption.strip().split("\n")[0]).strip()
+    if not title:
+        title = f"{course_name} {kind}"
+    if title.isupper() or title.islower():
+        title = title.title()
+    if kind == "exam":
+        if stage and stage.lower() not in title.lower():
+            title += f" {stage}"
+        if year and year not in title:
+            title += f" {year}"
+    return re.sub(r"\s+", " ", title)[:80]
+
+
+def disambiguate(title: str, taken: set, *, year: Optional[str], pages: Optional[int], size: Optional[int]) -> str:
+    """Same title already used for a DIFFERENT file in this course/kind -> add what tells them apart."""
+    options = []
+    if year and year not in title:
+        options.append(f"{title} ({year})")
+    if pages:
+        options.append(f"{title} ({pages} p)")
+    if size:
+        options.append(f"{title} ({size / 1048576:.1f} MB)")
+    for candidate in options:
+        if norm_title_key(candidate) not in taken:
+            return candidate
+    n = 2
+    while norm_title_key(f"{title} ({n})") in taken:
+        n += 1
+    return f"{title} ({n})"
+
+
+# ---------------------------------------------------------------- near-duplicate fingerprint
+def simhash64(text: str) -> Optional[int]:
+    """64-bit SimHash of word 3-grams (signed, fits a BIGINT). None if there is too little text."""
+    ws = words(text)
+    if len(ws) < 40:
+        return None
+    vec = [0] * 64
+    for i in range(len(ws) - 2):
+        h = int.from_bytes(hashlib.blake2b(" ".join(ws[i : i + 3]).encode(), digest_size=8).digest(), "big")
+        for b in range(64):
+            vec[b] += 1 if (h >> b) & 1 else -1
+    out = 0
+    for b in range(64):
+        if vec[b] > 0:
+            out |= 1 << b
+    return out - (1 << 64) if out >= (1 << 63) else out
+
+
+def hamming(a: int, b: int) -> int:
+    return ((a ^ b) & ((1 << 64) - 1)).bit_count()
+
+
+NEAR_DUPLICATE_BITS = 3

app/services/doc_analyzer.py

@@ -0,0 +1,69 @@
+"""Read the first pages of a file to learn its text and page count. Sync: run in a thread."""
+import html
+import io
+import logging
+import os
+import re
+import zipfile
+from dataclasses import dataclass
+from typing import Optional
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class DocInfo:
+    text: str = ""
+    pages: Optional[int] = None
+
+
+def _pdf(data: bytes) -> DocInfo:
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        return DocInfo()
+    logging.getLogger("pypdf").setLevel(logging.ERROR)
+    reader = PdfReader(io.BytesIO(data))
+    pages = len(reader.pages)
+    chunks = []
+    for i in range(min(3, pages)):
+        chunks.append(reader.pages[i].extract_text() or "")
+    return DocInfo(text=" ".join(chunks), pages=pages)
+
+
+def _pptx(data: bytes) -> DocInfo:
+    with zipfile.ZipFile(io.BytesIO(data)) as z:
+        slides = sorted(
+            (n for n in z.namelist() if re.fullmatch(r"ppt/slides/slide\d+\.xml", n)),
+            key=lambda n: int(re.findall(r"\d+", n)[0]),
+        )
+        chunks = []
+        for name in slides[:3]:
+            xml = z.read(name).decode("utf-8", "ignore")
+            chunks.append(" ".join(html.unescape(t) for t in re.findall(r"<a:t>([^<]*)</a:t>", xml)))
+    return DocInfo(text=" ".join(chunks), pages=len(slides))
+
+
+def _docx(data: bytes) -> DocInfo:
+    with zipfile.ZipFile(io.BytesIO(data)) as z:
+        xml = z.read("word/document.xml").decode("utf-8", "ignore")
+    text = " ".join(html.unescape(t) for t in re.findall(r"<w:t[^>]*>([^<]*)</w:t>", xml))
+    return DocInfo(text=text)
+
+
+def analyze(file_name: Optional[str], data: bytes) -> DocInfo:
+    ext = os.path.splitext(file_name or "")[1].lower()
+    try:
+        if ext == ".pdf":
+            info = _pdf(data)
+        elif ext == ".pptx":
+            info = _pptx(data)
+        elif ext == ".docx":
+            info = _docx(data)
+        else:
+            return DocInfo()
+    except Exception as exc:  # encrypted / scanned / corrupt files must never break an upload
+        log.info("Could not read %s: %r", file_name, exc)
+        return DocInfo()
+    info.text = re.sub(r"\s+", " ", info.text)[:6000]
+    return info

app/services/smart_ingest.py

@@ -0,0 +1,151 @@
+"""One upload -> course, kind, clean title, duplicate checks, stored. Used by the admin upload handler."""
+import asyncio
+import hashlib
+import logging
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+
+from aiogram import Bot
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Course, Material
+from app.db.repos import library as repo
+from app.services import classifier as C
+from app.services import library as lib
+from app.services.doc_analyzer import DocInfo, analyze
+
+log = logging.getLogger(__name__)
+BOT_API_DOWNLOAD_LIMIT = 20 * 1024 * 1024  # a bot can only download files up to 20 MB
+
+
+@dataclass
+class Outcome:
+    status: str  # saved | review | near_duplicate | duplicate
+    material: Optional[Material] = None
+    course: Optional[Course] = None
+    kind: str = "note"
+    title: str = ""
+    why: str = ""
+    notes: list[str] = field(default_factory=list)
+    suggestions: list = field(default_factory=list)
+    duplicate_of: Optional[Material] = None
+
+
+async def _download(bot: Bot, file_id: str) -> Optional[bytes]:
+    try:
+        buf = await asyncio.wait_for(bot.download(file_id), timeout=45)
+        return buf.read() if buf else None
+    except Exception as exc:
+        log.info("Download for analysis failed: %r", exc)
+        return None
+
+
+async def ingest_file(
+    bot: Bot,
+    session: AsyncSession,
+    *,
+    uploader_id: int,
+    file_type: str,
+    file_id: str,
+    unique_id: str,
+    file_name: Optional[str],
+    size: Optional[int],
+    caption: Optional[str],
+    forced_course: Optional[Course] = None,
+    forced_kind: Optional[str] = None,
+    forced_title: Optional[str] = None,
+    backup_channel: int = 0,
+    message=None,
+) -> Outcome:
+    # 1) exact same Telegram file
+    existing = await repo.find_by_unique_id(session, unique_id)
+    if existing:
+        return Outcome("duplicate", duplicate_of=existing, why="same Telegram file")
+
+    # 2) read the content (only possible up to 20 MB) -> hash, text, pages
+    content_hash, info = None, DocInfo()
+    if file_type == "document" and size is not None and size <= BOT_API_DOWNLOAD_LIMIT:
+        data = await _download(bot, file_id)
+        if data:
+            content_hash = hashlib.sha256(data).hexdigest()
+            twin = await repo.find_by_hash(session, content_hash)
+            if twin:
+                return Outcome("duplicate", duplicate_of=twin, why="identical content, different name")
+            info = await asyncio.to_thread(analyze, file_name, data)
+
+    ext = os.path.splitext(file_name or "")[1].lower()
+    notes: list[str] = []
+
+    # 3) which course?
+    suggestions: list = []
+    confident = True
+    course = forced_course
+    if course is None:
+        courses = await lib.all_courses_cached(session)
+        sources = [(file_name or "", "name"), (caption or "", "name"), (info.text[:3000], "text")]
+        best, confident, suggestions = C.decide(C.rank_courses(courses, sources))
+        course = best if confident else None
+
+    # 4) which kind?
+    if forced_kind:
+        kind, why = forced_kind, "set by you"
+    else:
+        kind, why = C.detect_kind(file_name or "", caption or "", info.text, info.pages, size, ext)
+
+    # 5) a clean title (+ year / Mid / Final for exams)
+    year = C.detect_year(file_name or "", caption or "", info.text[:1500])
+    stage = C.exam_stage(file_name or "", caption or "") if kind == "exam" else None
+    course_name = course.name if course else "Unsorted"
+    title = forced_title or C.clean_title(file_name, caption, course_name, kind, year, stage)
+
+    # 6) near-duplicate: same text, different bytes (re-scan, re-export, different compression)
+    sig = C.simhash64(info.text)
+    near = None
+    if sig is not None:
+        for mid, other_sig, other_title in await repo.all_sigs(session):
+            if C.hamming(sig, other_sig) <= C.NEAR_DUPLICATE_BITS:
+                near = await repo.get_material(session, mid)
+                break
+
+    # 7) same title, DIFFERENT file in this course/kind -> keep both, tell them apart
+    if course is not None:
+        taken = {C.norm_title_key(t) for t in await repo.titles_in_course(session, course.id, kind)}
+        if C.norm_title_key(title) in taken:
+            new_title = C.disambiguate(title, taken, year=year, pages=info.pages, size=size)
+            notes.append(f"same title as an existing different file -> saved as “{new_title}”")
+            title = new_title
+
+    status = "saved"
+    if near is not None:
+        status = "near_duplicate"
+    elif course is None:
+        status = "review"
+
+    material = await repo.add_material(
+        session,
+        course_id=course.id if course else None,
+        kind=kind,
+        title=title[:250],
+        file_id=file_id,
+        file_type=file_type,
+        file_unique_id=unique_id,
+        uploader_id=uploader_id,
+        content_hash=content_hash,
+        text_sig=sig,
+        pages=info.pages,
+        size=size,
+        source_name=(file_name or "")[:250] or None,
+        status="ok" if status == "saved" else "review",
+    )
+    if backup_channel and message is not None:
+        try:
+            copied = await message.copy_to(backup_channel)
+            material.library_chat_id, material.library_message_id = backup_channel, copied.message_id
+        except Exception as exc:  # the backup copy is optional
+            log.warning("Library backup copy failed: %s", exc)
+
+    return Outcome(
+        status, material=material, course=course, kind=kind, title=material.title, why=why,
+        notes=notes, suggestions=suggestions, duplicate_of=near,
+    )

requirements.txt

@@ -3,3 +3,4 @@ sqlalchemy[asyncio]>=2.0
 asyncpg>=0.29
 aiosqlite>=0.20
 pydantic-settings>=2.2
+pypdf>=4

tests/test_logic.py

@@ -80,6 +80,9 @@ import random  # noqa: E402
 from app.core import season  # noqa: E402
 from app.services import xp_rules as R  # noqa: E402
 from app.services.fuzzy import match_score  # noqa: E402
+from types import SimpleNamespace as NS  # noqa: E402
+from app.core.seed_courses import FRESHMAN_COURSES  # noqa: E402
+from app.services import classifier as C  # noqa: E402
 from app.services.season_logic import Candidate, draw, flags, rank_candidates  # noqa: E402
 from datetime import datetime, timedelta  # noqa: E402
 
@@ -152,6 +155,100 @@ def test_fuzzy_search_tolerates_typos():
     assert match_score(["physics"], physics) > match_score(["phyy"], physics)
 
 
+COURSES = [
+    NS(id=i, stream=None if st == "all" else st, year=y, code=code, name=name)
+    for i, (st, y, _sem, code, name) in enumerate(FRESHMAN_COURSES, start=1)
+]
+
+
+def _pick(file_name, caption="", text=""):
+    sources = [(file_name, "name"), (caption, "name"), (text, "text")]
+    best, confident, sugg = C.decide(C.rank_courses(COURSES, sources))
+    return (best.name if best else None), confident
+
+
+def test_classifier_courses():
+    assert _pick("Math_1003_Module.pdf") == ("Mathematics for Natural Sciences", True)
+    assert _pick("General Physics Final Exam 2016.pdf") == ("General Physics", True)
+    assert _pick("Entrepreneurship ppt chapter 1.pptx") == ("Entrepreneurship", True)
+    assert _pick("Eng 1002 - Communicative English II notes.pdf")[0] == "Communicative English Language Skills II"
+    assert _pick("english skills I handout.pdf")[0] == "Communicative English Language Skills I"
+    assert _pick("chem module.pdf")[0] == "General Chemistry"
+    assert _pick("calculus chapter 3.pdf")[0] == "Mathematics for Natural Sciences"
+    assert _pick("ታሪክ ማስታወሻ.pdf")[0] == "History of Ethiopia and the Horn"
+
+
+def test_classifier_unsure_goes_to_review():
+    assert _pick("IMG_20261014_final.pdf")[1] is False
+    assert _pick("random_file123.pdf") == (None, False)
+    assert _pick("Mathematics module.pdf")[1] is False  # three math courses tie: ask the admin
+
+
+def test_classifier_uses_pdf_text_when_name_is_useless():
+    text = "Debre Berhan University General Chemistry module atoms and molecules chapter one"
+    assert _pick("scan0001.pdf", text=text)[0] == "General Chemistry"
+
+
+def test_kind_detection():
+    assert C.detect_kind("Phys final exam.pdf", "", "", None, None, ".pdf")[0] == "exam"
+    assert C.detect_kind("lecture1.pptx", "", "", None, None, ".pptx")[0] == "slide"
+    assert C.detect_kind("Math module.pdf", "", "", 120, None, ".pdf")[0] == "note"
+    assert C.detect_kind("Math module.pdf", "", "", 260, None, ".pdf")[0] == "book"
+    assert C.detect_kind("scan.pdf", "", "Time allowed: 2 hours. Answer all questions", None, None, ".pdf")[0] == "exam"
+    assert C.detect_kind("whatever.pdf", "", "", None, None, ".pdf")[0] == "note"
+
+
+def test_titles():
+    assert C.clean_title("Phys_1009_FINAL_EXAM_(1).pdf", None, "General Physics", "exam", "2016", "Final") == "Phys 1009 FINAL EXAM 2016"
+    assert C.clean_title("@some_channel 123456789012 notes.pdf", None, "X", "note", None, None) == "Notes"
+    assert C.clean_title(None, None, "General Physics", "note", None, None) == "General Physics note"
+
+
+def test_same_title_different_files_are_told_apart():
+    taken = {C.norm_title_key("Math Module")}
+    assert C.disambiguate("Math Module", taken, year=None, pages=245, size=5_000_000) == "Math Module (245 p)"
+    taken.add(C.norm_title_key("Math Module (245 p)"))
+    assert C.disambiguate("Math Module", taken, year=None, pages=245, size=5_000_000) == "Math Module (4.8 MB)"
+    taken.add(C.norm_title_key("Math Module (4.8 MB)"))
+    assert C.disambiguate("Math Module", taken, year=None, pages=245, size=5_000_000) == "Math Module (2)"
+
+
+def test_simhash_near_duplicates():
+    base = " ".join(f"word{i % 37} term{i % 11}" for i in range(400))
+    edited = base + " extra footer line added by a scanner app"
+    other = " ".join(f"zeta{i % 53} omega{i % 7}" for i in range(400))
+    a, b, c = C.simhash64(base), C.simhash64(edited), C.simhash64(other)
+    assert C.hamming(a, b) <= C.NEAR_DUPLICATE_BITS
+    assert C.hamming(a, c) > 10
+    assert C.simhash64("too short") is None
+
+
+def test_doc_analyzer_reads_pptx_and_docx():
+    import io
+    import zipfile
+
+    from app.services.doc_analyzer import analyze
+
+    def make(files: dict) -> bytes:
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as z:
+            for name, content in files.items():
+                z.writestr(name, content)
+        return buf.getvalue()
+
+    pptx = make({
+        "ppt/slides/slide1.xml": "<a:t>General Chemistry</a:t><a:t>Atoms &amp; molecules</a:t>",
+        "ppt/slides/slide2.xml": "<a:t>Chapter one</a:t>",
+        "ppt/slides/slide10.xml": "<a:t>Last</a:t>",
+    })
+    info = analyze("chem.pptx", pptx)
+    assert info.pages == 3 and "General Chemistry" in info.text and "Atoms & molecules" in info.text
+    docx = make({"word/document.xml": "<w:p><w:t>Entrepreneurship</w:t><w:t xml:space='preserve'> module</w:t></w:p>"})
+    assert "Entrepreneurship module" in analyze("m.docx", docx).text
+    assert analyze("broken.pdf", b"not a pdf").text == ""  # never raises
+    assert analyze("pic.png", b"x").pages is None
+
+
 if __name__ == "__main__":
     for name, fn in list(globals().items()):
         if name.startswith("test_"):