Refactor Minerva import script to use metadata and improve functionality

- Removed hardcoded video data and replaced it with dynamic loading from `data/minerva_metadata.json`.
- Updated the `phrases` function to handle metadata and return structured data for each reviewed line.
- Modified command-line arguments to allow importing specific YouTube IDs.
- Enhanced validation checks to ensure the integrity of imported lines.
- Updated tests to reflect changes in the import logic and ensure coverage for new functionality.

app/lab.py

@@ -18,6 +18,10 @@ The IPA follows the bank's (phrases.py), and a word the bank has keeps the bank'
 """
 from __future__ import annotations
 
+import json
+from functools import lru_cache
+from pathlib import Path
+
 from .align import POOLS
 from .scorer import LETTER_GOOD
 from .tips import VOWELS
@@ -96,6 +100,50 @@ ITEMS = [
     _item("vow-miin", "vowels", "iː", "מי", ("مين", "m iː n"), pair="i / iː"),
 ]
 BY_ID = {item["id"]: item for item in ITEMS}
+MINERVA_METADATA = Path(__file__).resolve().parent.parent / "data" / "minerva_metadata.json"
+
+
+@lru_cache(maxsize=1)
+def minerva_metadata() -> tuple[dict, ...]:
+    """The reviewed Minerva captions used by both the lab page and every rescore.
+
+    Supabase owns the recording and its historical result; this file owns the sentence, timing and expected phones.
+    Keeping that boundary explicit means an OCR correction takes effect on the next server start without reimporting
+    audio or editing database rows.
+    """
+    raw = json.loads(MINERVA_METADATA.read_text(encoding="utf-8"))
+    if not isinstance(raw, list):
+        raise ValueError("Minerva metadata must be a JSON array")
+    videos, seen = [], set()
+    for video in raw:
+        video_id = str(video.get("youtube_id") or "")
+        segments = video.get("segments")
+        if not video_id or video_id in seen or not isinstance(segments, list):
+            raise ValueError(f"invalid Minerva video metadata: {video_id or '<missing id>'}")
+        seen.add(video_id)
+        lines = []
+        for line, segment in enumerate(segments, 1):
+            timing = segment.get("voice_time_seconds") or {}
+            required = (segment.get("arabic"), segment.get("hebrew_transliteration"),
+                        segment.get("hebrew"), segment.get("expected_phonemes"))
+            if not all(isinstance(value, str) and value.strip() for value in required):
+                raise ValueError(f"incomplete Minerva metadata: {video_id}:{line:02d}")
+            try:
+                start, end = float(timing["start"]), float(timing["end"])
+            except (KeyError, TypeError, ValueError) as exc:
+                raise ValueError(f"invalid Minerva timing: {video_id}:{line:02d}") from exc
+            if end <= start:
+                raise ValueError(f"invalid Minerva timing: {video_id}:{line:02d}")
+            lines.append({"item": f"minerva:{video_id}:{line:02d}", "line": line, "start": start, "end": end,
+                          "ar": required[0], "tr": required[1], "he": required[2], "ipa": required[3]})
+        videos.append({"id": video_id, "title": str(video.get("title_hebrew") or video_id),
+                       "filename": str(video.get("file_name") or ""), "segments": tuple(lines)})
+    return tuple(videos)
+
+
+def minerva_segments() -> dict[str, dict]:
+    """Reviewed Minerva segments by their persisted lab item key."""
+    return {segment["item"]: segment for video in minerva_metadata() for segment in video["segments"]}
 
 
 def phrase(item: dict) -> dict:
@@ -112,6 +160,10 @@ def sample_phrase(sample: dict) -> dict | None:
     item = BY_ID.get(sample.get("item"))
     if item is not None:
         return phrase(item)
+    segment = minerva_segments().get(sample.get("item"))
+    if segment is not None:
+        return {"cat": "lab", "key": segment["item"], "he": segment["he"],
+                "variants": [(segment["ar"], segment["ipa"])]}
     ar, ipa = sample.get("ar"), sample.get("ipa")
     if not ar or not ipa:
         return None

app/main.py

@@ -796,37 +796,35 @@ def create_app(scorer: Scorer | None = None, tts: SawtakTTS | None = None,
 
     @app.get("/api/lab/minerva")
     def lab_minerva(a: SupabaseAuth = Depends(teacher_only)):
-        """The imported Minerva pilot, grouped into videos for the lab's native-speech runner."""
-        titles = {"U8aBRd3-wlM": "במכולת", "OuAwxXUZBU4": "אסיפת הורים", "V2y6LqnoXTo": "הקדרה"}
-        files = {"U8aBRd3-wlM": "במכולת ＊ פִי א-דֻכַּאן ＊  في الدّكّان ＊ קטע שיחה בערבית מדוברת [U8aBRd3-wlM].webm",
-                 "OuAwxXUZBU4": "אסיפת הורים ＊ אִגתִמַאע אַהַאלִי ＊ إجتماع أهالي ＊ קטע שיחה בערבית מדוברת [OuAwxXUZBU4].webm",
-                 "V2y6LqnoXTo": "הקְדֵרָה הולידה קְדֵרָה ＊ אִ-טֻנגָ'רָה וִלדַת טֻנגָ'רָה ＊ الطّنجرة ولدت طنجرة ＊ סיפור בערבית מדוברת [V2y6LqnoXTo].mkv"}
+        """The imported Minerva audio joined with its reviewed JSON sentence metadata."""
         try:
             samples = a.lab_samples(full=True)
         except AuthUnavailable as e:
             raise lab_error(e)
+        manifest = lab.minerva_metadata()
+        definitions = {segment["item"]: (video, segment) for video in manifest for segment in video["segments"]}
         videos: dict[str, dict] = {}
         for s in samples:
-            match = re.fullmatch(r"minerva:([\w-]{11}):(\d+)", s.get("item") or "")
-            if (s.get("speaker") or "") != "מינרווה" or not match:
+            definition = definitions.get(s.get("item"))
+            if (s.get("speaker") or "") != "מינרווה" or definition is None:
                 continue
-            video_id, line = match.group(1), int(match.group(2))
-            timing = re.fullmatch(rf"minerva:{re.escape(video_id)}:([0-9.]+)-([0-9.]+)", s.get("device") or "")
+            meta_video, segment = definition
+            video_id = meta_video["id"]
             stored = s.get("result") or {}
             excluded = stored.get("_lab_excluded_words") or {}
             if not isinstance(excluded, dict):
                 excluded = {}
-            video = videos.setdefault(video_id, {"id": video_id, "title": titles.get(video_id, video_id),
-                                                  "filename": files.get(video_id, ""), "samples": []})
-            video["samples"].append({"id": s["id"], "item": s["item"], "line": line,
-                                     "start": float(timing.group(1)) if timing else None,
-                                     "end": float(timing.group(2)) if timing else None,
-                                     "ar": s.get("ar") or "", "ipa": s.get("ipa") or "",
+            video = videos.setdefault(video_id, {"id": video_id, "title": meta_video["title"],
+                                                  "filename": meta_video["filename"], "samples": []})
+            video["samples"].append({"id": s["id"], "item": s["item"], "line": segment["line"],
+                                     "start": segment["start"], "end": segment["end"],
+                                     "ar": segment["ar"], "tr": segment["tr"], "he": segment["he"],
+                                     "ipa": segment["ipa"],
                                      "score": s.get("score"), "heard_ipa": s.get("heard_ipa") or "",
                                      "flagged": s.get("flagged") or [],
                                      "unrelated": bool(stored.get("unrelated")), "audio": bool(s.get("audio")),
                                      "excluded_words": {str(k): str(v) for k, v in excluded.items()}})
-        order = list(titles)
+        order = [video["id"] for video in manifest]
         out = list(videos.values())
         out.sort(key=lambda v: order.index(v["id"]) if v["id"] in order else len(order))
         for video in out:

scripts/import_minerva.py

@@ -1,13 +1,12 @@
-#!/usr/bin/env python3
-"""Import the three-video Minerva pilot into the lab as native, correctly spoken samples.
+"""Import the Minerva videos into the lab as native, correctly spoken samples.
 
-The videos have no subtitle stream.  PIL/Tesseract are deliberately not dependencies: the reviewed, pointed-Hebrew
-captions and the detected time spans are the pilot manifest below.  ``app.texts.text_phrases`` remains the single
-source of Arabic spelling and IPA, exactly as it is for teacher-authored texts.
+The videos have no subtitle stream: their captions are burned in.  ``data/minerva_metadata.json`` holds the reviewed
+caption of every line (Arabic, pointed-Hebrew transliteration, translation, expected IPA) with its spoken time span,
+so the importer only cuts, scores and uploads.  The lab page and every rescore read the same file.
 
 Usage:
     python scripts/import_minerva.py --dry-run
-    python scripts/import_minerva.py --email teacher@example.com
+    python scripts/import_minerva.py --email teacher@example.com [--video <youtube-id> ...]
 
 Existing ``minerva:<video-id>:<line>`` rows are skipped, making an interrupted import safe to resume.
 """
@@ -18,7 +17,6 @@ import os
 import subprocess
 import sys
 import uuid
-from dataclasses import dataclass
 from datetime import date
 from pathlib import Path
 
@@ -31,101 +29,11 @@ from app.auth import AuthUnavailable, load_auth  # noqa: E402
 from app.main import MODEL_ID, build_version  # noqa: E402
 from app.reference import References  # noqa: E402
 from app.scorer import decode_audio, load_scorer  # noqa: E402
-from app.texts import text_phrases  # noqa: E402
 from app.tts import VOICES, load_tts  # noqa: E402
 
 SPEAKER = "מינרווה"
 
 
-@dataclass(frozen=True)
-class Video:
-    youtube_id: str
-    title: str
-    lines: tuple[tuple[float, float, str], ...]
-
-
-# Caption changes were measured at 5 fps.  The first story line is its spoken title, before the first full caption.
-PILOT = (
-    Video("U8aBRd3-wlM", "במכולת", (
-        (9.0, 10.8, "פִיה עִנְדַכּ חַלִיב טַארֶה?"),
-        (10.8, 12.8, "אַה, גֻ'וַּא פִי ת-תַּלַּאגֶ'ה, עַלַא אִידֶכּ לְ-יַמִין."),
-        (12.8, 14.2, "מֻמְתַאז."),
-        (14.2, 17.6, "מֻמְכּן תְחֻטְלִי כַּמַאן וַקִיַּת גֻ'בְּנֶה צַפְרַא?"),
-        (17.6, 21.8, "אַה טַבְּעַן, בַּס תְוַאנִי אַחְ'לַצ מַעַאה לְ-חִסַאב."),
-        (21.8, 24.6, "תְפַדַّ'לִי."),
-        (24.6, 27.6, "הַאדַא הֻוֶּ? אַיוַה, קַדֵּיש לְ-חִסַאב?"),
-        (27.6, 30.4, "חַ'מְסֶה וְעִשְרִין שֵיקֶל. תְפַדַّל."),
-        (30.4, 33.0, "מַעְכִּיש אַצְעַ'ר? מַעִיש צַרַאפֶת מִיֶּה."),
-        (33.0, 35.4, "לַחְטַ'ה אַפְחַצ."),
-        (35.4, 39.2, "לַא וַאללַה מַעִיש, פִיה בַּס עִשְרִין אִזַא בִּיפִידוּכּ."),
-        (39.2, 42.8, "לַא מַעְלֵיש, מֻש מֻהִם, בַּדַבֶּרְהַא."),
-        (42.8, 44.6, "תְפַדַّלִי אִלְ-בַּאקִי, מְעַוַّדִין."),
-        (44.6, 46.4, "תִסְלַם, יַעְטִיכּ אִלְ-עַאפְיֶה!"),
-        (46.4, 48.4, "אַללַה יְעַאפִיכּ, מַע אִלְ-סַלַאמֶה!"),
-    )),
-    Video("OuAwxXUZBU4", "אסיפת הורים", (
-        (8.8, 11.8, "תַמַאם, כִּיפֶכּ? שוּ צַאר מַעֶכּ?"),
-        (11.8, 14.2, "בְּסֵידֶר, אִסַּא טִלְעֵת מִן אִלְ-מַדְרַסֶה."),
-        (14.2, 17.4, "קַעַדֶת מַע אִלְ-מֻרַבִּיֶה וּבַּאקִי לְ-מֻעַלִּמַאת."),
-        (17.4, 21.2, "מִיס נַוַאל בְּתְקוּל אִנּוּ רַבִּיע וַלַד בִּגַ'נֵּן וְעַקְלוֹ כְּבִּיר."),
-        (21.2, 24.4, "מֻאַדַّבּ, בְּחִבּ יְסַאעֶד עֵ'ירוֹ, שַאטֶר וּפַתֶח."),
-        (24.4, 25.8, "בַּס מִתְל אִלְ-עַאדֶה."),
-        (25.8, 29.4, "לַאזֶם יִשְתַעֵ'ל אַכְּתַר עַלַא מַוְדוּע אִלְ-תַרְכִּיז."),
-        (29.4, 31.0, "כְּתִיר בִּשַתּ וּבִּסְרַח."),
-        (31.0, 35.8, "וְהַאדַא בִּחַלִּיה יַעְמַל אַעְ'לַאט זְעִ'ירֶה פִי דִּרַאסֶה."),
-        (35.8, 37.4, "הֵי וּבַּאקִי לְ-מֻעַלִּמַאת מִנְתַבְּהַאת."),
-        (37.4, 39.2, "אִנּוּ מֻשְכִּלְתוֹ מֻש עַדַם פַהֶם."),
-        (39.2, 41.0, "בִּלְ-עַכֶּס, עַרְבִּיהֶן שַאטֶר כְּתִיר."),
-        (41.0, 46.0, "בַּס מֻשְכִּלְתוֹ קִלֶּת תַרְכִּיז."),
-        (46.0, 49.4, "אִדַ'א הֵיכּ אִלְ-וַדְע פִש עַלַא שוּ נִקְלַק."),
-        (49.4, 52.6, "הֻוֶּ מְתַחַסֵּן כְּתִיר וּמֻש לַאזֶם נְדַקֵּק עַלַא כֻּל אִשִי."),
-        (52.6, 56.4, "רַאח יִתְעַ'יַּר מַע אִלְ-וַקְת וִלְ-גִ'יל אִלוֹ חַקּוֹ."),
-        (56.4, 59.8, "מַדְבּוּט. אַנַא בַּקוּל לַאזֶם נְגִ'יבְלוֹ הַדִיֶּה זְעִ'ירֶה."),
-        (59.8, 61.6, "מִתְל מַא וַעַדְנַאה."),
-        (61.6, 64.2, "אַה אַכִּיד, אִלְ-יוֹם מִנְטַלַע כֻּלְנַא מִשְוַאר."),
-        (64.2, 65.2, "וּמְנְגִ'יבְלוֹ שַעְ'לֶה."),
-        (65.2, 69.2, "חַ'לִּיה יְחִס אִנּוּ אִחְנַא מְקַדְּרִין תַעַבּוֹ וְגִ'דִּיֶּתוֹ."),
-        (69.2, 74.2, "טַיֵּב מֻמְתַאז, בְּלַא מַנְחַכִּי, בַּאי."),
-    )),
-    Video("V2y6LqnoXTo", "הקדרה", (
-        # The title is spoken over the intro card; 5.6-6.4 is the silent transition into the first story caption.
-        (0.4, 5.6, "אִלְ-טֻנְגַ'רַה וִלְדַת טֻנְגַ'רַה."),
-        (6.4, 10.2, "מַרַּה טַלַבּ גַ'מַא מִן גַ'ארוֹ יַעְטִיה טֻנְגַ'רַה כְּבִּירֶה."),
-        (10.2, 14.2, "עַשַאן יִקְדַר יִטְבַּח וְיִטְעַמִּי מַעַאזִימוֹ לְ-כֻּתַאר."),
-        (14.2, 17.2, "אִלִּי גַ'איִין עִנְדוֹ עַלַא אִלְ-עַ'דַא."),
-        (17.2, 21.0, "אִלְ-גַ'אר וַאפַק וְאַעְטַאה טֻנְגַ'רַה לְגַ'מַא."),
-        (21.0, 25.0, "בַּעְד אַכַּם יוֹם רַגַ'ע גַ'מַא אִלְ-טֻנְגַ'רַה לְ-כְּבִּירֶה."),
-        (25.0, 28.6, "וּבְּקַלְבְּהַא טֻנְגַ'רַה זְעִ'ירוֹנֶה."),
-        (28.6, 31.8, "סַאַל אִלְ-גַ'אר עַן הַאי טֻנְגַ'רַה לְ-זְעִ'ירֶה."),
-        (31.8, 36.2, "וְגַ'מַא פַסַּרְלוֹ אִנְהַא אִנְוַלַדַת מִן אִלְ-טֻנְגַ'רַה לְ-כְּבִּירֶה."),
-        (36.2, 40.4, "פִי לְ-פַתְרַה אִלִּי כַּאנֶת אִלְ-טֻנְגַ'רַה לְ-כְּבִּירֶה עִנְדוֹ."),
-        (40.4, 42.2, "קִבֵּל אִלְ-גַ'אר מִן גַ'מַא לְ-עַ'בִּי."),
-        (42.2, 45.8, "הַאי לְ-בִּנְת לְ-זְעִ'ירֶה בְּכֻל פַרַח וּסֻרוּר."),
-        (45.8, 48.4, "וּפִי קַלְבּוֹ תְחַוַּת עַלַא גַ'מַא."),
-        (48.4, 51.2, "אִנּוּ וַאחַד אַהְבַּל מַעְתוֹה."),
-        (51.2, 54.2, "בַּעְד אַכַּם יוֹם אַגַ'א גַ'מַא עִנְד גַ'ארוֹ."),
-        (54.2, 58.4, "יִטְלֻבּ מִנּוֹ יִמְסִכּוֹ אִלְ-טֻנְגַ'רַה כַּמַאן מַרַּה."),
-        (58.4, 61.0, "פְרִח אִלְ-גַ'אר וְקַאל פִי קַלְבּוֹ."),
-        (61.0, 64.0, "הַלַק בְּתוּלַד אִלְ-טֻנְגַ'רַה כַּמַאן בִּנְת."),
-        (64.0, 68.0, "וּמִין בִּעְרַף, בַּלְכִּי הַאי לְ-מַרַּה וַלַדַת תוֹאַם."),
-        (68.0, 72.6, "וְהֵיכּ מַסַכּ אִלְ-גַ'אר טֻנְגַ'רְתוֹ לְגַ'מַא."),
-        (72.6, 76.4, "מַרְקַת עִדֶּת אַיַּאם וְגַ'מַא מַא רַגַ'עְש אִלְ-טֻנְגַ'רַה."),
-        (76.4, 79.4, "עַשַאן הֵיכּ רַאח אִלְ-גַ'אר לְעִנְד גַ'מַא."),
-        (79.4, 81.8, "וְטַלַבּ מִנּוֹ יִרַגְ'עְהַא."),
-        (81.8, 85.6, "בַּס גַ'מַא קַאלְלוֹ אִנּוּ אִלְ-טֻנְגַ'רַה מַאתַת."),
-        (85.6, 88.0, "אִלְ-גַ'אר תְעַצַּב כְּתִיר וְקַאל."),
-        (88.0, 90.6, "כִּיף בְּתִסְמַח לְנַפְסַכּ תִתְחַוַּת עַלַיّ?"),
-        (90.6, 93.6, "שוּ? אִלְ-טֻנְגַ'רַה כַּאיְנֶה כַּאִן חַיّ?"),
-        (93.6, 95.8, "בְּתְעִיש וּבְּתְמוּת?"),
-        (95.8, 97.8, "גַ'מַא רַד עַלֵיה וְקַאל."),
-        (97.8, 101.0, "לַא סַמַח אַללַה, אִתְחַוַּת עַלַא שַחְ'צ מִתְלַכּ."),
-        (101.0, 104.4, "בַּס אִדַ'א מֻמְכִּן אִלְ-טֻנְגַ'רַה תוּלַד."),
-        (104.4, 106.4, "מַעְנַאה מֻמְכִּן אִנְהַא תְמוּת."),
-        (106.4, 113.8, "וְהַאדִי נִהַאיַת אִלְ-קִצַּה."),
-    )),
-)
-
-
 def source_file(folder: Path, youtube_id: str) -> Path:
     found = sorted(folder.glob(f"*{youtube_id}*"))
     if len(found) != 1:
@@ -142,16 +50,16 @@ def wav_segment(video: Path, start: float, end: float) -> bytes:
     return done.stdout
 
 
-def phrases() -> tuple[list[tuple[Video, int, float, float, dict]], list[str]]:
-    out, guessed = [], []
-    for video in PILOT:
-        for number, (start, end, tr) in enumerate(video.lines, 1):
-            made, new = text_phrases(f"minerva:{video.youtube_id}", [[f"{video.title} {number}", tr]])
-            phrase = made[0]
-            phrase["key"] = f"minerva:{video.youtube_id}:{number:02d}"
-            out.append((video, number, start, end, phrase))
-            guessed.extend(x for x in new if x not in guessed)
-    return out, guessed
+def phrases(only: set[str] | None = None) -> list[tuple[dict, dict, dict]]:
+    """(video, segment, phrase) for every reviewed line, optionally limited to some YouTube ids."""
+    out = []
+    for video in lab.minerva_metadata():
+        if only and video["id"] not in only:
+            continue
+        for segment in video["segments"]:
+            out.append((video, segment, {"cat": "lab", "key": segment["item"], "he": segment["he"],
+                                         "variants": [(segment["ar"], segment["ipa"])]}))
+    return out
 
 
 def default_email() -> str:
@@ -164,21 +72,19 @@ def main() -> int:
     ap.add_argument("--videos-dir", type=Path, default=Path("/home/eyalshw/Videos/minerva"))
     ap.add_argument("--email", default="", help="teacher email recorded as the importer (or MINERVA_IMPORT_EMAIL)")
     ap.add_argument("--dry-run", action="store_true", help="validate videos, segments and transcripts; do not score/upload")
-    ap.add_argument("--verify", action="store_true", help="verify the 70 remote rows and audio names; do not score/upload")
+    ap.add_argument("--video", action="append", default=[], help="import only this YouTube id (repeatable)")
+    ap.add_argument("--verify", action="store_true", help="verify the remote rows and audio names; do not score/upload")
     ap.add_argument("--no-baseline", action="store_true", help="score without the native TTS baseline")
     args = ap.parse_args()
     load_dotenv()
 
-    entries, guessed = phrases()
-    if [len(v.lines) for v in PILOT] != [15, 22, 33] or len(entries) != 70:
-        raise RuntimeError("the Minerva pilot must stay exactly 15 + 22 + 33 = 70 lines")
-    files = {v.youtube_id: source_file(args.videos_dir, v.youtube_id) for v in PILOT}
-    for video, number, start, end, phrase in entries:
-        if end <= start or not phrase["variants"]:
-            raise ValueError(f"invalid line {video.youtube_id}:{number}")
-    print(f"validated {len(entries)} lines in {len(files)} videos")
-    if guessed and args.dry_run:
-        print(f"words read by rule ({len(guessed)}): {' '.join(guessed)}")
+    entries = phrases(set(args.video) or None)
+    if not entries:
+        print("no Minerva lines selected", file=sys.stderr)
+        return 2
+    files = {v["id"]: source_file(args.videos_dir, v["id"]) for v, _, _ in entries}
+    total = len(entries)
+    print(f"validated {total} lines in {len(files)} videos")
     if args.dry_run:
         return 0
 
@@ -197,29 +103,29 @@ def main() -> int:
         print(f"lab is unavailable: {e}", file=sys.stderr)
         return 2
     if args.verify:
-        expected = {phrase["key"] for _, _, _, _, phrase in entries}
+        expected = {phrase["key"] for _, _, phrase in entries}
         audio = sum(bool(s.get("audio")) for s in remote if s.get("item") in expected)
         missing = sorted(expected - existing)
-        extra = sorted(existing - expected)
-        print(f"remote Minerva pilot: {len(expected & existing)}/70 rows, {audio}/70 audio files named")
+        extra = sorted(existing - expected) if not args.video else []
+        print(f"remote Minerva: {len(expected & existing)}/{total} rows, {audio}/{total} audio files named")
         if missing:
             print(f"missing: {' '.join(missing)}")
         if extra:
             print(f"other Minerva rows: {' '.join(extra)}")
-        return 0 if not missing and audio == 70 else 1
+        return 0 if not missing and audio == total else 1
 
     model = os.environ.get("MODEL_ID", MODEL_ID)
     scorer = load_scorer(model)
     tts = None if args.no_baseline else load_tts()
     refs = References(scorer, tts, list(VOICES.values())) if tts else None
     saved = skipped = 0
-    for n, (video, number, start, end, phrase) in enumerate(entries, 1):
-        item = phrase["key"]
+    for n, (video, segment, phrase) in enumerate(entries, 1):
+        item, start, end = phrase["key"], segment["start"], segment["end"]
         if item in existing:
             skipped += 1
-            print(f"[{n:02d}/70] skip {item}")
+            print(f"[{n}/{total}] skip {item}")
             continue
-        data = wav_segment(files[video.youtube_id], start, end)
+        data = wav_segment(files[video["id"]], start, end)
         wav = decode_audio(data)
         baseline = refs.get(item, phrase) if refs else []
         result = scorer.score(wav, phrase, baseline, diagnostics=True)
@@ -227,15 +133,15 @@ def main() -> int:
             raise RuntimeError(f"{item}: scoring failed: {result['error']}")
         row = {"email": email, "item": item, "ar": result["variant"]["ar"], "ipa": result["variant"]["ipa"],
                "speaker": SPEAKER, "label": "correct", "mistake": None,
-               "device": f"minerva:{video.youtube_id}:{start:.1f}-{end:.1f}", "score": result["score"],
+               "device": f"minerva:{video['id']}:{start:.1f}-{end:.1f}", "score": result["score"],
                "heard_ipa": result["heard_ipa"], "flagged": lab.flagged_sounds(result), "result": result,
                "model": model, "version": build_version()}
-        name = f"{date.today():%Y-%m}/minerva-{video.youtube_id}-{number:02d}-{uuid.uuid4().hex}.wav"
+        name = f"{date.today():%Y-%m}/minerva-{video['id']}-{segment['line']:02d}-{uuid.uuid4().hex}.wav"
         if not auth.add_lab_sample(row, data, "audio/wav", name):
             raise RuntimeError(f"{item}: row saved but audio upload failed; stop before creating more incomplete rows")
         saved += 1
-        print(f"[{n:02d}/70] saved {item}: {result['score']}")
-    print(f"Minerva pilot complete: {saved} saved, {skipped} already present")
+        print(f"[{n}/{total}] saved {item}: {result['score']}")
+    print(f"Minerva import complete: {saved} saved, {skipped} already present")
     return 0
 
 

tests/test_lab.py

@@ -238,7 +238,12 @@ def test_minerva_lab_groups_imported_segments_by_video():
         d = c.get("/api/lab/minerva", headers=teacher).json()
         assert len(d["videos"]) == 1 and d["videos"][0]["id"] == "U8aBRd3-wlM"
         assert [s["line"] for s in d["videos"][0]["samples"]] == [1, 2]
-        assert d["videos"][0]["samples"][0]["start"] == 1.0
+        # Sentence data comes from data/minerva_metadata.json, not the deliberately different stored row above.
+        first = d["videos"][0]["samples"][0]
+        assert first["start"] == 9.0 and first["end"] == 10.8
+        assert first["ar"] == "فيه عندك حليب طارة؟"
+        assert first["ipa"] == "f iː h | ʕ i n d a k | ħ a l iː b | tˤ aː r e"
+        assert first["tr"] == "פִיה עִנְדַכּ חַלִיב טַארֶה?" and first["he"] == "יש לך חלב טרי?"
         audio = c.get("/api/lab/samples/1/audio", headers=teacher)
         assert audio.status_code == 200 and audio.headers["content-type"] == "audio/wav"
         assert c.get("/api/lab/samples/99/audio", headers=teacher).status_code == 404
@@ -444,6 +449,10 @@ def test_lab_opens_the_trace_of_any_saved_recording():
 def test_imported_lab_line_reconstructs_its_phrase_outside_the_fixed_list():
     from app.lab import sample_phrase
 
+    reviewed = sample_phrase({"id": 6, "item": "minerva:U8aBRd3-wlM:01", "ar": "wrong", "ipa": "x"})
+    assert reviewed["variants"] == [("فيه عندك حليب طارة؟",
+                                      "f iː h | ʕ i n d a k | ħ a l iː b | tˤ aː r e")]
+
     phrase = sample_phrase({"id": 7, "item": "minerva:video:01", "ar": "حال", "ipa": "ħ aː l",
                             "speaker": "מינרווה"})
     assert phrase == {"cat": "lab", "key": "minerva:video:01", "he": "מינרווה",

tests/test_minerva_import.py

@@ -1,10 +1,10 @@
-from scripts.import_minerva import PILOT, phrases
+from scripts.import_minerva import phrases
 
 
-def test_minerva_pilot_manifest_has_exactly_the_reviewed_three_videos_and_70_lines():
-    assert [v.youtube_id for v in PILOT] == ["U8aBRd3-wlM", "OuAwxXUZBU4", "V2y6LqnoXTo"]
-    assert [len(v.lines) for v in PILOT] == [15, 22, 33]
-    made, _ = phrases()
-    assert len(made) == 70
-    assert len({p[4]["key"] for p in made}) == 70
-    assert all(p[2] < p[3] and p[4]["variants"] for p in made)
+def test_minerva_import_takes_every_reviewed_line_from_the_metadata():
+    made = phrases()
+    keys = [p[2]["key"] for p in made]
+    assert len(keys) == len(set(keys))
+    assert all(s["start"] < s["end"] and p["variants"] == [(s["ar"], s["ipa"])] for _, s, p in made)
+    pilot = phrases({"U8aBRd3-wlM", "OuAwxXUZBU4", "V2y6LqnoXTo"})
+    assert len(pilot) == 70 and pilot[0][2]["key"] == "minerva:U8aBRd3-wlM:01"