File size: 4,130 Bytes
8fcf7c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
from __future__ import annotations

import json
import tomllib
from collections import Counter
from pathlib import Path

from sponsor_detection.data.benchmark import TARGET_CLASSES, write_json_lines_atomic
from sponsor_detection.data.benchmark_review import load_verified_jsonl
from sponsor_detection.data.profile import sha256_file, write_json_atomic


def apply_benchmark_annotations(
    review_path: Path,
    review_manifest_path: Path,
    annotations_path: Path,
) -> dict[str, object]:
    records, manifest = load_verified_jsonl(review_path, review_manifest_path)
    annotations = json.loads(annotations_path.read_text(encoding="utf-8"))
    if not isinstance(annotations, dict):
        raise ValueError("benchmark annotations must be keyed by video ID")
    expected_ids = {str(record["video_id"]) for record in records}
    annotation_ids = {str(video_id) for video_id in annotations}
    if annotation_ids != expected_ids:
        missing = sorted(expected_ids - annotation_ids)
        unexpected = sorted(annotation_ids - expected_ids)
        raise ValueError(
            f"benchmark annotations do not match review records; "
            f"missing={missing}, unexpected={unexpected}"
        )

    counts: Counter[str] = Counter()
    interval_count = 0
    for record in records:
        video_id = str(record["video_id"])
        annotation = annotations[video_id]
        final_class = annotation.get("final_class")
        intervals = annotation.get("intervals")
        notes = annotation.get("notes", "")
        if final_class not in TARGET_CLASSES:
            raise ValueError(f"video {video_id} has an invalid final class")
        if not isinstance(intervals, list) or not isinstance(notes, str):
            raise ValueError(f"video {video_id} has an invalid annotation")
        duration_ms = record.get("duration_ms")
        for interval in intervals:
            start_ms = interval.get("start_ms") if isinstance(interval, dict) else None
            end_ms = interval.get("end_ms") if isinstance(interval, dict) else None
            if (
                not isinstance(start_ms, int)
                or isinstance(start_ms, bool)
                or not isinstance(end_ms, int)
                or isinstance(end_ms, bool)
                or start_ms < 0
                or end_ms <= start_ms
                or (isinstance(duration_ms, int) and end_ms > duration_ms)
            ):
                raise ValueError(f"video {video_id} contains an invalid interval")
        if final_class == "sponsor_positive" and not intervals:
            raise ValueError(f"positive video {video_id} has no sponsor interval")
        if final_class != "sponsor_positive" and intervals:
            raise ValueError(f"negative video {video_id} has sponsor intervals")
        record["annotation"] = {
            "status": "reviewed",
            "final_class": final_class,
            "intervals": intervals,
            "notes": notes,
        }
        counts[str(final_class)] += 1
        interval_count += len(intervals)

    source_review_sha256 = sha256_file(review_path)
    write_json_lines_atomic(review_path, records)
    manifest["status"] = "reviewed"
    manifest["annotations"] = {
        "source_review_sha256": source_review_sha256,
        "annotations_path": str(annotations_path),
        "annotations_sha256": sha256_file(annotations_path),
        "reviewed_videos": len(records),
        "reviewed_by_class": dict(counts),
        "sponsor_intervals": interval_count,
    }
    manifest["output"] = {
        "path": str(review_path),
        "bytes": review_path.stat().st_size,
        "sha256": sha256_file(review_path),
    }
    write_json_atomic(review_manifest_path, manifest)
    return manifest


def apply_benchmark_annotations_from_config(path: Path) -> dict[str, object]:
    with path.open("rb") as source:
        configuration = tomllib.load(source)["benchmark_annotations"]
    return apply_benchmark_annotations(
        Path(configuration["review_path"]),
        Path(configuration["review_manifest_path"]),
        Path(configuration["annotations_path"]),
    )