File size: 9,191 Bytes
e35b538
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
"""Streamlit comparison of the archived Transformer predictions on one INCLUDE panel."""

import csv
import gzip
import io
import json
from pathlib import Path
from urllib.parse import quote

import streamlit as st
import streamlit.components.v1 as components


ARMS = {
    "rolling": "Rolling Transformer · epoch 31",
    "causal": "Causal Transformer · epoch 102",
}
DATA = Path(__file__).with_name("data") / "include_validation.json.gz"
VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset"
VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9"


@st.cache_data
def load_panel():
    with gzip.open(DATA, "rt", encoding="utf-8") as source:
        return json.load(source)


def outcome_counts(clips, arm):
    correct = wrong_early = correct_early = unconfirmed = 0
    for clip in clips:
        prediction = clip[arm]
        correct += (prediction["last"] or {}).get("label") == clip["label"]
        early = prediction["early"]
        if early is None:
            unconfirmed += 1
        elif early["label"] == clip["label"]:
            correct_early += 1
        else:
            wrong_early += 1
    return correct, wrong_early, correct_early, unconfirmed


def score(value, total):
    return f"{value / total:.2%} ({value}/{total})"


def export_scores(clips):
    buffer = io.StringIO()
    writer = csv.writer(buffer)
    writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds",
                     "first_confirmation", "confirmation_seconds", "outcome"))
    for clip in clips:
        for arm, model in ARMS.items():
            prediction = clip[arm]
            last, early = prediction["last"] or {}, prediction["early"] or {}
            outcome = ("Unconfirmed" if not early else
                       "Correct" if early["label"] == clip["label"] else "Wrong")
            writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"),
                             early.get("label"), early.get("time"), outcome))
    return buffer.getvalue()


def public_video(path):
    if not path:
        return None
    parts = path.split("/")
    if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}:
        raise ValueError("Unexpected public video path")
    if any(part in {"", ".", ".."} for part in parts):
        raise ValueError("Invalid public video path")
    return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}"


def render_playback(video_url, clip):
    # Inline script data is escaped before interpolation into the iframe document.
    payload = json.dumps({
        "video": video_url, "arms": ARMS, "duration": clip["duration"],
        "predictions": {arm: clip[arm] for arm in ARMS},
    }, ensure_ascii=True).replace("<", "\\u003c")
    template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8")
    components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True)


def render_transformer_comparison():
    st.title("SignLanguageTranslate")
    st.header("Indian Sign Language model comparison")
    try:
        panel = load_panel()
    except (OSError, ValueError):
        st.error("The archived evaluation could not be loaded.")
        return

    if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}:
        st.error("The archived evaluation points to a different video dataset revision.")
        return
    clips = panel["clips"]
    st.caption("720 isolated INCLUDE clips from the saved iPad validation split. "
               "Both models were evaluated on the same Apple Vision keypoints.")
    from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings

    revision = settings()
    live = False
    if revision:
        if st.button("Run both models on all 720 validation clips"):
            try:
                with st.spinner("Running both models on the frozen validation clips…"):
                    st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips)
            except (OSError, ValueError, RuntimeError):
                st.error("Live evaluation could not load the pinned artifacts or finish inference.")
        results = st.session_state.get("transformer_evaluation")
        if results is not None:
            clips = results
            live = True
    else:
        st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.")
    st.subheader("Validation accuracy")
    st.caption("Computed by the Space" if live else "Archived evaluation on the Mac")
    table = []
    for arm, name in ARMS.items():
        correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm)
        table.append({
            "Model": name,
            "Final accuracy": score(correct, len(clips)),
            "Wrong early confirmations": score(wrong_early, len(clips)),
            "Correct early confirmations": correct_early,
            "Unconfirmed clips": unconfirmed,
        })
    st.dataframe(table, use_container_width=True, hide_index=True)
    st.caption("Final accuracy uses the last scheduled prediction before the clip ends. "
               "An early confirmation is the first label repeated across three consecutive updates.")
    st.download_button("Download validation results (CSV)", export_scores(clips),
                       file_name="include-validation-results.csv", mime="text/csv")

    st.subheader("Replay predictions")
    labels = sorted({clip["label"] for clip in clips}, key=str.casefold)
    selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0)
    matching = [clip for clip in clips if clip["label"] == selected_label]
    default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0)
    clip = st.selectbox("Validation clip", matching, index=default_clip,
                        format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s"
                        + (" · source video available" if item["videoPath"] else ""))
    if revision and not live:
        if st.button("Run both models on this clip", key=f"predict-{clip['id']}"):
            try:
                with st.spinner("Running both models on the saved poses…"):
                    inputs, models = load_assets(revision)
                    index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"])
                    st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip)
            except (OSError, ValueError, RuntimeError):
                st.error("Live inference could not load the pinned artifacts or predict this clip.")
        result = st.session_state.get("transformer_clip")
        if result is not None and result["id"] == clip["id"]:
            clip = result
            live = True
    video_url = public_video(clip["videoPath"])
    if video_url:
        split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]]
        st.caption(f"Public video repository split: {split}.")
        st.markdown(f"[Open source video]({video_url})")
    else:
        st.caption("Use the timeline slider to review this clip’s archived predictions.")
    st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.")
    render_playback(video_url, clip)
    with st.expander("All prediction updates"):
        updates = {}
        for arm, name in ARMS.items():
            for tick in clip[arm]["ticks"]:
                updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"]
        st.dataframe(list(updates.values()), use_container_width=True, hide_index=True)

    with st.expander("Dataset splits and model sources"):
        st.markdown(
            "- This evaluation uses 720 of 743 clips in the saved iPad validation split. "
            "The models share 256 output labels. This panel covers 254 labels, and 23 clips "
            "were excluded because their labels are outside the model vocabulary. "
            "Epoch 102 was selected using this validation panel. Signer independence is unverified.\n"
            "- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, "
            "245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. "
            "Training membership for the published legacy weights remains unverified.\n"
            "- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · "
            "[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · "
            "[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · "
            "[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · "
            "[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)"
        )