Spaces:
Sleeping
Sleeping
Sync from GitHub via hub-sync
Browse files
app.py
CHANGED
|
@@ -339,7 +339,7 @@ def classify_sentence_clip(video_path: str | None) -> tuple[str, dict[str, Any]]
|
|
| 339 |
"## Best Sentence Match",
|
| 340 |
"",
|
| 341 |
f"# {label}",
|
| 342 |
-
f"Nearest
|
| 343 |
"",
|
| 344 |
"### Top Sentence Matches",
|
| 345 |
]
|
|
@@ -354,9 +354,12 @@ def classify_sentence_clip(video_path: str | None) -> tuple[str, dict[str, Any]]
|
|
| 354 |
]
|
| 355 |
if runtime.embedding_index is not None and runtime.embedding_index.index_path:
|
| 356 |
lines.append(f"Index path: {runtime.embedding_index.index_path}")
|
|
|
|
|
|
|
|
|
|
| 357 |
lines += [
|
| 358 |
"",
|
| 359 |
-
"Note: this result comes from the 3D CNN embedding space matched against the repeated-sentence
|
| 360 |
]
|
| 361 |
return "\n".join(lines), details
|
| 362 |
|
|
|
|
| 339 |
"## Best Sentence Match",
|
| 340 |
"",
|
| 341 |
f"# {label}",
|
| 342 |
+
f"Nearest support-clip similarity: **{confidence:.3f}**",
|
| 343 |
"",
|
| 344 |
"### Top Sentence Matches",
|
| 345 |
]
|
|
|
|
| 354 |
]
|
| 355 |
if runtime.embedding_index is not None and runtime.embedding_index.index_path:
|
| 356 |
lines.append(f"Index path: {runtime.embedding_index.index_path}")
|
| 357 |
+
support_split = getattr(runtime.embedding_index, "support_split", None) if runtime.embedding_index is not None else None
|
| 358 |
+
if support_split:
|
| 359 |
+
lines.append(f"Support set: {support_split}")
|
| 360 |
lines += [
|
| 361 |
"",
|
| 362 |
+
"Note: this result comes from the 3D CNN embedding space matched against the repeated-sentence reference set.",
|
| 363 |
]
|
| 364 |
return "\n".join(lines), details
|
| 365 |
|
models/cnn-3d-sentence-top25.index.npz
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a424c9249ba9169b37d80bb0e1bb619efb85d6c30af58c747a34abbfd0a29269
|
| 3 |
+
size 124491
|
scripts/build_sentence_embedding_index.py
CHANGED
|
@@ -34,9 +34,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
| 34 |
)
|
| 35 |
parser.add_argument(
|
| 36 |
"--split",
|
| 37 |
-
default="
|
| 38 |
-
choices=("train", "val", "test"),
|
| 39 |
-
help="Which manifest split to index.",
|
| 40 |
)
|
| 41 |
parser.add_argument(
|
| 42 |
"--top-k",
|
|
@@ -79,7 +79,9 @@ def main() -> None:
|
|
| 79 |
if runtime.embedding_model is None:
|
| 80 |
raise RuntimeError("The sentence model does not expose a clip_embedding layer.")
|
| 81 |
|
| 82 |
-
records =
|
|
|
|
|
|
|
| 83 |
if not records:
|
| 84 |
raise RuntimeError(f"No records found for split={args.split!r} in {manifest_path}.")
|
| 85 |
|
|
@@ -117,6 +119,7 @@ def main() -> None:
|
|
| 117 |
labels=np.asarray(labels, dtype="<U128"),
|
| 118 |
clip_ids=np.asarray(clip_ids, dtype="<U128"),
|
| 119 |
source_manifest=np.asarray(str(manifest_path), dtype="<U512"),
|
|
|
|
| 120 |
top_k=np.asarray(args.top_k, dtype=np.int32),
|
| 121 |
candidate_pool=np.asarray(args.candidate_pool, dtype=np.int32),
|
| 122 |
)
|
|
|
|
| 34 |
)
|
| 35 |
parser.add_argument(
|
| 36 |
"--split",
|
| 37 |
+
default="all",
|
| 38 |
+
choices=("all", "train", "val", "test"),
|
| 39 |
+
help="Which manifest split to index. Use 'all' for the full support set.",
|
| 40 |
)
|
| 41 |
parser.add_argument(
|
| 42 |
"--top-k",
|
|
|
|
| 79 |
if runtime.embedding_model is None:
|
| 80 |
raise RuntimeError("The sentence model does not expose a clip_embedding layer.")
|
| 81 |
|
| 82 |
+
records = load_clip_dataset(manifest_path)
|
| 83 |
+
if args.split != "all":
|
| 84 |
+
records = [record for record in records if record.split == args.split]
|
| 85 |
if not records:
|
| 86 |
raise RuntimeError(f"No records found for split={args.split!r} in {manifest_path}.")
|
| 87 |
|
|
|
|
| 119 |
labels=np.asarray(labels, dtype="<U128"),
|
| 120 |
clip_ids=np.asarray(clip_ids, dtype="<U128"),
|
| 121 |
source_manifest=np.asarray(str(manifest_path), dtype="<U512"),
|
| 122 |
+
support_split=np.asarray(args.split, dtype="<U32"),
|
| 123 |
top_k=np.asarray(args.top_k, dtype=np.int32),
|
| 124 |
candidate_pool=np.asarray(args.candidate_pool, dtype=np.int32),
|
| 125 |
)
|
src/bridgelink_asl/sentence_inference.py
CHANGED
|
@@ -22,6 +22,7 @@ class SentenceEmbeddingIndex:
|
|
| 22 |
candidate_pool: int = 10
|
| 23 |
source_manifest: str | None = None
|
| 24 |
index_path: str | None = None
|
|
|
|
| 25 |
|
| 26 |
def classify(
|
| 27 |
self,
|
|
@@ -68,6 +69,7 @@ class SentenceEmbeddingIndex:
|
|
| 68 |
"neighbor_similarities": [float(similarities[int(index)]) for index in vote_nearest],
|
| 69 |
"source_manifest": self.source_manifest,
|
| 70 |
"index_path": self.index_path,
|
|
|
|
| 71 |
"top_k": self.top_k,
|
| 72 |
"candidate_pool": self.candidate_pool,
|
| 73 |
}
|
|
@@ -267,6 +269,9 @@ def _load_embedding_index(index_path: str | Path) -> SentenceEmbeddingIndex:
|
|
| 267 |
source_manifest = None
|
| 268 |
if "source_manifest" in payload:
|
| 269 |
source_manifest = str(payload["source_manifest"].item())
|
|
|
|
|
|
|
|
|
|
| 270 |
top_k = int(payload["top_k"].item()) if "top_k" in payload else 3
|
| 271 |
candidate_pool = int(payload["candidate_pool"].item()) if "candidate_pool" in payload else 10
|
| 272 |
return SentenceEmbeddingIndex(
|
|
@@ -277,6 +282,7 @@ def _load_embedding_index(index_path: str | Path) -> SentenceEmbeddingIndex:
|
|
| 277 |
candidate_pool=candidate_pool,
|
| 278 |
source_manifest=source_manifest,
|
| 279 |
index_path=str(Path(index_path)),
|
|
|
|
| 280 |
)
|
| 281 |
|
| 282 |
|
|
|
|
| 22 |
candidate_pool: int = 10
|
| 23 |
source_manifest: str | None = None
|
| 24 |
index_path: str | None = None
|
| 25 |
+
support_split: str | None = None
|
| 26 |
|
| 27 |
def classify(
|
| 28 |
self,
|
|
|
|
| 69 |
"neighbor_similarities": [float(similarities[int(index)]) for index in vote_nearest],
|
| 70 |
"source_manifest": self.source_manifest,
|
| 71 |
"index_path": self.index_path,
|
| 72 |
+
"support_split": self.support_split,
|
| 73 |
"top_k": self.top_k,
|
| 74 |
"candidate_pool": self.candidate_pool,
|
| 75 |
}
|
|
|
|
| 269 |
source_manifest = None
|
| 270 |
if "source_manifest" in payload:
|
| 271 |
source_manifest = str(payload["source_manifest"].item())
|
| 272 |
+
support_split = None
|
| 273 |
+
if "support_split" in payload:
|
| 274 |
+
support_split = str(payload["support_split"].item())
|
| 275 |
top_k = int(payload["top_k"].item()) if "top_k" in payload else 3
|
| 276 |
candidate_pool = int(payload["candidate_pool"].item()) if "candidate_pool" in payload else 10
|
| 277 |
return SentenceEmbeddingIndex(
|
|
|
|
| 282 |
candidate_pool=candidate_pool,
|
| 283 |
source_manifest=source_manifest,
|
| 284 |
index_path=str(Path(index_path)),
|
| 285 |
+
support_split=support_split,
|
| 286 |
)
|
| 287 |
|
| 288 |
|
tests/test_app_live_dual_models.py
CHANGED
|
@@ -7,84 +7,11 @@ import numpy as np
|
|
| 7 |
import app
|
| 8 |
|
| 9 |
|
| 10 |
-
def
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
state = app.init_live_state()
|
| 14 |
-
|
| 15 |
-
assert "sentence_buffer" in state
|
| 16 |
-
assert state["sentence_buffer"].maxlen == 16
|
| 17 |
-
assert state["sentence_caption"] == ""
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def test_sentence_status_markdown_uses_confidence_gate(monkeypatch) -> None:
|
| 21 |
-
monkeypatch.setattr(app, "SENTENCE_RUNTIME", SimpleNamespace(frame_count=16), raising=False)
|
| 22 |
-
state = app.init_live_state()
|
| 23 |
-
state["sentence_caption"] = "Thank you."
|
| 24 |
-
state["sentence_candidate_label"] = "Thank you."
|
| 25 |
-
state["sentence_candidate_confidence"] = 0.81
|
| 26 |
-
|
| 27 |
-
status = app._sentence_status_markdown(state)
|
| 28 |
-
|
| 29 |
-
assert "Sentence Watch" in status
|
| 30 |
-
assert "Thank you." in status
|
| 31 |
-
assert "Closed vocabulary" in status
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
def test_update_sentence_live_state_requires_confidence(monkeypatch) -> None:
|
| 35 |
-
class FakeRuntime:
|
| 36 |
-
frame_count = 1
|
| 37 |
-
image_size = 8
|
| 38 |
-
|
| 39 |
-
def predict_clip(self, clip_volume):
|
| 40 |
-
assert clip_volume.shape == (1, 8, 8, 3)
|
| 41 |
-
return "Hello!", 0.42, [("Hello!", 0.42)]
|
| 42 |
-
|
| 43 |
-
monkeypatch.setattr(app, "SENTENCE_RUNTIME", FakeRuntime(), raising=False)
|
| 44 |
-
monkeypatch.setattr(app, "SENTENCE_MIN_CONFIDENCE", 0.55, raising=False)
|
| 45 |
-
monkeypatch.setattr(app, "SENTENCE_STREAM_STRIDE", 1, raising=False)
|
| 46 |
-
|
| 47 |
-
state = app.init_live_state()
|
| 48 |
-
state["frame_idx"] = 1
|
| 49 |
-
frame = np.zeros((16, 16, 3), dtype=np.uint8)
|
| 50 |
-
|
| 51 |
-
app._update_sentence_live_state(frame, state)
|
| 52 |
-
|
| 53 |
-
assert state["sentence_candidate_label"] is None
|
| 54 |
-
assert state["sentence_caption"] == ""
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
def test_update_sentence_live_state_emits_after_stability(monkeypatch) -> None:
|
| 58 |
-
class FakeRuntime:
|
| 59 |
-
frame_count = 1
|
| 60 |
-
image_size = 8
|
| 61 |
-
|
| 62 |
-
def predict_clip(self, clip_volume):
|
| 63 |
-
return "Thank you.", 0.91, [("Thank you.", 0.91)]
|
| 64 |
-
|
| 65 |
-
monkeypatch.setattr(app, "SENTENCE_RUNTIME", FakeRuntime(), raising=False)
|
| 66 |
-
monkeypatch.setattr(app, "SENTENCE_MIN_CONFIDENCE", 0.55, raising=False)
|
| 67 |
-
monkeypatch.setattr(app, "SENTENCE_STREAM_STRIDE", 1, raising=False)
|
| 68 |
-
monkeypatch.setattr(app, "SENTENCE_STABILITY_K", 1, raising=False)
|
| 69 |
-
|
| 70 |
-
state = app.init_live_state()
|
| 71 |
-
frame = np.zeros((16, 16, 3), dtype=np.uint8)
|
| 72 |
-
|
| 73 |
-
state["frame_idx"] = 1
|
| 74 |
-
app._update_sentence_live_state(frame, state)
|
| 75 |
-
|
| 76 |
-
assert state["sentence_candidate_label"] == "Thank you."
|
| 77 |
-
assert state["sentence_caption"] == "Thank you."
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
def test_reset_live_returns_both_status_panels(monkeypatch) -> None:
|
| 81 |
-
monkeypatch.setattr(app, "SENTENCE_RUNTIME", SimpleNamespace(frame_count=16), raising=False)
|
| 82 |
-
|
| 83 |
-
state, sign_status, sentence_status = app.reset_live()
|
| 84 |
|
| 85 |
assert "buffer" in state
|
| 86 |
-
assert "Caption" in
|
| 87 |
-
assert "Sentence Watch" in sentence_status
|
| 88 |
|
| 89 |
|
| 90 |
def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
|
|
@@ -94,7 +21,10 @@ def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
|
|
| 94 |
model_path = "models/cnn-3d-sentence-top25.keras"
|
| 95 |
inference_mode = "embedding_knn"
|
| 96 |
embedding_model = object()
|
| 97 |
-
embedding_index = SimpleNamespace(
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
def predict_clip_retrieval(self, clip_volume):
|
| 100 |
return (
|
|
@@ -120,5 +50,6 @@ def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
|
|
| 120 |
assert "Best Sentence Match" in markdown
|
| 121 |
assert "Thank you." in markdown
|
| 122 |
assert "Inference mode: embedding_knn" in markdown
|
|
|
|
| 123 |
assert details["accepted_label"] == "Thank you."
|
| 124 |
assert details["sentence_inference_mode"] == "embedding_knn"
|
|
|
|
| 7 |
import app
|
| 8 |
|
| 9 |
|
| 10 |
+
def test_reset_live_returns_state_and_status() -> None:
|
| 11 |
+
state, status = app.reset_live()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
assert "buffer" in state
|
| 14 |
+
assert "Caption" in status
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
|
|
|
|
| 21 |
model_path = "models/cnn-3d-sentence-top25.keras"
|
| 22 |
inference_mode = "embedding_knn"
|
| 23 |
embedding_model = object()
|
| 24 |
+
embedding_index = SimpleNamespace(
|
| 25 |
+
index_path="models/cnn-3d-sentence-top25.index.npz",
|
| 26 |
+
support_split="all",
|
| 27 |
+
)
|
| 28 |
|
| 29 |
def predict_clip_retrieval(self, clip_volume):
|
| 30 |
return (
|
|
|
|
| 50 |
assert "Best Sentence Match" in markdown
|
| 51 |
assert "Thank you." in markdown
|
| 52 |
assert "Inference mode: embedding_knn" in markdown
|
| 53 |
+
assert "Support set: all" in markdown
|
| 54 |
assert details["accepted_label"] == "Thank you."
|
| 55 |
assert details["sentence_inference_mode"] == "embedding_knn"
|
tests/test_sentence_inference.py
CHANGED
|
@@ -99,6 +99,7 @@ def test_load_sentence_runtime_loads_embedding_index(tmp_path: Path) -> None:
|
|
| 99 |
labels=np.array(["Hello!", "Thank you."], dtype="<U32"),
|
| 100 |
clip_ids=np.array(["clip_a", "clip_b"], dtype="<U32"),
|
| 101 |
source_manifest=np.array(str(tmp_path / "manifest.jsonl"), dtype="<U256"),
|
|
|
|
| 102 |
top_k=np.array(3, dtype=np.int32),
|
| 103 |
candidate_pool=np.array(10, dtype=np.int32),
|
| 104 |
)
|
|
@@ -122,6 +123,7 @@ def test_load_sentence_runtime_loads_embedding_index(tmp_path: Path) -> None:
|
|
| 122 |
assert runtime.embedding_model is fake_embedding_model
|
| 123 |
assert runtime.embedding_index is not None
|
| 124 |
assert runtime.embedding_index.index_path == str(index_path)
|
|
|
|
| 125 |
assert runtime.inference_mode == "embedding_knn"
|
| 126 |
|
| 127 |
|
|
|
|
| 99 |
labels=np.array(["Hello!", "Thank you."], dtype="<U32"),
|
| 100 |
clip_ids=np.array(["clip_a", "clip_b"], dtype="<U32"),
|
| 101 |
source_manifest=np.array(str(tmp_path / "manifest.jsonl"), dtype="<U256"),
|
| 102 |
+
support_split=np.array("all", dtype="<U16"),
|
| 103 |
top_k=np.array(3, dtype=np.int32),
|
| 104 |
candidate_pool=np.array(10, dtype=np.int32),
|
| 105 |
)
|
|
|
|
| 123 |
assert runtime.embedding_model is fake_embedding_model
|
| 124 |
assert runtime.embedding_index is not None
|
| 125 |
assert runtime.embedding_index.index_path == str(index_path)
|
| 126 |
+
assert runtime.embedding_index.support_split == "all"
|
| 127 |
assert runtime.inference_mode == "embedding_knn"
|
| 128 |
|
| 129 |
|