ofraij123 commited on
Commit
deebb21
·
verified ·
1 Parent(s): 2cad25f

Sync from GitHub via hub-sync

Browse files
app.py CHANGED
@@ -339,7 +339,7 @@ def classify_sentence_clip(video_path: str | None) -> tuple[str, dict[str, Any]]
339
  "## Best Sentence Match",
340
  "",
341
  f"# {label}",
342
- f"Nearest training-clip similarity: **{confidence:.3f}**",
343
  "",
344
  "### Top Sentence Matches",
345
  ]
@@ -354,9 +354,12 @@ def classify_sentence_clip(video_path: str | None) -> tuple[str, dict[str, Any]]
354
  ]
355
  if runtime.embedding_index is not None and runtime.embedding_index.index_path:
356
  lines.append(f"Index path: {runtime.embedding_index.index_path}")
 
 
 
357
  lines += [
358
  "",
359
- "Note: this result comes from the 3D CNN embedding space matched against the repeated-sentence training subset.",
360
  ]
361
  return "\n".join(lines), details
362
 
 
339
  "## Best Sentence Match",
340
  "",
341
  f"# {label}",
342
+ f"Nearest support-clip similarity: **{confidence:.3f}**",
343
  "",
344
  "### Top Sentence Matches",
345
  ]
 
354
  ]
355
  if runtime.embedding_index is not None and runtime.embedding_index.index_path:
356
  lines.append(f"Index path: {runtime.embedding_index.index_path}")
357
+ support_split = getattr(runtime.embedding_index, "support_split", None) if runtime.embedding_index is not None else None
358
+ if support_split:
359
+ lines.append(f"Support set: {support_split}")
360
  lines += [
361
  "",
362
+ "Note: this result comes from the 3D CNN embedding space matched against the repeated-sentence reference set.",
363
  ]
364
  return "\n".join(lines), details
365
 
models/cnn-3d-sentence-top25.index.npz CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2039d69e2d0454c07f1897dd02c6c6d860a3bcbf1f24e66908b55d10ba30efc7
3
- size 87547
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a424c9249ba9169b37d80bb0e1bb619efb85d6c30af58c747a34abbfd0a29269
3
+ size 124491
scripts/build_sentence_embedding_index.py CHANGED
@@ -34,9 +34,9 @@ def build_parser() -> argparse.ArgumentParser:
34
  )
35
  parser.add_argument(
36
  "--split",
37
- default="train",
38
- choices=("train", "val", "test"),
39
- help="Which manifest split to index.",
40
  )
41
  parser.add_argument(
42
  "--top-k",
@@ -79,7 +79,9 @@ def main() -> None:
79
  if runtime.embedding_model is None:
80
  raise RuntimeError("The sentence model does not expose a clip_embedding layer.")
81
 
82
- records = [record for record in load_clip_dataset(manifest_path) if record.split == args.split]
 
 
83
  if not records:
84
  raise RuntimeError(f"No records found for split={args.split!r} in {manifest_path}.")
85
 
@@ -117,6 +119,7 @@ def main() -> None:
117
  labels=np.asarray(labels, dtype="<U128"),
118
  clip_ids=np.asarray(clip_ids, dtype="<U128"),
119
  source_manifest=np.asarray(str(manifest_path), dtype="<U512"),
 
120
  top_k=np.asarray(args.top_k, dtype=np.int32),
121
  candidate_pool=np.asarray(args.candidate_pool, dtype=np.int32),
122
  )
 
34
  )
35
  parser.add_argument(
36
  "--split",
37
+ default="all",
38
+ choices=("all", "train", "val", "test"),
39
+ help="Which manifest split to index. Use 'all' for the full support set.",
40
  )
41
  parser.add_argument(
42
  "--top-k",
 
79
  if runtime.embedding_model is None:
80
  raise RuntimeError("The sentence model does not expose a clip_embedding layer.")
81
 
82
+ records = load_clip_dataset(manifest_path)
83
+ if args.split != "all":
84
+ records = [record for record in records if record.split == args.split]
85
  if not records:
86
  raise RuntimeError(f"No records found for split={args.split!r} in {manifest_path}.")
87
 
 
119
  labels=np.asarray(labels, dtype="<U128"),
120
  clip_ids=np.asarray(clip_ids, dtype="<U128"),
121
  source_manifest=np.asarray(str(manifest_path), dtype="<U512"),
122
+ support_split=np.asarray(args.split, dtype="<U32"),
123
  top_k=np.asarray(args.top_k, dtype=np.int32),
124
  candidate_pool=np.asarray(args.candidate_pool, dtype=np.int32),
125
  )
src/bridgelink_asl/sentence_inference.py CHANGED
@@ -22,6 +22,7 @@ class SentenceEmbeddingIndex:
22
  candidate_pool: int = 10
23
  source_manifest: str | None = None
24
  index_path: str | None = None
 
25
 
26
  def classify(
27
  self,
@@ -68,6 +69,7 @@ class SentenceEmbeddingIndex:
68
  "neighbor_similarities": [float(similarities[int(index)]) for index in vote_nearest],
69
  "source_manifest": self.source_manifest,
70
  "index_path": self.index_path,
 
71
  "top_k": self.top_k,
72
  "candidate_pool": self.candidate_pool,
73
  }
@@ -267,6 +269,9 @@ def _load_embedding_index(index_path: str | Path) -> SentenceEmbeddingIndex:
267
  source_manifest = None
268
  if "source_manifest" in payload:
269
  source_manifest = str(payload["source_manifest"].item())
 
 
 
270
  top_k = int(payload["top_k"].item()) if "top_k" in payload else 3
271
  candidate_pool = int(payload["candidate_pool"].item()) if "candidate_pool" in payload else 10
272
  return SentenceEmbeddingIndex(
@@ -277,6 +282,7 @@ def _load_embedding_index(index_path: str | Path) -> SentenceEmbeddingIndex:
277
  candidate_pool=candidate_pool,
278
  source_manifest=source_manifest,
279
  index_path=str(Path(index_path)),
 
280
  )
281
 
282
 
 
22
  candidate_pool: int = 10
23
  source_manifest: str | None = None
24
  index_path: str | None = None
25
+ support_split: str | None = None
26
 
27
  def classify(
28
  self,
 
69
  "neighbor_similarities": [float(similarities[int(index)]) for index in vote_nearest],
70
  "source_manifest": self.source_manifest,
71
  "index_path": self.index_path,
72
+ "support_split": self.support_split,
73
  "top_k": self.top_k,
74
  "candidate_pool": self.candidate_pool,
75
  }
 
269
  source_manifest = None
270
  if "source_manifest" in payload:
271
  source_manifest = str(payload["source_manifest"].item())
272
+ support_split = None
273
+ if "support_split" in payload:
274
+ support_split = str(payload["support_split"].item())
275
  top_k = int(payload["top_k"].item()) if "top_k" in payload else 3
276
  candidate_pool = int(payload["candidate_pool"].item()) if "candidate_pool" in payload else 10
277
  return SentenceEmbeddingIndex(
 
282
  candidate_pool=candidate_pool,
283
  source_manifest=source_manifest,
284
  index_path=str(Path(index_path)),
285
+ support_split=support_split,
286
  )
287
 
288
 
tests/test_app_live_dual_models.py CHANGED
@@ -7,84 +7,11 @@ import numpy as np
7
  import app
8
 
9
 
10
- def test_init_live_state_includes_sentence_buffer(monkeypatch) -> None:
11
- monkeypatch.setattr(app, "SENTENCE_RUNTIME", SimpleNamespace(frame_count=16), raising=False)
12
-
13
- state = app.init_live_state()
14
-
15
- assert "sentence_buffer" in state
16
- assert state["sentence_buffer"].maxlen == 16
17
- assert state["sentence_caption"] == ""
18
-
19
-
20
- def test_sentence_status_markdown_uses_confidence_gate(monkeypatch) -> None:
21
- monkeypatch.setattr(app, "SENTENCE_RUNTIME", SimpleNamespace(frame_count=16), raising=False)
22
- state = app.init_live_state()
23
- state["sentence_caption"] = "Thank you."
24
- state["sentence_candidate_label"] = "Thank you."
25
- state["sentence_candidate_confidence"] = 0.81
26
-
27
- status = app._sentence_status_markdown(state)
28
-
29
- assert "Sentence Watch" in status
30
- assert "Thank you." in status
31
- assert "Closed vocabulary" in status
32
-
33
-
34
- def test_update_sentence_live_state_requires_confidence(monkeypatch) -> None:
35
- class FakeRuntime:
36
- frame_count = 1
37
- image_size = 8
38
-
39
- def predict_clip(self, clip_volume):
40
- assert clip_volume.shape == (1, 8, 8, 3)
41
- return "Hello!", 0.42, [("Hello!", 0.42)]
42
-
43
- monkeypatch.setattr(app, "SENTENCE_RUNTIME", FakeRuntime(), raising=False)
44
- monkeypatch.setattr(app, "SENTENCE_MIN_CONFIDENCE", 0.55, raising=False)
45
- monkeypatch.setattr(app, "SENTENCE_STREAM_STRIDE", 1, raising=False)
46
-
47
- state = app.init_live_state()
48
- state["frame_idx"] = 1
49
- frame = np.zeros((16, 16, 3), dtype=np.uint8)
50
-
51
- app._update_sentence_live_state(frame, state)
52
-
53
- assert state["sentence_candidate_label"] is None
54
- assert state["sentence_caption"] == ""
55
-
56
-
57
- def test_update_sentence_live_state_emits_after_stability(monkeypatch) -> None:
58
- class FakeRuntime:
59
- frame_count = 1
60
- image_size = 8
61
-
62
- def predict_clip(self, clip_volume):
63
- return "Thank you.", 0.91, [("Thank you.", 0.91)]
64
-
65
- monkeypatch.setattr(app, "SENTENCE_RUNTIME", FakeRuntime(), raising=False)
66
- monkeypatch.setattr(app, "SENTENCE_MIN_CONFIDENCE", 0.55, raising=False)
67
- monkeypatch.setattr(app, "SENTENCE_STREAM_STRIDE", 1, raising=False)
68
- monkeypatch.setattr(app, "SENTENCE_STABILITY_K", 1, raising=False)
69
-
70
- state = app.init_live_state()
71
- frame = np.zeros((16, 16, 3), dtype=np.uint8)
72
-
73
- state["frame_idx"] = 1
74
- app._update_sentence_live_state(frame, state)
75
-
76
- assert state["sentence_candidate_label"] == "Thank you."
77
- assert state["sentence_caption"] == "Thank you."
78
-
79
-
80
- def test_reset_live_returns_both_status_panels(monkeypatch) -> None:
81
- monkeypatch.setattr(app, "SENTENCE_RUNTIME", SimpleNamespace(frame_count=16), raising=False)
82
-
83
- state, sign_status, sentence_status = app.reset_live()
84
 
85
  assert "buffer" in state
86
- assert "Caption" in sign_status
87
- assert "Sentence Watch" in sentence_status
88
 
89
 
90
  def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
@@ -94,7 +21,10 @@ def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
94
  model_path = "models/cnn-3d-sentence-top25.keras"
95
  inference_mode = "embedding_knn"
96
  embedding_model = object()
97
- embedding_index = SimpleNamespace(index_path="models/cnn-3d-sentence-top25.index.npz")
 
 
 
98
 
99
  def predict_clip_retrieval(self, clip_volume):
100
  return (
@@ -120,5 +50,6 @@ def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
120
  assert "Best Sentence Match" in markdown
121
  assert "Thank you." in markdown
122
  assert "Inference mode: embedding_knn" in markdown
 
123
  assert details["accepted_label"] == "Thank you."
124
  assert details["sentence_inference_mode"] == "embedding_knn"
 
7
  import app
8
 
9
 
10
+ def test_reset_live_returns_state_and_status() -> None:
11
+ state, status = app.reset_live()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  assert "buffer" in state
14
+ assert "Caption" in status
 
15
 
16
 
17
  def test_classify_sentence_clip_uses_retrieval_mode(monkeypatch) -> None:
 
21
  model_path = "models/cnn-3d-sentence-top25.keras"
22
  inference_mode = "embedding_knn"
23
  embedding_model = object()
24
+ embedding_index = SimpleNamespace(
25
+ index_path="models/cnn-3d-sentence-top25.index.npz",
26
+ support_split="all",
27
+ )
28
 
29
  def predict_clip_retrieval(self, clip_volume):
30
  return (
 
50
  assert "Best Sentence Match" in markdown
51
  assert "Thank you." in markdown
52
  assert "Inference mode: embedding_knn" in markdown
53
+ assert "Support set: all" in markdown
54
  assert details["accepted_label"] == "Thank you."
55
  assert details["sentence_inference_mode"] == "embedding_knn"
tests/test_sentence_inference.py CHANGED
@@ -99,6 +99,7 @@ def test_load_sentence_runtime_loads_embedding_index(tmp_path: Path) -> None:
99
  labels=np.array(["Hello!", "Thank you."], dtype="<U32"),
100
  clip_ids=np.array(["clip_a", "clip_b"], dtype="<U32"),
101
  source_manifest=np.array(str(tmp_path / "manifest.jsonl"), dtype="<U256"),
 
102
  top_k=np.array(3, dtype=np.int32),
103
  candidate_pool=np.array(10, dtype=np.int32),
104
  )
@@ -122,6 +123,7 @@ def test_load_sentence_runtime_loads_embedding_index(tmp_path: Path) -> None:
122
  assert runtime.embedding_model is fake_embedding_model
123
  assert runtime.embedding_index is not None
124
  assert runtime.embedding_index.index_path == str(index_path)
 
125
  assert runtime.inference_mode == "embedding_knn"
126
 
127
 
 
99
  labels=np.array(["Hello!", "Thank you."], dtype="<U32"),
100
  clip_ids=np.array(["clip_a", "clip_b"], dtype="<U32"),
101
  source_manifest=np.array(str(tmp_path / "manifest.jsonl"), dtype="<U256"),
102
+ support_split=np.array("all", dtype="<U16"),
103
  top_k=np.array(3, dtype=np.int32),
104
  candidate_pool=np.array(10, dtype=np.int32),
105
  )
 
123
  assert runtime.embedding_model is fake_embedding_model
124
  assert runtime.embedding_index is not None
125
  assert runtime.embedding_index.index_path == str(index_path)
126
+ assert runtime.embedding_index.support_split == "all"
127
  assert runtime.inference_mode == "embedding_knn"
128
 
129