Spaces:
Sleeping
Sleeping
Sync app from GitHub Actions
Browse files- src/ui/app_factory.py +223 -29
- tests/unit/test_app_factory_audio_helpers.py +101 -0
src/ui/app_factory.py
CHANGED
|
@@ -123,10 +123,34 @@ def _candidate_metadata_files(project_slug: str) -> list[str]:
|
|
| 123 |
]
|
| 124 |
|
| 125 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
def _pick_row_value(raw: dict[str, object], keys: list[str]) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
for key in keys:
|
| 128 |
value = raw.get(key)
|
| 129 |
-
if value
|
|
|
|
|
|
|
| 130 |
continue
|
| 131 |
text = str(value).strip()
|
| 132 |
if text:
|
|
@@ -148,6 +172,130 @@ def _normalize_audio_id(audio_value: str) -> str:
|
|
| 148 |
return normalized
|
| 149 |
|
| 150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
def _build_detection_from_row(raw: dict[str, object], row_index: int, project_slug: str) -> Detection | None:
|
| 152 |
row_project = _pick_row_value(raw, ["project_slug", "project", "project_id"])
|
| 153 |
if row_project and row_project != project_slug:
|
|
@@ -173,6 +321,7 @@ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_sl
|
|
| 173 |
)
|
| 174 |
if not audio_id:
|
| 175 |
return None
|
|
|
|
| 176 |
|
| 177 |
scientific_name = _pick_row_value(
|
| 178 |
raw,
|
|
@@ -189,20 +338,18 @@ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_sl
|
|
| 189 |
if not scientific_name:
|
| 190 |
scientific_name = "Unknown species"
|
| 191 |
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
)
|
| 196 |
-
|
|
|
|
|
|
|
| 197 |
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
)
|
| 202 |
-
end_time = _to_float(
|
| 203 |
-
raw.get("end_time", raw.get("end", raw.get("stop", raw.get("segment_end", 0.0)))),
|
| 204 |
-
0.0,
|
| 205 |
-
)
|
| 206 |
if end_time <= 0.0:
|
| 207 |
duration = _to_float(raw.get("duration", 0.0), 0.0)
|
| 208 |
if duration > 0.0:
|
|
@@ -210,10 +357,24 @@ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_sl
|
|
| 210 |
if end_time <= start_time:
|
| 211 |
end_time = start_time + 1.0
|
| 212 |
|
| 213 |
-
detection_key
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
try:
|
| 219 |
return Detection(
|
|
@@ -223,7 +384,7 @@ def _build_detection_from_row(raw: dict[str, object], row_index: int, project_sl
|
|
| 223 |
confidence=confidence,
|
| 224 |
start_time=start_time,
|
| 225 |
end_time=end_time,
|
| 226 |
-
source_metadata=
|
| 227 |
)
|
| 228 |
except Exception:
|
| 229 |
return None
|
|
@@ -547,32 +708,64 @@ def _merge_files_index_with_detection_rows(
|
|
| 547 |
) -> list[dict[str, object]]:
|
| 548 |
detection_by_species_and_source: dict[tuple[str, str], dict[str, object]] = {}
|
| 549 |
detection_by_source: dict[str, dict[str, object]] = {}
|
|
|
|
|
|
|
| 550 |
|
| 551 |
for row in detection_rows:
|
| 552 |
source_file = _pick_row_value(row, ["source_file", "audio_file", "file", "filename", "path"])
|
| 553 |
species = _pick_row_value(row, ["scientific_name", "species", "label", "logical_group"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 554 |
if source_file:
|
| 555 |
detection_by_source.setdefault(source_file, row)
|
| 556 |
if species:
|
| 557 |
detection_by_species_and_source.setdefault((species, source_file), row)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 558 |
|
| 559 |
merged: list[dict[str, object]] = []
|
| 560 |
for row in file_rows:
|
| 561 |
combined = dict(row)
|
| 562 |
original_relative_path = _pick_row_value(row, ["original_relative_path", "relative_path", "stored_path", "audio_id"])
|
| 563 |
-
|
|
|
|
|
|
|
| 564 |
if not filename and original_relative_path:
|
| 565 |
-
filename = Path(original_relative_path).name
|
| 566 |
species = _pick_row_value(row, ["logical_group", "scientific_name", "species"])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 567 |
|
| 568 |
match = None
|
| 569 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 570 |
match = detection_by_species_and_source.get((species, filename))
|
| 571 |
if match is None and filename:
|
| 572 |
match = detection_by_source.get(filename)
|
| 573 |
if match:
|
| 574 |
for key, value in match.items():
|
| 575 |
combined.setdefault(key, value)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
merged.append(combined)
|
| 577 |
return merged
|
| 578 |
|
|
@@ -662,13 +855,14 @@ def _load_detections_from_parquet_shards(
|
|
| 662 |
|
| 663 |
|
| 664 |
def _parse_segment_filename_hint(filename: str) -> tuple[float, float, float]:
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
confidence
|
| 671 |
-
|
|
|
|
| 672 |
|
| 673 |
# Fallback pattern without confidence: ..._12.0-15.0s
|
| 674 |
basic_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s", filename)
|
|
|
|
| 123 |
]
|
| 124 |
|
| 125 |
|
| 126 |
+
def _is_blank_value(value: object) -> bool:
|
| 127 |
+
if value is None:
|
| 128 |
+
return True
|
| 129 |
+
try:
|
| 130 |
+
if isinstance(value, float) and np.isnan(value):
|
| 131 |
+
return True
|
| 132 |
+
if isinstance(value, np.generic):
|
| 133 |
+
return _is_blank_value(value.item())
|
| 134 |
+
except Exception:
|
| 135 |
+
pass
|
| 136 |
+
text = str(value).strip()
|
| 137 |
+
return not text or text.lower() in {"nan", "none", "null"}
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _normalized_field_name(value: str) -> str:
|
| 141 |
+
return re.sub(r"[^a-z0-9]+", "", value.strip().lower())
|
| 142 |
+
|
| 143 |
+
|
| 144 |
def _pick_row_value(raw: dict[str, object], keys: list[str]) -> str:
|
| 145 |
+
normalized_lookup: dict[str, object] = {}
|
| 146 |
+
for raw_key, raw_value in raw.items():
|
| 147 |
+
normalized_lookup.setdefault(_normalized_field_name(str(raw_key)), raw_value)
|
| 148 |
+
|
| 149 |
for key in keys:
|
| 150 |
value = raw.get(key)
|
| 151 |
+
if _is_blank_value(value):
|
| 152 |
+
value = normalized_lookup.get(_normalized_field_name(key))
|
| 153 |
+
if _is_blank_value(value):
|
| 154 |
continue
|
| 155 |
text = str(value).strip()
|
| 156 |
if text:
|
|
|
|
| 172 |
return normalized
|
| 173 |
|
| 174 |
|
| 175 |
+
def _normalize_match_text(value: str) -> str:
|
| 176 |
+
return re.sub(r"[^a-z0-9]+", "", value.strip().replace("\\", "/").lower())
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _time_match_key(value: object) -> str:
|
| 180 |
+
return f"{_to_float(value, -1.0):.3f}"
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _normalize_confidence(value: object, default: float) -> float:
|
| 184 |
+
confidence = _to_float(value, default)
|
| 185 |
+
if confidence > 1.0:
|
| 186 |
+
confidence = confidence / 100.0
|
| 187 |
+
return max(0.0, min(1.0, confidence))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _parse_segment_filename_metadata(path_value: str) -> dict[str, object]:
|
| 191 |
+
normalized = str(path_value or "").replace("\\", "/").strip()
|
| 192 |
+
if not normalized:
|
| 193 |
+
return {}
|
| 194 |
+
|
| 195 |
+
filename = normalized.rsplit("/", 1)[-1]
|
| 196 |
+
suffix = Path(filename).suffix.lower() or ".wav"
|
| 197 |
+
stem = filename[: -len(suffix)] if suffix and filename.lower().endswith(suffix) else filename
|
| 198 |
+
|
| 199 |
+
uploader_key = ""
|
| 200 |
+
key_match = re.search(r"__([0-9a-fA-F]{8,40})$", stem)
|
| 201 |
+
if key_match:
|
| 202 |
+
uploader_key = key_match.group(1).lower()
|
| 203 |
+
stem = stem[: key_match.start()]
|
| 204 |
+
|
| 205 |
+
segment_match = re.match(
|
| 206 |
+
r"^(?P<source_stem>.+)_(?P<start>\d+(?:\.\d+)?)-(?P<end>\d+(?:\.\d+)?)s(?:_(?P<confidence>\d+(?:\.\d+)?%?))?$",
|
| 207 |
+
stem,
|
| 208 |
+
)
|
| 209 |
+
if not segment_match:
|
| 210 |
+
return {"uploader_key": uploader_key} if uploader_key else {}
|
| 211 |
+
|
| 212 |
+
confidence_raw = segment_match.group("confidence")
|
| 213 |
+
confidence: float | None = None
|
| 214 |
+
if confidence_raw:
|
| 215 |
+
confidence = _normalize_confidence(confidence_raw.rstrip("%"), 0.0)
|
| 216 |
+
|
| 217 |
+
return {
|
| 218 |
+
"source_file": f"{segment_match.group('source_stem')}{suffix}",
|
| 219 |
+
"start_time": float(segment_match.group("start")),
|
| 220 |
+
"end_time": float(segment_match.group("end")),
|
| 221 |
+
"confidence": confidence,
|
| 222 |
+
"uploader_key": uploader_key,
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _segment_metadata_from_row(raw: dict[str, object], audio_id: str = "") -> dict[str, object]:
|
| 227 |
+
combined: dict[str, object] = {}
|
| 228 |
+
for key in [
|
| 229 |
+
"original_relative_path",
|
| 230 |
+
"relative_path",
|
| 231 |
+
"stored_path",
|
| 232 |
+
"segment_path_in_repo",
|
| 233 |
+
"segment_relpath",
|
| 234 |
+
"segment_path",
|
| 235 |
+
"audio_id",
|
| 236 |
+
"audio_path",
|
| 237 |
+
"filename",
|
| 238 |
+
]:
|
| 239 |
+
value = _pick_row_value(raw, [key])
|
| 240 |
+
if value:
|
| 241 |
+
parsed = _parse_segment_filename_metadata(value)
|
| 242 |
+
if parsed:
|
| 243 |
+
for parsed_key, parsed_value in parsed.items():
|
| 244 |
+
if parsed_value not in (None, ""):
|
| 245 |
+
combined.setdefault(parsed_key, parsed_value)
|
| 246 |
+
if audio_id:
|
| 247 |
+
parsed = _parse_segment_filename_metadata(audio_id)
|
| 248 |
+
for parsed_key, parsed_value in parsed.items():
|
| 249 |
+
if parsed_value not in (None, ""):
|
| 250 |
+
combined.setdefault(parsed_key, parsed_value)
|
| 251 |
+
return combined
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
_CONFIDENCE_KEYS = ["confidence", "score", "probability", "prediction_confidence", "Confidence"]
|
| 255 |
+
_START_TIME_KEYS = [
|
| 256 |
+
"start_time",
|
| 257 |
+
"start",
|
| 258 |
+
"begin",
|
| 259 |
+
"offset",
|
| 260 |
+
"segment_start",
|
| 261 |
+
"Start_tim",
|
| 262 |
+
"Start_time",
|
| 263 |
+
"Start Time",
|
| 264 |
+
"Start (s)",
|
| 265 |
+
]
|
| 266 |
+
_END_TIME_KEYS = ["end_time", "end", "stop", "segment_end", "End_time", "End Time", "End (s)"]
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _derive_detection_key(
|
| 270 |
+
raw: dict[str, object],
|
| 271 |
+
*,
|
| 272 |
+
project_slug: str,
|
| 273 |
+
audio_id: str,
|
| 274 |
+
scientific_name: str,
|
| 275 |
+
start_time: float,
|
| 276 |
+
end_time: float,
|
| 277 |
+
row_index: int,
|
| 278 |
+
segment_metadata: dict[str, object],
|
| 279 |
+
) -> tuple[str, str]:
|
| 280 |
+
explicit_key = _pick_row_value(raw, ["detection_key", "segment_id", "id", "uid", "key"])
|
| 281 |
+
if explicit_key:
|
| 282 |
+
return explicit_key, "metadata"
|
| 283 |
+
|
| 284 |
+
original_relative_path = _pick_row_value(raw, ["original_relative_path", "relative_path"])
|
| 285 |
+
if original_relative_path:
|
| 286 |
+
return hashlib.sha1(original_relative_path.encode("utf-8")).hexdigest()[:16], "original_relative_path_sha1"
|
| 287 |
+
|
| 288 |
+
stored_path = _pick_row_value(raw, ["stored_path", "segment_path_in_repo"])
|
| 289 |
+
uploader_key = str(segment_metadata.get("uploader_key") or "").strip()
|
| 290 |
+
if uploader_key and len(uploader_key) >= 12:
|
| 291 |
+
return f"hfup_{uploader_key[:12]}", "stored_filename_uploader_key"
|
| 292 |
+
if stored_path:
|
| 293 |
+
return hashlib.sha1(stored_path.encode("utf-8")).hexdigest()[:16], "stored_path_sha1"
|
| 294 |
+
|
| 295 |
+
stable = f"{project_slug}|{audio_id}|{scientific_name}|{start_time:.3f}|{end_time:.3f}|{row_index}"
|
| 296 |
+
return hashlib.sha1(stable.encode("utf-8")).hexdigest()[:16], "computed_legacy"
|
| 297 |
+
|
| 298 |
+
|
| 299 |
def _build_detection_from_row(raw: dict[str, object], row_index: int, project_slug: str) -> Detection | None:
|
| 300 |
row_project = _pick_row_value(raw, ["project_slug", "project", "project_id"])
|
| 301 |
if row_project and row_project != project_slug:
|
|
|
|
| 321 |
)
|
| 322 |
if not audio_id:
|
| 323 |
return None
|
| 324 |
+
segment_metadata = _segment_metadata_from_row(raw, audio_id)
|
| 325 |
|
| 326 |
scientific_name = _pick_row_value(
|
| 327 |
raw,
|
|
|
|
| 338 |
if not scientific_name:
|
| 339 |
scientific_name = "Unknown species"
|
| 340 |
|
| 341 |
+
raw_confidence = _pick_row_value(raw, _CONFIDENCE_KEYS)
|
| 342 |
+
if raw_confidence:
|
| 343 |
+
confidence = _normalize_confidence(raw_confidence, 0.0)
|
| 344 |
+
elif segment_metadata.get("confidence") is not None:
|
| 345 |
+
confidence = _normalize_confidence(segment_metadata.get("confidence"), 0.0)
|
| 346 |
+
else:
|
| 347 |
+
confidence = 0.0
|
| 348 |
|
| 349 |
+
raw_start_time = _pick_row_value(raw, _START_TIME_KEYS)
|
| 350 |
+
raw_end_time = _pick_row_value(raw, _END_TIME_KEYS)
|
| 351 |
+
start_time = _to_float(raw_start_time, _to_float(segment_metadata.get("start_time"), 0.0))
|
| 352 |
+
end_time = _to_float(raw_end_time, _to_float(segment_metadata.get("end_time"), 0.0))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
if end_time <= 0.0:
|
| 354 |
duration = _to_float(raw.get("duration", 0.0), 0.0)
|
| 355 |
if duration > 0.0:
|
|
|
|
| 357 |
if end_time <= start_time:
|
| 358 |
end_time = start_time + 1.0
|
| 359 |
|
| 360 |
+
detection_key, detection_key_source = _derive_detection_key(
|
| 361 |
+
raw,
|
| 362 |
+
project_slug=project_slug,
|
| 363 |
+
audio_id=audio_id,
|
| 364 |
+
scientific_name=scientific_name,
|
| 365 |
+
start_time=start_time,
|
| 366 |
+
end_time=end_time,
|
| 367 |
+
row_index=row_index,
|
| 368 |
+
segment_metadata=segment_metadata,
|
| 369 |
+
)
|
| 370 |
+
source_metadata = dict(raw)
|
| 371 |
+
if segment_metadata:
|
| 372 |
+
for key, value in segment_metadata.items():
|
| 373 |
+
if value is not None and value != "":
|
| 374 |
+
source_metadata.setdefault(f"segment_{key}", value)
|
| 375 |
+
source_metadata.setdefault("detection_key_source", detection_key_source)
|
| 376 |
+
legacy_stable = f"{project_slug}|{audio_id}|{scientific_name}|{start_time:.3f}|{end_time:.3f}|{row_index}"
|
| 377 |
+
source_metadata.setdefault("legacy_detection_key", hashlib.sha1(legacy_stable.encode("utf-8")).hexdigest()[:16])
|
| 378 |
|
| 379 |
try:
|
| 380 |
return Detection(
|
|
|
|
| 384 |
confidence=confidence,
|
| 385 |
start_time=start_time,
|
| 386 |
end_time=end_time,
|
| 387 |
+
source_metadata=source_metadata,
|
| 388 |
)
|
| 389 |
except Exception:
|
| 390 |
return None
|
|
|
|
| 708 |
) -> list[dict[str, object]]:
|
| 709 |
detection_by_species_and_source: dict[tuple[str, str], dict[str, object]] = {}
|
| 710 |
detection_by_source: dict[str, dict[str, object]] = {}
|
| 711 |
+
detection_by_species_source_time: dict[tuple[str, str, str, str], dict[str, object]] = {}
|
| 712 |
+
detection_by_source_time: dict[tuple[str, str, str], dict[str, object]] = {}
|
| 713 |
|
| 714 |
for row in detection_rows:
|
| 715 |
source_file = _pick_row_value(row, ["source_file", "audio_file", "file", "filename", "path"])
|
| 716 |
species = _pick_row_value(row, ["scientific_name", "species", "label", "logical_group"])
|
| 717 |
+
start_time = _pick_row_value(row, _START_TIME_KEYS)
|
| 718 |
+
end_time = _pick_row_value(row, _END_TIME_KEYS)
|
| 719 |
+
normalized_source = _normalize_match_text(Path(source_file).name if source_file else "")
|
| 720 |
+
normalized_species = _normalize_match_text(species.replace("_", " ") if species else "")
|
| 721 |
if source_file:
|
| 722 |
detection_by_source.setdefault(source_file, row)
|
| 723 |
if species:
|
| 724 |
detection_by_species_and_source.setdefault((species, source_file), row)
|
| 725 |
+
if normalized_source and start_time and end_time:
|
| 726 |
+
time_key = (_time_match_key(start_time), _time_match_key(end_time))
|
| 727 |
+
detection_by_source_time.setdefault((normalized_source, *time_key), row)
|
| 728 |
+
if normalized_species:
|
| 729 |
+
detection_by_species_source_time.setdefault((normalized_species, normalized_source, *time_key), row)
|
| 730 |
|
| 731 |
merged: list[dict[str, object]] = []
|
| 732 |
for row in file_rows:
|
| 733 |
combined = dict(row)
|
| 734 |
original_relative_path = _pick_row_value(row, ["original_relative_path", "relative_path", "stored_path", "audio_id"])
|
| 735 |
+
segment_metadata = _segment_metadata_from_row(row)
|
| 736 |
+
segment_source_file = str(segment_metadata.get("source_file") or "")
|
| 737 |
+
filename = segment_source_file or _pick_row_value(row, ["filename"])
|
| 738 |
if not filename and original_relative_path:
|
| 739 |
+
filename = Path(original_relative_path.replace("\\", "/")).name
|
| 740 |
species = _pick_row_value(row, ["logical_group", "scientific_name", "species"])
|
| 741 |
+
normalized_source = _normalize_match_text(Path(filename).name if filename else "")
|
| 742 |
+
normalized_species = _normalize_match_text(species.replace("_", " ") if species else "")
|
| 743 |
+
start_time = segment_metadata.get("start_time")
|
| 744 |
+
end_time = segment_metadata.get("end_time")
|
| 745 |
|
| 746 |
match = None
|
| 747 |
+
if normalized_source and start_time is not None and end_time is not None:
|
| 748 |
+
time_key = (_time_match_key(start_time), _time_match_key(end_time))
|
| 749 |
+
if normalized_species:
|
| 750 |
+
match = detection_by_species_source_time.get((normalized_species, normalized_source, *time_key))
|
| 751 |
+
if match is None:
|
| 752 |
+
match = detection_by_source_time.get((normalized_source, *time_key))
|
| 753 |
+
if match is None and species and filename:
|
| 754 |
match = detection_by_species_and_source.get((species, filename))
|
| 755 |
if match is None and filename:
|
| 756 |
match = detection_by_source.get(filename)
|
| 757 |
if match:
|
| 758 |
for key, value in match.items():
|
| 759 |
combined.setdefault(key, value)
|
| 760 |
+
if segment_metadata:
|
| 761 |
+
if not _pick_row_value(combined, _START_TIME_KEYS) and segment_metadata.get("start_time") is not None:
|
| 762 |
+
combined["start_time"] = segment_metadata["start_time"]
|
| 763 |
+
if not _pick_row_value(combined, _END_TIME_KEYS) and segment_metadata.get("end_time") is not None:
|
| 764 |
+
combined["end_time"] = segment_metadata["end_time"]
|
| 765 |
+
if not _pick_row_value(combined, _CONFIDENCE_KEYS) and segment_metadata.get("confidence") is not None:
|
| 766 |
+
combined["confidence"] = segment_metadata["confidence"]
|
| 767 |
+
if segment_source_file:
|
| 768 |
+
combined.setdefault("segment_source_file", segment_source_file)
|
| 769 |
merged.append(combined)
|
| 770 |
return merged
|
| 771 |
|
|
|
|
| 855 |
|
| 856 |
|
| 857 |
def _parse_segment_filename_hint(filename: str) -> tuple[float, float, float]:
|
| 858 |
+
metadata = _parse_segment_filename_metadata(filename)
|
| 859 |
+
if metadata:
|
| 860 |
+
start_time = _to_float(metadata.get("start_time"), 0.0)
|
| 861 |
+
end_time = _to_float(metadata.get("end_time"), 1.0)
|
| 862 |
+
confidence = metadata.get("confidence")
|
| 863 |
+
if confidence is None:
|
| 864 |
+
confidence = 0.5
|
| 865 |
+
return start_time, end_time, _normalize_confidence(confidence, 0.5)
|
| 866 |
|
| 867 |
# Fallback pattern without confidence: ..._12.0-15.0s
|
| 868 |
basic_match = re.search(r"_(\d+(?:\.\d+)?)\-(\d+(?:\.\d+)?)s", filename)
|
tests/unit/test_app_factory_audio_helpers.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from dataclasses import dataclass
|
| 2 |
from datetime import date
|
| 3 |
from pathlib import Path
|
|
|
|
| 4 |
import json
|
| 5 |
|
| 6 |
import pandas as pd
|
|
@@ -1362,6 +1363,18 @@ def test_parse_segment_filename_hint_reads_time_and_confidence() -> None:
|
|
| 1362 |
assert conf == 0.85
|
| 1363 |
|
| 1364 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1365 |
def test_build_detection_from_row_prefers_segment_path_for_audio_id() -> None:
|
| 1366 |
from src.ui import app_factory as module
|
| 1367 |
|
|
@@ -1381,6 +1394,26 @@ def test_build_detection_from_row_prefers_segment_path_for_audio_id() -> None:
|
|
| 1381 |
assert detection.audio_id == "segments/species_a/example.wav"
|
| 1382 |
|
| 1383 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1384 |
def test_load_dataset_detections_for_project_uses_parquet_shards_when_available(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 1385 |
from src.ui import app_factory as module
|
| 1386 |
|
|
@@ -1491,6 +1524,74 @@ def test_load_dataset_detections_for_project_uses_files_parquet_index(
|
|
| 1491 |
assert detections[0].end_time == 3.0
|
| 1492 |
|
| 1493 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1494 |
def test_load_dataset_detections_for_project_reads_known_files_index_before_repo_tree(
|
| 1495 |
monkeypatch: pytest.MonkeyPatch,
|
| 1496 |
tmp_path: Path,
|
|
|
|
| 1 |
from dataclasses import dataclass
|
| 2 |
from datetime import date
|
| 3 |
from pathlib import Path
|
| 4 |
+
import hashlib
|
| 5 |
import json
|
| 6 |
|
| 7 |
import pandas as pd
|
|
|
|
| 1363 |
assert conf == 0.85
|
| 1364 |
|
| 1365 |
|
| 1366 |
+
def test_parse_segment_filename_hint_reads_uploader_hash_filename() -> None:
|
| 1367 |
+
from src.ui import app_factory as module
|
| 1368 |
+
|
| 1369 |
+
start, end, conf = module._parse_segment_filename_hint(
|
| 1370 |
+
"Catim_LS-1500_U10_20250224_063500_33.0-36.0s_95__cf04dd382267.wav"
|
| 1371 |
+
)
|
| 1372 |
+
|
| 1373 |
+
assert start == 33.0
|
| 1374 |
+
assert end == 36.0
|
| 1375 |
+
assert conf == 0.95
|
| 1376 |
+
|
| 1377 |
+
|
| 1378 |
def test_build_detection_from_row_prefers_segment_path_for_audio_id() -> None:
|
| 1379 |
from src.ui import app_factory as module
|
| 1380 |
|
|
|
|
| 1394 |
assert detection.audio_id == "segments/species_a/example.wav"
|
| 1395 |
|
| 1396 |
|
| 1397 |
+
def test_build_detection_from_row_accepts_birdnet_column_variants() -> None:
|
| 1398 |
+
from src.ui import app_factory as module
|
| 1399 |
+
|
| 1400 |
+
row = {
|
| 1401 |
+
"segment_path_in_repo": "audio/segments/Accipiter_striatus/example_33.0-36.0s_95__abc123def456.wav",
|
| 1402 |
+
"Scientific Name": "Accipiter striatus",
|
| 1403 |
+
"Confidence": 0.731,
|
| 1404 |
+
"Start_tim": 33.0,
|
| 1405 |
+
"End_time": 36.0,
|
| 1406 |
+
}
|
| 1407 |
+
|
| 1408 |
+
detection = module._build_detection_from_row(row, 0, "project-a")
|
| 1409 |
+
|
| 1410 |
+
assert detection is not None
|
| 1411 |
+
assert detection.scientific_name == "Accipiter striatus"
|
| 1412 |
+
assert detection.confidence == 0.731
|
| 1413 |
+
assert detection.start_time == 33.0
|
| 1414 |
+
assert detection.end_time == 36.0
|
| 1415 |
+
|
| 1416 |
+
|
| 1417 |
def test_load_dataset_detections_for_project_uses_parquet_shards_when_available(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 1418 |
from src.ui import app_factory as module
|
| 1419 |
|
|
|
|
| 1524 |
assert detections[0].end_time == 3.0
|
| 1525 |
|
| 1526 |
|
| 1527 |
+
def test_files_index_merges_detection_csv_by_source_species_and_time(
|
| 1528 |
+
monkeypatch: pytest.MonkeyPatch,
|
| 1529 |
+
tmp_path: Path,
|
| 1530 |
+
) -> None:
|
| 1531 |
+
from src.ui import app_factory as module
|
| 1532 |
+
|
| 1533 |
+
original_relative_path = "Accipiter striatus/Catim_LS-1500_U10_20250224_063500_33.0-36.0s_95%.wav"
|
| 1534 |
+
files_index = tmp_path / "files.parquet"
|
| 1535 |
+
pd.DataFrame.from_records(
|
| 1536 |
+
[
|
| 1537 |
+
{
|
| 1538 |
+
"stored_path": "audio/Accipiter_striatus/shard-000001/Catim_LS-1500_U10_20250224_063500_33.0-36.0s_95__cf04dd382267.wav",
|
| 1539 |
+
"original_relative_path": original_relative_path,
|
| 1540 |
+
"logical_group": "Accipiter striatus",
|
| 1541 |
+
"filename": "Catim_LS-1500_U10_20250224_063500_33.0-36.0s_95%.wav",
|
| 1542 |
+
"size": 123,
|
| 1543 |
+
}
|
| 1544 |
+
]
|
| 1545 |
+
).to_parquet(files_index, index=False)
|
| 1546 |
+
|
| 1547 |
+
detections_csv = tmp_path / "detections.csv"
|
| 1548 |
+
detections_csv.write_text(
|
| 1549 |
+
"source_file,scientific_name,confidence,start_time,end_time\n"
|
| 1550 |
+
"Catim_LS-1500_U10_20250224_063500.wav,Accipiter striatus,0.812345,33,36\n",
|
| 1551 |
+
encoding="utf-8",
|
| 1552 |
+
)
|
| 1553 |
+
|
| 1554 |
+
class TreeApiShouldNotRun:
|
| 1555 |
+
def __init__(self, token: str | None = None) -> None:
|
| 1556 |
+
_ = token
|
| 1557 |
+
|
| 1558 |
+
def list_repo_files(self, repo_id: str, repo_type: str = "dataset") -> list[str]:
|
| 1559 |
+
_ = repo_id
|
| 1560 |
+
_ = repo_type
|
| 1561 |
+
raise AssertionError("tree discovery should not run when index/files.parquet is available")
|
| 1562 |
+
|
| 1563 |
+
def fake_download(repo_id: str, repo_type: str, filename: str, token: str | None = None) -> str:
|
| 1564 |
+
_ = repo_id
|
| 1565 |
+
_ = repo_type
|
| 1566 |
+
_ = token
|
| 1567 |
+
if filename == "index/files.parquet":
|
| 1568 |
+
return str(files_index)
|
| 1569 |
+
if filename == "index/detections.csv":
|
| 1570 |
+
return str(detections_csv)
|
| 1571 |
+
raise FileNotFoundError(filename)
|
| 1572 |
+
|
| 1573 |
+
monkeypatch.setattr(module, "HfApi", TreeApiShouldNotRun)
|
| 1574 |
+
monkeypatch.setattr(module, "hf_hub_download", fake_download)
|
| 1575 |
+
|
| 1576 |
+
project = Project(
|
| 1577 |
+
project_slug="ppbio-rabeca",
|
| 1578 |
+
name="PPBIO RABECA",
|
| 1579 |
+
dataset_repo_id="jrrribeiro/PPBIO-RABECA",
|
| 1580 |
+
active=True,
|
| 1581 |
+
)
|
| 1582 |
+
|
| 1583 |
+
detections, warning = module._load_dataset_detections_for_project(project)
|
| 1584 |
+
|
| 1585 |
+
assert warning == ""
|
| 1586 |
+
assert len(detections) == 1
|
| 1587 |
+
assert detections[0].detection_key == hashlib.sha1(original_relative_path.encode("utf-8")).hexdigest()[:16]
|
| 1588 |
+
assert detections[0].audio_id == "Accipiter_striatus/shard-000001/Catim_LS-1500_U10_20250224_063500_33.0-36.0s_95__cf04dd382267.wav"
|
| 1589 |
+
assert detections[0].confidence == 0.812345
|
| 1590 |
+
assert detections[0].start_time == 33.0
|
| 1591 |
+
assert detections[0].end_time == 36.0
|
| 1592 |
+
assert detections[0].source_metadata["segment_uploader_key"] == "cf04dd382267"
|
| 1593 |
+
|
| 1594 |
+
|
| 1595 |
def test_load_dataset_detections_for_project_reads_known_files_index_before_repo_tree(
|
| 1596 |
monkeypatch: pytest.MonkeyPatch,
|
| 1597 |
tmp_path: Path,
|