jrrribeiro commited on
Commit
67145bb
·
1 Parent(s): e50e848

test(cli): add unit tests for index and verify helpers

Browse files
Files changed (1) hide show
  1. tests/unit/test_hf_dataset_cli.py +135 -0
tests/unit/test_hf_dataset_cli.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import pandas as pd
5
+ import pytest
6
+
7
+ from cli.hf_dataset_cli import (
8
+ SCHEMA_VERSION,
9
+ build_manifest_payload,
10
+ build_shards_in_directory,
11
+ collect_verify_errors,
12
+ load_detections_table,
13
+ )
14
+
15
+
16
+ def _sample_rows() -> list[dict[str, object]]:
17
+ return [
18
+ {
19
+ "detection_key": "aaaaaaaaaaaaaaaa",
20
+ "audio_id": "audio_1",
21
+ "scientific_name": "Cyanocorax cyanopogon",
22
+ "confidence": 0.91,
23
+ "start_time": 0.0,
24
+ "end_time": 1.2,
25
+ },
26
+ {
27
+ "detection_key": "bbbbbbbbbbbbbbbb",
28
+ "audio_id": "audio_2",
29
+ "scientific_name": "Ramphastos toco",
30
+ "confidence": 0.88,
31
+ "start_time": 0.5,
32
+ "end_time": 2.2,
33
+ },
34
+ {
35
+ "detection_key": "cccccccccccccccc",
36
+ "audio_id": "audio_2",
37
+ "scientific_name": "Psarocolius decumanus",
38
+ "confidence": 0.77,
39
+ "start_time": 1.1,
40
+ "end_time": 2.8,
41
+ },
42
+ ]
43
+
44
+
45
+ def test_load_detections_table_csv(tmp_path: Path) -> None:
46
+ csv_path = tmp_path / "detections.csv"
47
+ pd.DataFrame(_sample_rows()).to_csv(csv_path, index=False)
48
+
49
+ loaded = load_detections_table(str(csv_path))
50
+
51
+ assert len(loaded) == 3
52
+ assert list(loaded.columns).count("detection_key") == 1
53
+
54
+
55
+ def test_load_detections_missing_columns(tmp_path: Path) -> None:
56
+ csv_path = tmp_path / "bad.csv"
57
+ pd.DataFrame([{"detection_key": "aaaaaaaaaaaaaaaa", "audio_id": "audio_1"}]).to_csv(csv_path, index=False)
58
+
59
+ with pytest.raises(ValueError):
60
+ _ = load_detections_table(str(csv_path))
61
+
62
+
63
+ def test_build_shards_in_directory(tmp_path: Path) -> None:
64
+ frame = pd.DataFrame(_sample_rows())
65
+ output_dir = tmp_path / "index" / "shards"
66
+
67
+ shards = build_shards_in_directory(frame=frame, output_dir=output_dir, shard_size=2)
68
+
69
+ assert len(shards) == 2
70
+ assert (output_dir / "shard-00000.parquet").exists()
71
+ assert (output_dir / "shard-00001.parquet").exists()
72
+ assert shards[0].rows == 2
73
+ assert shards[1].rows == 1
74
+
75
+
76
+ def test_manifest_payload_has_expected_stats() -> None:
77
+ frame = pd.DataFrame(_sample_rows())
78
+ payload = build_manifest_payload(
79
+ project_slug="ppbio-rabeca",
80
+ dataset_repo="org/birdnet-ppbio-rabeca-dataset",
81
+ frame=frame,
82
+ shard_size=1000,
83
+ shard_metadata=[],
84
+ )
85
+
86
+ assert payload["schema_version"] == SCHEMA_VERSION
87
+ assert payload["index"]["total_detections"] == 3
88
+ assert payload["index"]["total_audio_files"] == 2
89
+
90
+
91
+ def test_collect_verify_errors_for_missing_prefix_and_shard() -> None:
92
+ manifest_payload = {
93
+ "schema_version": SCHEMA_VERSION,
94
+ "project_slug": "ppbio-rabeca",
95
+ "index": {
96
+ "shards": [
97
+ {
98
+ "path": "index/shards/shard-00000.parquet",
99
+ "rows": 2,
100
+ "sha256": "abc",
101
+ "size_bytes": 10,
102
+ }
103
+ ]
104
+ },
105
+ }
106
+
107
+ repo_files = {
108
+ "manifest.json",
109
+ "audio/.gitkeep",
110
+ "index/.gitkeep",
111
+ "validations/.gitkeep",
112
+ "audit/.gitkeep",
113
+ }
114
+
115
+ errors = collect_verify_errors(
116
+ repo_files=repo_files,
117
+ manifest_payload=manifest_payload,
118
+ project_slug="ppbio-rabeca",
119
+ )
120
+
121
+ assert any("index/shards/" in message for message in errors)
122
+ assert any("Shard referenced in manifest" in message for message in errors)
123
+
124
+
125
+ def test_manifest_json_roundtrip(tmp_path: Path) -> None:
126
+ payload = {
127
+ "schema_version": SCHEMA_VERSION,
128
+ "project_slug": "ppbio-rabeca",
129
+ "index": {"shards": []},
130
+ }
131
+ manifest_path = tmp_path / "manifest.json"
132
+ manifest_path.write_text(json.dumps(payload), encoding="utf-8")
133
+
134
+ loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
135
+ assert loaded["project_slug"] == "ppbio-rabeca"