jrrribeiro commited on
Commit
20c353c
·
1 Parent(s): eb46582

test/docs: cover sync-audio behavior and CLI usage

Browse files
Files changed (2) hide show
  1. README.md +4 -1
  2. tests/unit/test_hf_dataset_cli.py +96 -0
README.md CHANGED
@@ -22,7 +22,10 @@ Comandos principais:
22
  2. Gerar manifesto e shards iniciais de indice:
23
  python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
24
 
25
- 3. Verificar consistencia basica do projeto:
 
 
 
26
  python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
27
 
28
  Estrutura base criada por projeto/dataset:
 
22
  2. Gerar manifesto e shards iniciais de indice:
23
  python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
24
 
25
+ 3. Fazer upload de audio em lotes com retry e resume:
26
+ python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
27
+
28
+ 4. Verificar consistencia basica do projeto:
29
  python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
30
 
31
  Estrutura base criada por projeto/dataset:
tests/unit/test_hf_dataset_cli.py CHANGED
@@ -6,10 +6,15 @@ import pytest
6
 
7
  from cli.hf_dataset_cli import (
8
  SCHEMA_VERSION,
 
 
 
9
  build_manifest_payload,
10
  build_shards_in_directory,
11
  collect_verify_errors,
 
12
  load_detections_table,
 
13
  )
14
 
15
 
@@ -133,3 +138,94 @@ def test_manifest_json_roundtrip(tmp_path: Path) -> None:
133
 
134
  loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
135
  assert loaded["project_slug"] == "ppbio-rabeca"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
  from cli.hf_dataset_cli import (
8
  SCHEMA_VERSION,
9
+ _chunk_items,
10
+ _load_resume_state,
11
+ _save_resume_state,
12
  build_manifest_payload,
13
  build_shards_in_directory,
14
  collect_verify_errors,
15
+ discover_audio_files,
16
  load_detections_table,
17
+ sync_audio_batches,
18
  )
19
 
20
 
 
138
 
139
  loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
140
  assert loaded["project_slug"] == "ppbio-rabeca"
141
+
142
+
143
+ def test_discover_audio_files_filters_supported_extensions(tmp_path: Path) -> None:
144
+ audio_dir = tmp_path / "audio"
145
+ audio_dir.mkdir(parents=True)
146
+ (audio_dir / "a.wav").write_bytes(b"wav")
147
+ (audio_dir / "b.mp3").write_bytes(b"mp3")
148
+ (audio_dir / "c.txt").write_text("ignore", encoding="utf-8")
149
+
150
+ files = discover_audio_files(str(audio_dir))
151
+ names = [item.name for item in files]
152
+
153
+ assert names == ["a.wav", "b.mp3"]
154
+
155
+
156
+ def test_chunk_items() -> None:
157
+ files = [Path(f"f{i}.wav") for i in range(5)]
158
+ chunks = _chunk_items(files, 2)
159
+
160
+ assert len(chunks) == 3
161
+ assert len(chunks[0]) == 2
162
+ assert len(chunks[1]) == 2
163
+ assert len(chunks[2]) == 1
164
+
165
+
166
+ def test_resume_state_roundtrip(tmp_path: Path) -> None:
167
+ state_file = tmp_path / "state.json"
168
+ _save_resume_state(state_file=state_file, uploaded={"audio/a.wav"}, failed={"audio/b.wav"})
169
+
170
+ loaded = _load_resume_state(state_file=state_file)
171
+
172
+ assert loaded["uploaded"] == ["audio/a.wav"]
173
+ assert loaded["failed"] == ["audio/b.wav"]
174
+
175
+
176
+ def test_sync_audio_batches_skips_remote_and_checkpointed(tmp_path: Path) -> None:
177
+ class FakeApi:
178
+ def __init__(self) -> None:
179
+ self.files = {
180
+ "audio/existing.wav",
181
+ "manifest.json",
182
+ "audio/.gitkeep",
183
+ "index/.gitkeep",
184
+ "index/shards/.gitkeep",
185
+ "validations/.gitkeep",
186
+ "audit/.gitkeep",
187
+ }
188
+ self.uploaded: list[str] = []
189
+
190
+ def create_repo(self, **kwargs: object) -> None:
191
+ _ = kwargs
192
+
193
+ def list_repo_files(self, **kwargs: object) -> list[str]:
194
+ _ = kwargs
195
+ return sorted(self.files)
196
+
197
+ def upload_file(self, *, path_or_fileobj: object, path_in_repo: str, **kwargs: object) -> None:
198
+ _ = path_or_fileobj
199
+ _ = kwargs
200
+ self.uploaded.append(path_in_repo)
201
+ self.files.add(path_in_repo)
202
+
203
+ audio_dir = tmp_path / "audio"
204
+ audio_dir.mkdir(parents=True)
205
+ (audio_dir / "existing.wav").write_bytes(b"x")
206
+ (audio_dir / "new.wav").write_bytes(b"x")
207
+ (audio_dir / "checkpointed.wav").write_bytes(b"x")
208
+
209
+ state_file = tmp_path / "resume.json"
210
+ state_file.write_text(
211
+ json.dumps({"uploaded": ["audio/checkpointed.wav"], "failed": []}),
212
+ encoding="utf-8",
213
+ )
214
+
215
+ api = FakeApi()
216
+ result = sync_audio_batches(
217
+ api=api,
218
+ project_slug="ppbio-rabeca",
219
+ dataset_repo="org/birdnet-ppbio-rabeca-dataset",
220
+ local_audio_dir=str(audio_dir),
221
+ batch_size=2,
222
+ max_retries=1,
223
+ retry_backoff_seconds=0,
224
+ resume_state_file=str(state_file),
225
+ )
226
+
227
+ assert result["total_local_audio_files"] == 3
228
+ assert result["pending_uploads"] == 1
229
+ assert result["uploaded_now"] == 1
230
+ assert result["failed"] == 0
231
+ assert api.uploaded == ["audio/new.wav"]