Spaces:
Sleeping
Sleeping
Commit ·
20c353c
1
Parent(s): eb46582
test/docs: cover sync-audio behavior and CLI usage
Browse files- README.md +4 -1
- tests/unit/test_hf_dataset_cli.py +96 -0
README.md
CHANGED
|
@@ -22,7 +22,10 @@ Comandos principais:
|
|
| 22 |
2. Gerar manifesto e shards iniciais de indice:
|
| 23 |
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 24 |
|
| 25 |
-
3.
|
|
|
|
|
|
|
|
|
|
| 26 |
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 27 |
|
| 28 |
Estrutura base criada por projeto/dataset:
|
|
|
|
| 22 |
2. Gerar manifesto e shards iniciais de indice:
|
| 23 |
python -m cli.hf_dataset_cli build-index --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --detections-file detections.csv --shard-size 10000
|
| 24 |
|
| 25 |
+
3. Fazer upload de audio em lotes com retry e resume:
|
| 26 |
+
python -m cli.hf_dataset_cli sync-audio --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset --local-audio-dir ./audio --batch-size 100 --max-retries 3 --resume-state-file .sync-audio-state.json
|
| 27 |
+
|
| 28 |
+
4. Verificar consistencia basica do projeto:
|
| 29 |
python -m cli.hf_dataset_cli verify-project --project-slug ppbio-rabeca --dataset-repo USUARIO/birdnet-ppbio-rabeca-dataset
|
| 30 |
|
| 31 |
Estrutura base criada por projeto/dataset:
|
tests/unit/test_hf_dataset_cli.py
CHANGED
|
@@ -6,10 +6,15 @@ import pytest
|
|
| 6 |
|
| 7 |
from cli.hf_dataset_cli import (
|
| 8 |
SCHEMA_VERSION,
|
|
|
|
|
|
|
|
|
|
| 9 |
build_manifest_payload,
|
| 10 |
build_shards_in_directory,
|
| 11 |
collect_verify_errors,
|
|
|
|
| 12 |
load_detections_table,
|
|
|
|
| 13 |
)
|
| 14 |
|
| 15 |
|
|
@@ -133,3 +138,94 @@ def test_manifest_json_roundtrip(tmp_path: Path) -> None:
|
|
| 133 |
|
| 134 |
loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 135 |
assert loaded["project_slug"] == "ppbio-rabeca"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
from cli.hf_dataset_cli import (
|
| 8 |
SCHEMA_VERSION,
|
| 9 |
+
_chunk_items,
|
| 10 |
+
_load_resume_state,
|
| 11 |
+
_save_resume_state,
|
| 12 |
build_manifest_payload,
|
| 13 |
build_shards_in_directory,
|
| 14 |
collect_verify_errors,
|
| 15 |
+
discover_audio_files,
|
| 16 |
load_detections_table,
|
| 17 |
+
sync_audio_batches,
|
| 18 |
)
|
| 19 |
|
| 20 |
|
|
|
|
| 138 |
|
| 139 |
loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 140 |
assert loaded["project_slug"] == "ppbio-rabeca"
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def test_discover_audio_files_filters_supported_extensions(tmp_path: Path) -> None:
|
| 144 |
+
audio_dir = tmp_path / "audio"
|
| 145 |
+
audio_dir.mkdir(parents=True)
|
| 146 |
+
(audio_dir / "a.wav").write_bytes(b"wav")
|
| 147 |
+
(audio_dir / "b.mp3").write_bytes(b"mp3")
|
| 148 |
+
(audio_dir / "c.txt").write_text("ignore", encoding="utf-8")
|
| 149 |
+
|
| 150 |
+
files = discover_audio_files(str(audio_dir))
|
| 151 |
+
names = [item.name for item in files]
|
| 152 |
+
|
| 153 |
+
assert names == ["a.wav", "b.mp3"]
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def test_chunk_items() -> None:
|
| 157 |
+
files = [Path(f"f{i}.wav") for i in range(5)]
|
| 158 |
+
chunks = _chunk_items(files, 2)
|
| 159 |
+
|
| 160 |
+
assert len(chunks) == 3
|
| 161 |
+
assert len(chunks[0]) == 2
|
| 162 |
+
assert len(chunks[1]) == 2
|
| 163 |
+
assert len(chunks[2]) == 1
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def test_resume_state_roundtrip(tmp_path: Path) -> None:
|
| 167 |
+
state_file = tmp_path / "state.json"
|
| 168 |
+
_save_resume_state(state_file=state_file, uploaded={"audio/a.wav"}, failed={"audio/b.wav"})
|
| 169 |
+
|
| 170 |
+
loaded = _load_resume_state(state_file=state_file)
|
| 171 |
+
|
| 172 |
+
assert loaded["uploaded"] == ["audio/a.wav"]
|
| 173 |
+
assert loaded["failed"] == ["audio/b.wav"]
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def test_sync_audio_batches_skips_remote_and_checkpointed(tmp_path: Path) -> None:
|
| 177 |
+
class FakeApi:
|
| 178 |
+
def __init__(self) -> None:
|
| 179 |
+
self.files = {
|
| 180 |
+
"audio/existing.wav",
|
| 181 |
+
"manifest.json",
|
| 182 |
+
"audio/.gitkeep",
|
| 183 |
+
"index/.gitkeep",
|
| 184 |
+
"index/shards/.gitkeep",
|
| 185 |
+
"validations/.gitkeep",
|
| 186 |
+
"audit/.gitkeep",
|
| 187 |
+
}
|
| 188 |
+
self.uploaded: list[str] = []
|
| 189 |
+
|
| 190 |
+
def create_repo(self, **kwargs: object) -> None:
|
| 191 |
+
_ = kwargs
|
| 192 |
+
|
| 193 |
+
def list_repo_files(self, **kwargs: object) -> list[str]:
|
| 194 |
+
_ = kwargs
|
| 195 |
+
return sorted(self.files)
|
| 196 |
+
|
| 197 |
+
def upload_file(self, *, path_or_fileobj: object, path_in_repo: str, **kwargs: object) -> None:
|
| 198 |
+
_ = path_or_fileobj
|
| 199 |
+
_ = kwargs
|
| 200 |
+
self.uploaded.append(path_in_repo)
|
| 201 |
+
self.files.add(path_in_repo)
|
| 202 |
+
|
| 203 |
+
audio_dir = tmp_path / "audio"
|
| 204 |
+
audio_dir.mkdir(parents=True)
|
| 205 |
+
(audio_dir / "existing.wav").write_bytes(b"x")
|
| 206 |
+
(audio_dir / "new.wav").write_bytes(b"x")
|
| 207 |
+
(audio_dir / "checkpointed.wav").write_bytes(b"x")
|
| 208 |
+
|
| 209 |
+
state_file = tmp_path / "resume.json"
|
| 210 |
+
state_file.write_text(
|
| 211 |
+
json.dumps({"uploaded": ["audio/checkpointed.wav"], "failed": []}),
|
| 212 |
+
encoding="utf-8",
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
api = FakeApi()
|
| 216 |
+
result = sync_audio_batches(
|
| 217 |
+
api=api,
|
| 218 |
+
project_slug="ppbio-rabeca",
|
| 219 |
+
dataset_repo="org/birdnet-ppbio-rabeca-dataset",
|
| 220 |
+
local_audio_dir=str(audio_dir),
|
| 221 |
+
batch_size=2,
|
| 222 |
+
max_retries=1,
|
| 223 |
+
retry_backoff_seconds=0,
|
| 224 |
+
resume_state_file=str(state_file),
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
assert result["total_local_audio_files"] == 3
|
| 228 |
+
assert result["pending_uploads"] == 1
|
| 229 |
+
assert result["uploaded_now"] == 1
|
| 230 |
+
assert result["failed"] == 0
|
| 231 |
+
assert api.uploaded == ["audio/new.wav"]
|