jrrribeiro commited on
Commit
e50e848
·
1 Parent(s): 262df77

feat(cli): implement create-project and build-index foundations

Browse files
Files changed (1) hide show
  1. cli/hf_dataset_cli.py +349 -7
cli/hf_dataset_cli.py CHANGED
@@ -1,4 +1,312 @@
1
  import argparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
 
4
  def build_parser() -> argparse.ArgumentParser:
@@ -8,25 +316,59 @@ def build_parser() -> argparse.ArgumentParser:
8
  create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
9
  create_project.add_argument("--project-slug", required=True)
10
  create_project.add_argument("--dataset-repo", required=True)
 
 
 
 
 
 
 
 
 
 
 
11
 
12
- sub.add_parser("verify-project", help="Verify project integrity")
13
  return parser
14
 
15
 
16
- def main() -> None:
17
  parser = build_parser()
18
  args = parser.parse_args()
 
19
 
20
  if args.command == "create-project":
21
- print(f"[TODO] create-project: {args.project_slug} in {args.dataset_repo}")
22
- return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
 
24
  if args.command == "verify-project":
25
- print("[TODO] verify-project")
26
- return
 
 
 
 
 
27
 
28
  parser.error("Unknown command")
 
29
 
30
 
31
  if __name__ == "__main__":
32
- main()
 
1
  import argparse
2
+ import hashlib
3
+ import json
4
+ import math
5
+ import tempfile
6
+ from dataclasses import asdict, dataclass
7
+ from datetime import UTC, datetime
8
+ from io import BytesIO
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import pandas as pd
13
+ from huggingface_hub import HfApi, hf_hub_download
14
+
15
+
16
+ SCHEMA_VERSION = "1.0.0"
17
+ REQUIRED_PROJECT_PREFIXES = (
18
+ "audio/",
19
+ "index/",
20
+ "index/shards/",
21
+ "validations/",
22
+ "audit/",
23
+ )
24
+ REQUIRED_PLACEHOLDER_FILES = (
25
+ "audio/.gitkeep",
26
+ "index/.gitkeep",
27
+ "index/shards/.gitkeep",
28
+ "validations/.gitkeep",
29
+ "audit/.gitkeep",
30
+ )
31
+ REQUIRED_DETECTIONS_COLUMNS = (
32
+ "detection_key",
33
+ "audio_id",
34
+ "scientific_name",
35
+ "confidence",
36
+ "start_time",
37
+ "end_time",
38
+ )
39
+
40
+
41
+ @dataclass(slots=True)
42
+ class ShardMetadata:
43
+ path: str
44
+ rows: int
45
+ sha256: str
46
+ size_bytes: int
47
+
48
+
49
+ def _utcnow_iso() -> str:
50
+ return datetime.now(UTC).replace(microsecond=0).isoformat()
51
+
52
+
53
+ def _upload_text(api: HfApi, dataset_repo: str, path_in_repo: str, content: str) -> None:
54
+ api.upload_file(
55
+ path_or_fileobj=BytesIO(content.encode("utf-8")),
56
+ path_in_repo=path_in_repo,
57
+ repo_id=dataset_repo,
58
+ repo_type="dataset",
59
+ )
60
+
61
+
62
+ def _upload_empty_file(api: HfApi, dataset_repo: str, path_in_repo: str) -> None:
63
+ api.upload_file(
64
+ path_or_fileobj=BytesIO(b""),
65
+ path_in_repo=path_in_repo,
66
+ repo_id=dataset_repo,
67
+ repo_type="dataset",
68
+ )
69
+
70
+
71
+ def ensure_project_dataset_structure(
72
+ api: HfApi,
73
+ project_slug: str,
74
+ dataset_repo: str,
75
+ create_private_repo: bool,
76
+ ) -> dict[str, Any]:
77
+ api.create_repo(
78
+ repo_id=dataset_repo,
79
+ repo_type="dataset",
80
+ private=create_private_repo,
81
+ exist_ok=True,
82
+ )
83
+
84
+ files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
85
+ created_paths: list[str] = []
86
+
87
+ for placeholder in REQUIRED_PLACEHOLDER_FILES:
88
+ if placeholder not in files:
89
+ _upload_empty_file(api=api, dataset_repo=dataset_repo, path_in_repo=placeholder)
90
+ created_paths.append(placeholder)
91
+
92
+ manifest_path = "manifest.json"
93
+ if manifest_path not in files:
94
+ manifest = {
95
+ "schema_version": SCHEMA_VERSION,
96
+ "project_slug": project_slug,
97
+ "dataset_repo_id": dataset_repo,
98
+ "created_at": _utcnow_iso(),
99
+ "updated_at": _utcnow_iso(),
100
+ "index": {
101
+ "total_detections": 0,
102
+ "total_audio_files": 0,
103
+ "shard_size": 0,
104
+ "shards": [],
105
+ },
106
+ }
107
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo=manifest_path, content=json.dumps(manifest, indent=2))
108
+ created_paths.append(manifest_path)
109
+
110
+ return {
111
+ "dataset_repo": dataset_repo,
112
+ "project_slug": project_slug,
113
+ "created_paths": created_paths,
114
+ }
115
+
116
+
117
+ def load_detections_table(detections_file: str) -> pd.DataFrame:
118
+ input_path = Path(detections_file)
119
+ if not input_path.exists():
120
+ raise FileNotFoundError(f"Detections file not found: {detections_file}")
121
+
122
+ suffix = input_path.suffix.lower()
123
+ if suffix == ".csv":
124
+ frame = pd.read_csv(input_path)
125
+ elif suffix in {".jsonl", ".ndjson"}:
126
+ frame = pd.read_json(input_path, lines=True)
127
+ elif suffix == ".parquet":
128
+ frame = pd.read_parquet(input_path)
129
+ else:
130
+ raise ValueError("Unsupported detections format. Use .csv, .jsonl/.ndjson, or .parquet")
131
+
132
+ missing_columns = [col for col in REQUIRED_DETECTIONS_COLUMNS if col not in frame.columns]
133
+ if missing_columns:
134
+ raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
135
+
136
+ if frame.empty:
137
+ return frame
138
+
139
+ deduped = frame.drop_duplicates(subset=["detection_key"], keep="last")
140
+ return deduped.sort_values(by=["detection_key"]).reset_index(drop=True)
141
+
142
+
143
+ def _file_sha256(file_path: Path) -> str:
144
+ hasher = hashlib.sha256()
145
+ with file_path.open("rb") as handle:
146
+ for chunk in iter(lambda: handle.read(65536), b""):
147
+ hasher.update(chunk)
148
+ return hasher.hexdigest()
149
+
150
+
151
+ def build_shards_in_directory(frame: pd.DataFrame, output_dir: Path, shard_size: int) -> list[ShardMetadata]:
152
+ if shard_size <= 0:
153
+ raise ValueError("shard_size must be greater than zero")
154
+
155
+ output_dir.mkdir(parents=True, exist_ok=True)
156
+ shards: list[ShardMetadata] = []
157
+
158
+ if frame.empty:
159
+ return shards
160
+
161
+ total_rows = len(frame)
162
+ total_shards = math.ceil(total_rows / shard_size)
163
+
164
+ for index in range(total_shards):
165
+ start = index * shard_size
166
+ end = min(start + shard_size, total_rows)
167
+ shard_frame = frame.iloc[start:end]
168
+ shard_filename = f"shard-{index:05d}.parquet"
169
+ local_path = output_dir / shard_filename
170
+ shard_frame.to_parquet(local_path, index=False)
171
+
172
+ shards.append(
173
+ ShardMetadata(
174
+ path=f"index/shards/{shard_filename}",
175
+ rows=len(shard_frame),
176
+ sha256=_file_sha256(local_path),
177
+ size_bytes=local_path.stat().st_size,
178
+ )
179
+ )
180
+
181
+ return shards
182
+
183
+
184
+ def build_manifest_payload(
185
+ project_slug: str,
186
+ dataset_repo: str,
187
+ frame: pd.DataFrame,
188
+ shard_size: int,
189
+ shard_metadata: list[ShardMetadata],
190
+ ) -> dict[str, Any]:
191
+ now = _utcnow_iso()
192
+ return {
193
+ "schema_version": SCHEMA_VERSION,
194
+ "project_slug": project_slug,
195
+ "dataset_repo_id": dataset_repo,
196
+ "updated_at": now,
197
+ "index": {
198
+ "total_detections": len(frame),
199
+ "total_audio_files": int(frame["audio_id"].nunique()) if not frame.empty else 0,
200
+ "shard_size": shard_size,
201
+ "shards": [asdict(item) for item in shard_metadata],
202
+ },
203
+ }
204
+
205
+
206
+ def build_and_upload_index(
207
+ api: HfApi,
208
+ project_slug: str,
209
+ dataset_repo: str,
210
+ detections_file: str,
211
+ shard_size: int,
212
+ ) -> dict[str, Any]:
213
+ ensure_project_dataset_structure(
214
+ api=api,
215
+ project_slug=project_slug,
216
+ dataset_repo=dataset_repo,
217
+ create_private_repo=False,
218
+ )
219
+
220
+ frame = load_detections_table(detections_file=detections_file)
221
+
222
+ with tempfile.TemporaryDirectory(prefix="birdnet-index-") as temp_dir_str:
223
+ temp_dir = Path(temp_dir_str)
224
+ shard_dir = temp_dir / "index" / "shards"
225
+ metadata = build_shards_in_directory(frame=frame, output_dir=shard_dir, shard_size=shard_size)
226
+
227
+ for item in metadata:
228
+ local_path = temp_dir / item.path
229
+ api.upload_file(
230
+ path_or_fileobj=str(local_path),
231
+ path_in_repo=item.path,
232
+ repo_id=dataset_repo,
233
+ repo_type="dataset",
234
+ )
235
+
236
+ manifest = build_manifest_payload(
237
+ project_slug=project_slug,
238
+ dataset_repo=dataset_repo,
239
+ frame=frame,
240
+ shard_size=shard_size,
241
+ shard_metadata=metadata,
242
+ )
243
+ _upload_text(api=api, dataset_repo=dataset_repo, path_in_repo="manifest.json", content=json.dumps(manifest, indent=2))
244
+
245
+ return {
246
+ "project_slug": project_slug,
247
+ "dataset_repo": dataset_repo,
248
+ "total_detections": manifest["index"]["total_detections"],
249
+ "total_audio_files": manifest["index"]["total_audio_files"],
250
+ "total_shards": len(metadata),
251
+ }
252
+
253
+
254
+ def collect_verify_errors(repo_files: set[str], manifest_payload: dict[str, Any], project_slug: str) -> list[str]:
255
+ errors: list[str] = []
256
+
257
+ for prefix in REQUIRED_PROJECT_PREFIXES:
258
+ if not any(path.startswith(prefix) for path in repo_files):
259
+ errors.append(f"Missing prefix in dataset repo: {prefix}")
260
+
261
+ if manifest_payload.get("schema_version") != SCHEMA_VERSION:
262
+ errors.append(
263
+ f"Unexpected schema_version in manifest: {manifest_payload.get('schema_version')} (expected {SCHEMA_VERSION})"
264
+ )
265
+
266
+ if manifest_payload.get("project_slug") != project_slug:
267
+ errors.append(
268
+ f"Manifest project_slug mismatch: {manifest_payload.get('project_slug')} (expected {project_slug})"
269
+ )
270
+
271
+ index_section = manifest_payload.get("index", {})
272
+ shards = index_section.get("shards", [])
273
+ if not isinstance(shards, list):
274
+ errors.append("Manifest index.shards must be a list")
275
+ return errors
276
+
277
+ for shard in shards:
278
+ shard_path = shard.get("path")
279
+ if not shard_path or not isinstance(shard_path, str):
280
+ errors.append("Shard entry missing path")
281
+ continue
282
+ if shard_path not in repo_files:
283
+ errors.append(f"Shard referenced in manifest not found in repo: {shard_path}")
284
+
285
+ return errors
286
+
287
+
288
+ def verify_project(api: HfApi, project_slug: str, dataset_repo: str) -> dict[str, Any]:
289
+ repo_files = set(api.list_repo_files(repo_id=dataset_repo, repo_type="dataset"))
290
+ if "manifest.json" not in repo_files:
291
+ return {
292
+ "ok": False,
293
+ "errors": ["manifest.json not found in dataset repository"],
294
+ }
295
+
296
+ manifest_local = hf_hub_download(
297
+ repo_id=dataset_repo,
298
+ repo_type="dataset",
299
+ filename="manifest.json",
300
+ )
301
+ manifest_payload = json.loads(Path(manifest_local).read_text(encoding="utf-8"))
302
+ errors = collect_verify_errors(repo_files=repo_files, manifest_payload=manifest_payload, project_slug=project_slug)
303
+
304
+ return {
305
+ "ok": len(errors) == 0,
306
+ "errors": errors,
307
+ "total_files": len(repo_files),
308
+ "total_shards": len(manifest_payload.get("index", {}).get("shards", [])),
309
+ }
310
 
311
 
312
  def build_parser() -> argparse.ArgumentParser:
 
316
  create_project = sub.add_parser("create-project", help="Create project scaffold in HF dataset")
317
  create_project.add_argument("--project-slug", required=True)
318
  create_project.add_argument("--dataset-repo", required=True)
319
+ create_project.add_argument("--private", action="store_true", help="Create private dataset repository when absent")
320
+
321
+ build_index = sub.add_parser("build-index", help="Build and upload initial detection index")
322
+ build_index.add_argument("--project-slug", required=True)
323
+ build_index.add_argument("--dataset-repo", required=True)
324
+ build_index.add_argument("--detections-file", required=True)
325
+ build_index.add_argument("--shard-size", type=int, default=10000)
326
+
327
+ verify_project = sub.add_parser("verify-project", help="Verify project integrity")
328
+ verify_project.add_argument("--project-slug", required=True)
329
+ verify_project.add_argument("--dataset-repo", required=True)
330
 
 
331
  return parser
332
 
333
 
334
+ def main() -> int:
335
  parser = build_parser()
336
  args = parser.parse_args()
337
+ api = HfApi()
338
 
339
  if args.command == "create-project":
340
+ result = ensure_project_dataset_structure(
341
+ api=api,
342
+ project_slug=args.project_slug,
343
+ dataset_repo=args.dataset_repo,
344
+ create_private_repo=bool(args.private),
345
+ )
346
+ print(json.dumps(result, indent=2))
347
+ return 0
348
+
349
+ if args.command == "build-index":
350
+ result = build_and_upload_index(
351
+ api=api,
352
+ project_slug=args.project_slug,
353
+ dataset_repo=args.dataset_repo,
354
+ detections_file=args.detections_file,
355
+ shard_size=args.shard_size,
356
+ )
357
+ print(json.dumps(result, indent=2))
358
+ return 0
359
 
360
  if args.command == "verify-project":
361
+ result = verify_project(
362
+ api=api,
363
+ project_slug=args.project_slug,
364
+ dataset_repo=args.dataset_repo,
365
+ )
366
+ print(json.dumps(result, indent=2))
367
+ return 0 if result["ok"] else 1
368
 
369
  parser.error("Unknown command")
370
+ return 2
371
 
372
 
373
  if __name__ == "__main__":
374
+ raise SystemExit(main())