BirdNET-Validator-App / src /cli /project_cli.py
jrrribeiro's picture
feat(projects): add stable project IDs and harden concurrent validation writes
530fa70
Raw History Blame Contribute Delete
12.1 kB
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
from uuid import uuid4
DEFAULT_DATASET_DIRS = [
"audio",
"detections",
"index",
"exports",
]
EXIT_OK = 0
EXIT_VALIDATION_ERROR = 1
EXIT_IO_ERROR = 2
SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}$")
REPO_ID_PATTERN = re.compile(r"^[^\s/]+/[^\s/]+$")
class CliValidationError(ValueError):
pass
def _safe_read_text(path: Path) -> str:
try:
return path.read_text(encoding="utf-8")
except FileNotFoundError:
raise
except UnicodeDecodeError as exc:
raise CliValidationError(f"Invalid UTF-8 content in {path}: {exc}") from exc
except OSError as exc:
raise OSError(f"Could not read {path}: {exc}") from exc
def _validate_slug(slug: str) -> None:
if not SLUG_PATTERN.fullmatch(slug):
raise CliValidationError(
"Invalid slug. Use lowercase letters, numbers, and hyphens, length 2-63."
)
def _validate_repo_id(repo_id: str) -> None:
if not REPO_ID_PATTERN.fullmatch(repo_id):
raise CliValidationError(
"Invalid dataset_repo_id. Expected format 'owner/repo' without spaces."
)
def _load_json(path: Path, default: Any) -> Any:
if not path.exists():
return default
raw = _safe_read_text(path)
try:
return json.loads(raw)
except json.JSONDecodeError as exc:
raise CliValidationError(f"Invalid JSON in {path}: {exc}") from exc
def _write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp_path = path.with_suffix(path.suffix + ".tmp")
temp_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
temp_path.replace(path)
def _project_exists(projects: list[dict[str, Any]], slug: str) -> bool:
return any(p.get("project_slug") == slug for p in projects)
def _as_project_list(payload: Any, source: Path) -> list[dict[str, Any]]:
if not isinstance(payload, list):
raise CliValidationError(f"Expected list in {source}, got {type(payload).__name__}")
for idx, item in enumerate(payload):
if not isinstance(item, dict):
raise CliValidationError(f"Project entry at index {idx} in {source} is not an object")
for field in ("project_slug", "name", "dataset_repo_id"):
if field not in item or not isinstance(item[field], str) or not item[field].strip():
raise CliValidationError(
f"Project entry at index {idx} in {source} has invalid '{field}'"
)
return payload
def _as_access_map(payload: Any, source: Path) -> dict[str, dict[str, str]]:
if not isinstance(payload, dict):
raise CliValidationError(f"Expected object in {source}, got {type(payload).__name__}")
for username, roles in payload.items():
if not isinstance(username, str) or not username.strip():
raise CliValidationError(f"Invalid username key in {source}")
if not isinstance(roles, dict):
raise CliValidationError(f"Roles for '{username}' in {source} must be an object")
for project_slug, role in roles.items():
if not isinstance(project_slug, str) or not project_slug.strip():
raise CliValidationError(f"Invalid project slug for user '{username}' in {source}")
if role not in {"admin", "validator"}:
raise CliValidationError(
f"Invalid role '{role}' for user '{username}' and project '{project_slug}' in {source}"
)
return payload
def cmd_create_project(args: argparse.Namespace) -> int:
_validate_slug(args.slug)
_validate_repo_id(args.dataset_repo_id)
if args.owner and not args.owner.strip():
raise CliValidationError("Owner cannot be blank")
projects_file = Path(args.projects_file)
projects = _as_project_list(_load_json(projects_file, []), projects_file)
if _project_exists(projects, args.slug):
print(f"ERROR: project '{args.slug}' already exists")
return EXIT_VALIDATION_ERROR
project_entry: dict[str, Any] = {
"project_id": str(uuid4()),
"project_slug": args.slug,
"name": args.name,
"dataset_repo_id": args.dataset_repo_id,
"visibility": args.visibility,
"active": not args.inactive,
}
if args.owner:
project_entry["owner_username"] = args.owner
if args.dataset_token:
project_entry["dataset_token"] = args.dataset_token
projects.append(project_entry)
projects.sort(key=lambda p: str(p.get("project_slug", "")))
_write_json(projects_file, projects)
if args.user_access_file and args.owner:
access_file = Path(args.user_access_file)
access_payload = _as_access_map(_load_json(access_file, {}), access_file)
owner_roles = access_payload.setdefault(args.owner, {})
owner_roles[args.slug] = "admin"
_write_json(access_file, access_payload)
print(f"OK: project '{args.slug}' created")
return EXIT_OK
def cmd_init_dataset(args: argparse.Namespace) -> int:
_validate_slug(args.slug)
_validate_repo_id(args.dataset_repo_id)
root = Path(args.dataset_root) / args.slug
for dirname in DEFAULT_DATASET_DIRS:
(root / dirname).mkdir(parents=True, exist_ok=True)
detections_file = root / "detections" / "detections.jsonl"
if not detections_file.exists():
detections_file.write_text("", encoding="utf-8")
metadata = {
"project_slug": args.slug,
"dataset_repo_id": args.dataset_repo_id,
}
if args.name:
metadata["name"] = args.name
_write_json(root / "index" / "project_metadata.json", metadata)
print(f"OK: dataset scaffold initialized at {root}")
return EXIT_OK
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
if not path.exists():
return rows
raw_content = _safe_read_text(path)
for line_no, raw in enumerate(raw_content.splitlines(), start=1):
line = raw.strip()
if not line:
continue
try:
parsed = json.loads(line)
except json.JSONDecodeError as exc:
raise CliValidationError(
f"Invalid JSONL in {path} at line {line_no}: {exc.msg} (column {exc.colno})"
) from exc
if not isinstance(parsed, dict):
raise CliValidationError(
f"Invalid JSONL in {path} at line {line_no}: expected object per line"
)
required = ("detection_key", "audio_id", "scientific_name", "confidence")
for field in required:
if field not in parsed:
raise CliValidationError(
f"Invalid JSONL in {path} at line {line_no}: missing '{field}'"
)
try:
parsed["confidence"] = float(parsed["confidence"])
except (TypeError, ValueError) as exc:
raise CliValidationError(
f"Invalid JSONL in {path} at line {line_no}: 'confidence' must be numeric"
) from exc
rows.append(parsed)
return rows
def cmd_build_index(args: argparse.Namespace) -> int:
_validate_slug(args.slug)
project_root = Path(args.dataset_root) / args.slug
detections_file = project_root / "detections" / "detections.jsonl"
rows = _read_jsonl(detections_file)
sorted_rows = sorted(rows, key=lambda d: d["confidence"], reverse=True)
index_payload = {
"project_slug": args.slug,
"count": len(sorted_rows),
"detections": [
{
"detection_key": r.get("detection_key"),
"audio_id": r.get("audio_id"),
"scientific_name": r.get("scientific_name"),
"confidence": r["confidence"],
}
for r in sorted_rows
],
}
_write_json(project_root / "index" / "detections_index.json", index_payload)
print(f"OK: index generated with {len(sorted_rows)} detections")
return EXIT_OK
def cmd_verify_project(args: argparse.Namespace) -> int:
_validate_slug(args.slug)
projects_file = Path(args.projects_file)
projects = _as_project_list(_load_json(projects_file, []), projects_file)
findings: list[str] = []
if not _project_exists(projects, args.slug):
findings.append(f"project '{args.slug}' not found in {projects_file}")
project_root = Path(args.dataset_root) / args.slug
for dirname in DEFAULT_DATASET_DIRS:
expected = project_root / dirname
if not expected.exists():
findings.append(f"missing path: {expected}")
required_files = [
project_root / "detections" / "detections.jsonl",
project_root / "index" / "project_metadata.json",
project_root / "index" / "detections_index.json",
]
for required in required_files:
if not required.exists():
findings.append(f"missing path: {required}")
if findings:
if args.dry_run:
print("DRY-RUN: project verification found issues:")
else:
print("ERROR: project verification failed:")
for item in findings:
print(f" - {item}")
if args.dry_run:
print("DRY-RUN: completed without failing exit code")
return EXIT_OK
return EXIT_VALIDATION_ERROR
print(f"OK: project '{args.slug}' verified")
return EXIT_OK
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="birdnet-project")
sub = parser.add_subparsers(dest="command", required=True)
create_project = sub.add_parser("create-project", help="Create project in bootstrap JSON")
create_project.add_argument("--projects-file", required=True)
create_project.add_argument("--user-access-file")
create_project.add_argument("--slug", required=True)
create_project.add_argument("--name", required=True)
create_project.add_argument("--dataset-repo-id", required=True)
create_project.add_argument("--visibility", choices=["private", "collaborative"], default="collaborative")
create_project.add_argument("--owner")
create_project.add_argument("--dataset-token")
create_project.add_argument("--inactive", action="store_true")
create_project.set_defaults(func=cmd_create_project)
init_dataset = sub.add_parser("init-dataset", help="Create initial dataset folder scaffold")
init_dataset.add_argument("--dataset-root", required=True)
init_dataset.add_argument("--slug", required=True)
init_dataset.add_argument("--dataset-repo-id", required=True)
init_dataset.add_argument("--name")
init_dataset.set_defaults(func=cmd_init_dataset)
build_index = sub.add_parser("build-index", help="Build simple index from detections JSONL")
build_index.add_argument("--dataset-root", required=True)
build_index.add_argument("--slug", required=True)
build_index.set_defaults(func=cmd_build_index)
verify_project = sub.add_parser("verify-project", help="Verify project config and local scaffold")
verify_project.add_argument("--projects-file", required=True)
verify_project.add_argument("--dataset-root", required=True)
verify_project.add_argument("--slug", required=True)
verify_project.add_argument(
"--dry-run",
action="store_true",
help="Print verification findings without returning a failure exit code",
)
verify_project.set_defaults(func=cmd_verify_project)
return parser
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
try:
args = parser.parse_args(argv)
return int(args.func(args))
except CliValidationError as exc:
print(f"ERROR: {exc}")
return EXIT_VALIDATION_ERROR
except OSError as exc:
print(f"ERROR: I/O failure: {exc}")
return EXIT_IO_ERROR
if __name__ == "__main__":
raise SystemExit(main())