Spaces:
Sleeping
Sleeping
Commit ·
530fa70
1
Parent(s): bc22f19
feat(projects): add stable project IDs and harden concurrent validation writes
Browse files- src/cli/project_cli.py +2 -0
- src/domain/models.py +2 -0
- src/repositories/append_only_validation_repository.py +61 -42
- src/ui/admin_panel.py +1 -0
- src/ui/app_factory.py +10 -1
src/cli/project_cli.py
CHANGED
|
@@ -5,6 +5,7 @@ import json
|
|
| 5 |
import re
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
DEFAULT_DATASET_DIRS = [
|
|
@@ -119,6 +120,7 @@ def cmd_create_project(args: argparse.Namespace) -> int:
|
|
| 119 |
return EXIT_VALIDATION_ERROR
|
| 120 |
|
| 121 |
project_entry: dict[str, Any] = {
|
|
|
|
| 122 |
"project_slug": args.slug,
|
| 123 |
"name": args.name,
|
| 124 |
"dataset_repo_id": args.dataset_repo_id,
|
|
|
|
| 5 |
import re
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import Any
|
| 8 |
+
from uuid import uuid4
|
| 9 |
|
| 10 |
|
| 11 |
DEFAULT_DATASET_DIRS = [
|
|
|
|
| 120 |
return EXIT_VALIDATION_ERROR
|
| 121 |
|
| 122 |
project_entry: dict[str, Any] = {
|
| 123 |
+
"project_id": str(uuid4()),
|
| 124 |
"project_slug": args.slug,
|
| 125 |
"name": args.name,
|
| 126 |
"dataset_repo_id": args.dataset_repo_id,
|
src/domain/models.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
from enum import Enum
|
| 2 |
from typing import Optional
|
|
|
|
| 3 |
|
| 4 |
from pydantic import BaseModel, Field
|
| 5 |
|
|
@@ -16,6 +17,7 @@ class User(BaseModel):
|
|
| 16 |
|
| 17 |
|
| 18 |
class Project(BaseModel):
|
|
|
|
| 19 |
project_slug: str = Field(min_length=3)
|
| 20 |
name: str = Field(min_length=1)
|
| 21 |
dataset_repo_id: str = Field(min_length=3)
|
|
|
|
| 1 |
from enum import Enum
|
| 2 |
from typing import Optional
|
| 3 |
+
from uuid import uuid4
|
| 4 |
|
| 5 |
from pydantic import BaseModel, Field
|
| 6 |
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
class Project(BaseModel):
|
| 20 |
+
project_id: str = Field(default_factory=lambda: str(uuid4()), min_length=8)
|
| 21 |
project_slug: str = Field(min_length=3)
|
| 22 |
name: str = Field(min_length=1)
|
| 23 |
dataset_repo_id: str = Field(min_length=3)
|
src/repositories/append_only_validation_repository.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
import json
|
| 2 |
from datetime import UTC, datetime
|
| 3 |
from pathlib import Path
|
|
|
|
| 4 |
from uuid import uuid4
|
| 5 |
|
| 6 |
from src.domain.models import Validation
|
|
@@ -20,53 +21,71 @@ class OptimisticLockError(Exception):
|
|
| 20 |
class AppendOnlyValidationRepository:
|
| 21 |
def __init__(self, base_dir: str) -> None:
|
| 22 |
self._base_dir = Path(base_dir)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 25 |
project_dir = self._base_dir / project_slug / "validations"
|
| 26 |
project_dir.mkdir(parents=True, exist_ok=True)
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
| 70 |
|
| 71 |
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 72 |
project_dir = self._base_dir / project_slug / "validations"
|
|
|
|
| 1 |
import json
|
| 2 |
from datetime import UTC, datetime
|
| 3 |
from pathlib import Path
|
| 4 |
+
import threading
|
| 5 |
from uuid import uuid4
|
| 6 |
|
| 7 |
from src.domain.models import Validation
|
|
|
|
| 21 |
class AppendOnlyValidationRepository:
|
| 22 |
def __init__(self, base_dir: str) -> None:
|
| 23 |
self._base_dir = Path(base_dir)
|
| 24 |
+
self._locks: dict[str, threading.Lock] = {}
|
| 25 |
+
self._locks_guard = threading.Lock()
|
| 26 |
+
|
| 27 |
+
def _lock_for_project(self, project_slug: str) -> threading.Lock:
|
| 28 |
+
with self._locks_guard:
|
| 29 |
+
lock = self._locks.get(project_slug)
|
| 30 |
+
if lock is None:
|
| 31 |
+
lock = threading.Lock()
|
| 32 |
+
self._locks[project_slug] = lock
|
| 33 |
+
return lock
|
| 34 |
+
|
| 35 |
+
@staticmethod
|
| 36 |
+
def _atomic_write_json(path: Path, payload: object) -> None:
|
| 37 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 38 |
+
tmp_path = path.parent / f".{path.name}.tmp.{uuid4().hex}"
|
| 39 |
+
tmp_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
| 40 |
+
tmp_path.replace(path)
|
| 41 |
|
| 42 |
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 43 |
project_dir = self._base_dir / project_slug / "validations"
|
| 44 |
project_dir.mkdir(parents=True, exist_ok=True)
|
| 45 |
|
| 46 |
+
with self._lock_for_project(project_slug):
|
| 47 |
+
current_file = project_dir / "current.json"
|
| 48 |
+
if current_file.exists():
|
| 49 |
+
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 50 |
+
else:
|
| 51 |
+
current_payload = {}
|
| 52 |
+
|
| 53 |
+
current_item = current_payload.get(item.detection_key, {})
|
| 54 |
+
current_version = int(current_item.get("version", 0))
|
| 55 |
+
expected = expected_version if expected_version is not None else current_version
|
| 56 |
+
if expected != current_version:
|
| 57 |
+
raise OptimisticLockError(
|
| 58 |
+
detection_key=item.detection_key,
|
| 59 |
+
expected_version=expected,
|
| 60 |
+
current_version=current_version,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
new_version = current_version + 1
|
| 64 |
+
|
| 65 |
+
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 66 |
+
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 67 |
+
event = {
|
| 68 |
+
"event_id": str(uuid4()),
|
| 69 |
+
"timestamp": datetime.now(UTC).isoformat(),
|
| 70 |
+
"project_slug": project_slug,
|
| 71 |
+
"expected_version": expected,
|
| 72 |
+
"previous_version": current_version,
|
| 73 |
+
"new_version": new_version,
|
| 74 |
+
**item.model_dump(),
|
| 75 |
+
}
|
| 76 |
+
with events_file.open("a", encoding="utf-8") as handle:
|
| 77 |
+
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 78 |
+
|
| 79 |
+
current_payload[item.detection_key] = {
|
| 80 |
+
"status": item.status,
|
| 81 |
+
"corrected_species": item.corrected_species,
|
| 82 |
+
"notes": item.notes,
|
| 83 |
+
"validator": item.validator,
|
| 84 |
+
"updated_at": event["timestamp"],
|
| 85 |
+
"version": new_version,
|
| 86 |
+
}
|
| 87 |
+
self._atomic_write_json(current_file, current_payload)
|
| 88 |
+
return new_version
|
| 89 |
|
| 90 |
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 91 |
project_dir = self._base_dir / project_slug / "validations"
|
src/ui/admin_panel.py
CHANGED
|
@@ -54,6 +54,7 @@ class AdminPanelManager:
|
|
| 54 |
"""
|
| 55 |
return [
|
| 56 |
{
|
|
|
|
| 57 |
"project_slug": p.project_slug,
|
| 58 |
"name": p.name,
|
| 59 |
"dataset_repo_id": p.dataset_repo_id,
|
|
|
|
| 54 |
"""
|
| 55 |
return [
|
| 56 |
{
|
| 57 |
+
"project_id": p.project_id,
|
| 58 |
"project_slug": p.project_slug,
|
| 59 |
"name": p.name,
|
| 60 |
"dataset_repo_id": p.dataset_repo_id,
|
src/ui/app_factory.py
CHANGED
|
@@ -9,6 +9,7 @@ import wave
|
|
| 9 |
from datetime import date, datetime
|
| 10 |
from pathlib import Path
|
| 11 |
from typing import Protocol
|
|
|
|
| 12 |
|
| 13 |
import numpy as np
|
| 14 |
from huggingface_hub import HfApi, hf_hub_download
|
|
@@ -742,9 +743,15 @@ def _load_projects_from_file(projects_file_path: str | None) -> list[Project]:
|
|
| 742 |
if not isinstance(row, dict):
|
| 743 |
continue
|
| 744 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 745 |
projects.append(
|
| 746 |
Project(
|
| 747 |
-
|
|
|
|
| 748 |
name=str(row.get("name", "")).strip(),
|
| 749 |
dataset_repo_id=str(row.get("dataset_repo_id", "")).strip(),
|
| 750 |
visibility=str(row.get("visibility", "collaborative")).strip() or "collaborative",
|
|
@@ -829,6 +836,7 @@ def _persist_bootstrap_state(
|
|
| 829 |
project_rows = admin_manager.list_projects()
|
| 830 |
projects_payload = [
|
| 831 |
{
|
|
|
|
| 832 |
"project_slug": str(project.get("project_slug", "")).strip(),
|
| 833 |
"name": str(project.get("name", "")).strip(),
|
| 834 |
"dataset_repo_id": str(project.get("dataset_repo_id", "")).strip(),
|
|
@@ -2677,6 +2685,7 @@ def create_app() -> gr.Blocks:
|
|
| 2677 |
|
| 2678 |
created = admin_manager.register_project(
|
| 2679 |
Project(
|
|
|
|
| 2680 |
project_slug=slug,
|
| 2681 |
name=name,
|
| 2682 |
dataset_repo_id=repo_id,
|
|
|
|
| 9 |
from datetime import date, datetime
|
| 10 |
from pathlib import Path
|
| 11 |
from typing import Protocol
|
| 12 |
+
from uuid import NAMESPACE_URL, uuid4, uuid5
|
| 13 |
|
| 14 |
import numpy as np
|
| 15 |
from huggingface_hub import HfApi, hf_hub_download
|
|
|
|
| 743 |
if not isinstance(row, dict):
|
| 744 |
continue
|
| 745 |
try:
|
| 746 |
+
slug = str(row.get("project_slug", "")).strip()
|
| 747 |
+
project_id = str(row.get("project_id", "")).strip()
|
| 748 |
+
if not project_id and slug:
|
| 749 |
+
# Deterministic legacy migration so IDs are stable before first re-persist.
|
| 750 |
+
project_id = str(uuid5(NAMESPACE_URL, f"birdnet-validator:{slug}"))
|
| 751 |
projects.append(
|
| 752 |
Project(
|
| 753 |
+
project_id=project_id or str(uuid4()),
|
| 754 |
+
project_slug=slug,
|
| 755 |
name=str(row.get("name", "")).strip(),
|
| 756 |
dataset_repo_id=str(row.get("dataset_repo_id", "")).strip(),
|
| 757 |
visibility=str(row.get("visibility", "collaborative")).strip() or "collaborative",
|
|
|
|
| 836 |
project_rows = admin_manager.list_projects()
|
| 837 |
projects_payload = [
|
| 838 |
{
|
| 839 |
+
"project_id": str(project.get("project_id", "")).strip(),
|
| 840 |
"project_slug": str(project.get("project_slug", "")).strip(),
|
| 841 |
"name": str(project.get("name", "")).strip(),
|
| 842 |
"dataset_repo_id": str(project.get("dataset_repo_id", "")).strip(),
|
|
|
|
| 2685 |
|
| 2686 |
created = admin_manager.register_project(
|
| 2687 |
Project(
|
| 2688 |
+
project_id=str(uuid4()),
|
| 2689 |
project_slug=slug,
|
| 2690 |
name=name,
|
| 2691 |
dataset_repo_id=repo_id,
|