jrrribeiro commited on
Commit
530fa70
·
1 Parent(s): bc22f19

feat(projects): add stable project IDs and harden concurrent validation writes

Browse files
src/cli/project_cli.py CHANGED
@@ -5,6 +5,7 @@ import json
5
  import re
6
  from pathlib import Path
7
  from typing import Any
 
8
 
9
 
10
  DEFAULT_DATASET_DIRS = [
@@ -119,6 +120,7 @@ def cmd_create_project(args: argparse.Namespace) -> int:
119
  return EXIT_VALIDATION_ERROR
120
 
121
  project_entry: dict[str, Any] = {
 
122
  "project_slug": args.slug,
123
  "name": args.name,
124
  "dataset_repo_id": args.dataset_repo_id,
 
5
  import re
6
  from pathlib import Path
7
  from typing import Any
8
+ from uuid import uuid4
9
 
10
 
11
  DEFAULT_DATASET_DIRS = [
 
120
  return EXIT_VALIDATION_ERROR
121
 
122
  project_entry: dict[str, Any] = {
123
+ "project_id": str(uuid4()),
124
  "project_slug": args.slug,
125
  "name": args.name,
126
  "dataset_repo_id": args.dataset_repo_id,
src/domain/models.py CHANGED
@@ -1,5 +1,6 @@
1
  from enum import Enum
2
  from typing import Optional
 
3
 
4
  from pydantic import BaseModel, Field
5
 
@@ -16,6 +17,7 @@ class User(BaseModel):
16
 
17
 
18
  class Project(BaseModel):
 
19
  project_slug: str = Field(min_length=3)
20
  name: str = Field(min_length=1)
21
  dataset_repo_id: str = Field(min_length=3)
 
1
  from enum import Enum
2
  from typing import Optional
3
+ from uuid import uuid4
4
 
5
  from pydantic import BaseModel, Field
6
 
 
17
 
18
 
19
  class Project(BaseModel):
20
+ project_id: str = Field(default_factory=lambda: str(uuid4()), min_length=8)
21
  project_slug: str = Field(min_length=3)
22
  name: str = Field(min_length=1)
23
  dataset_repo_id: str = Field(min_length=3)
src/repositories/append_only_validation_repository.py CHANGED
@@ -1,6 +1,7 @@
1
  import json
2
  from datetime import UTC, datetime
3
  from pathlib import Path
 
4
  from uuid import uuid4
5
 
6
  from src.domain.models import Validation
@@ -20,53 +21,71 @@ class OptimisticLockError(Exception):
20
  class AppendOnlyValidationRepository:
21
  def __init__(self, base_dir: str) -> None:
22
  self._base_dir = Path(base_dir)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
 
24
  def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
25
  project_dir = self._base_dir / project_slug / "validations"
26
  project_dir.mkdir(parents=True, exist_ok=True)
27
 
28
- current_file = project_dir / "current.json"
29
- if current_file.exists():
30
- current_payload = json.loads(current_file.read_text(encoding="utf-8"))
31
- else:
32
- current_payload = {}
33
-
34
- current_item = current_payload.get(item.detection_key, {})
35
- current_version = int(current_item.get("version", 0))
36
- expected = expected_version if expected_version is not None else current_version
37
- if expected != current_version:
38
- raise OptimisticLockError(
39
- detection_key=item.detection_key,
40
- expected_version=expected,
41
- current_version=current_version,
42
- )
43
-
44
- new_version = current_version + 1
45
-
46
- event_date = datetime.now(UTC).strftime("%Y%m%d")
47
- events_file = project_dir / f"events-{event_date}.jsonl"
48
- event = {
49
- "event_id": str(uuid4()),
50
- "timestamp": datetime.now(UTC).isoformat(),
51
- "project_slug": project_slug,
52
- "expected_version": expected,
53
- "previous_version": current_version,
54
- "new_version": new_version,
55
- **item.model_dump(),
56
- }
57
- with events_file.open("a", encoding="utf-8") as handle:
58
- handle.write(json.dumps(event, ensure_ascii=True) + "\n")
59
-
60
- current_payload[item.detection_key] = {
61
- "status": item.status,
62
- "corrected_species": item.corrected_species,
63
- "notes": item.notes,
64
- "validator": item.validator,
65
- "updated_at": event["timestamp"],
66
- "version": new_version,
67
- }
68
- current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
69
- return new_version
 
70
 
71
  def list_events(self, project_slug: str) -> list[dict[str, object]]:
72
  project_dir = self._base_dir / project_slug / "validations"
 
1
  import json
2
  from datetime import UTC, datetime
3
  from pathlib import Path
4
+ import threading
5
  from uuid import uuid4
6
 
7
  from src.domain.models import Validation
 
21
  class AppendOnlyValidationRepository:
22
  def __init__(self, base_dir: str) -> None:
23
  self._base_dir = Path(base_dir)
24
+ self._locks: dict[str, threading.Lock] = {}
25
+ self._locks_guard = threading.Lock()
26
+
27
+ def _lock_for_project(self, project_slug: str) -> threading.Lock:
28
+ with self._locks_guard:
29
+ lock = self._locks.get(project_slug)
30
+ if lock is None:
31
+ lock = threading.Lock()
32
+ self._locks[project_slug] = lock
33
+ return lock
34
+
35
+ @staticmethod
36
+ def _atomic_write_json(path: Path, payload: object) -> None:
37
+ path.parent.mkdir(parents=True, exist_ok=True)
38
+ tmp_path = path.parent / f".{path.name}.tmp.{uuid4().hex}"
39
+ tmp_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
40
+ tmp_path.replace(path)
41
 
42
  def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
43
  project_dir = self._base_dir / project_slug / "validations"
44
  project_dir.mkdir(parents=True, exist_ok=True)
45
 
46
+ with self._lock_for_project(project_slug):
47
+ current_file = project_dir / "current.json"
48
+ if current_file.exists():
49
+ current_payload = json.loads(current_file.read_text(encoding="utf-8"))
50
+ else:
51
+ current_payload = {}
52
+
53
+ current_item = current_payload.get(item.detection_key, {})
54
+ current_version = int(current_item.get("version", 0))
55
+ expected = expected_version if expected_version is not None else current_version
56
+ if expected != current_version:
57
+ raise OptimisticLockError(
58
+ detection_key=item.detection_key,
59
+ expected_version=expected,
60
+ current_version=current_version,
61
+ )
62
+
63
+ new_version = current_version + 1
64
+
65
+ event_date = datetime.now(UTC).strftime("%Y%m%d")
66
+ events_file = project_dir / f"events-{event_date}.jsonl"
67
+ event = {
68
+ "event_id": str(uuid4()),
69
+ "timestamp": datetime.now(UTC).isoformat(),
70
+ "project_slug": project_slug,
71
+ "expected_version": expected,
72
+ "previous_version": current_version,
73
+ "new_version": new_version,
74
+ **item.model_dump(),
75
+ }
76
+ with events_file.open("a", encoding="utf-8") as handle:
77
+ handle.write(json.dumps(event, ensure_ascii=True) + "\n")
78
+
79
+ current_payload[item.detection_key] = {
80
+ "status": item.status,
81
+ "corrected_species": item.corrected_species,
82
+ "notes": item.notes,
83
+ "validator": item.validator,
84
+ "updated_at": event["timestamp"],
85
+ "version": new_version,
86
+ }
87
+ self._atomic_write_json(current_file, current_payload)
88
+ return new_version
89
 
90
  def list_events(self, project_slug: str) -> list[dict[str, object]]:
91
  project_dir = self._base_dir / project_slug / "validations"
src/ui/admin_panel.py CHANGED
@@ -54,6 +54,7 @@ class AdminPanelManager:
54
  """
55
  return [
56
  {
 
57
  "project_slug": p.project_slug,
58
  "name": p.name,
59
  "dataset_repo_id": p.dataset_repo_id,
 
54
  """
55
  return [
56
  {
57
+ "project_id": p.project_id,
58
  "project_slug": p.project_slug,
59
  "name": p.name,
60
  "dataset_repo_id": p.dataset_repo_id,
src/ui/app_factory.py CHANGED
@@ -9,6 +9,7 @@ import wave
9
  from datetime import date, datetime
10
  from pathlib import Path
11
  from typing import Protocol
 
12
 
13
  import numpy as np
14
  from huggingface_hub import HfApi, hf_hub_download
@@ -742,9 +743,15 @@ def _load_projects_from_file(projects_file_path: str | None) -> list[Project]:
742
  if not isinstance(row, dict):
743
  continue
744
  try:
 
 
 
 
 
745
  projects.append(
746
  Project(
747
- project_slug=str(row.get("project_slug", "")).strip(),
 
748
  name=str(row.get("name", "")).strip(),
749
  dataset_repo_id=str(row.get("dataset_repo_id", "")).strip(),
750
  visibility=str(row.get("visibility", "collaborative")).strip() or "collaborative",
@@ -829,6 +836,7 @@ def _persist_bootstrap_state(
829
  project_rows = admin_manager.list_projects()
830
  projects_payload = [
831
  {
 
832
  "project_slug": str(project.get("project_slug", "")).strip(),
833
  "name": str(project.get("name", "")).strip(),
834
  "dataset_repo_id": str(project.get("dataset_repo_id", "")).strip(),
@@ -2677,6 +2685,7 @@ def create_app() -> gr.Blocks:
2677
 
2678
  created = admin_manager.register_project(
2679
  Project(
 
2680
  project_slug=slug,
2681
  name=name,
2682
  dataset_repo_id=repo_id,
 
9
  from datetime import date, datetime
10
  from pathlib import Path
11
  from typing import Protocol
12
+ from uuid import NAMESPACE_URL, uuid4, uuid5
13
 
14
  import numpy as np
15
  from huggingface_hub import HfApi, hf_hub_download
 
743
  if not isinstance(row, dict):
744
  continue
745
  try:
746
+ slug = str(row.get("project_slug", "")).strip()
747
+ project_id = str(row.get("project_id", "")).strip()
748
+ if not project_id and slug:
749
+ # Deterministic legacy migration so IDs are stable before first re-persist.
750
+ project_id = str(uuid5(NAMESPACE_URL, f"birdnet-validator:{slug}"))
751
  projects.append(
752
  Project(
753
+ project_id=project_id or str(uuid4()),
754
+ project_slug=slug,
755
  name=str(row.get("name", "")).strip(),
756
  dataset_repo_id=str(row.get("dataset_repo_id", "")).strip(),
757
  visibility=str(row.get("visibility", "collaborative")).strip() or "collaborative",
 
836
  project_rows = admin_manager.list_projects()
837
  projects_payload = [
838
  {
839
+ "project_id": str(project.get("project_id", "")).strip(),
840
  "project_slug": str(project.get("project_slug", "")).strip(),
841
  "name": str(project.get("name", "")).strip(),
842
  "dataset_repo_id": str(project.get("dataset_repo_id", "")).strip(),
 
2685
 
2686
  created = admin_manager.register_project(
2687
  Project(
2688
+ project_id=str(uuid4()),
2689
  project_slug=slug,
2690
  name=name,
2691
  dataset_repo_id=repo_id,