Spaces:
Sleeping
Sleeping
Commit ·
a52e4fb
1
Parent(s): 0680cb5
feat(concurrency): add optimistic validation locking by expected version
Browse files
src/repositories/append_only_validation_repository.py
CHANGED
|
@@ -6,39 +6,67 @@ from uuid import uuid4
|
|
| 6 |
from src.domain.models import Validation
|
| 7 |
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
class AppendOnlyValidationRepository:
|
| 10 |
def __init__(self, base_dir: str) -> None:
|
| 11 |
self._base_dir = Path(base_dir)
|
| 12 |
|
| 13 |
-
def save_validation(self, project_slug: str, item: Validation) ->
|
| 14 |
project_dir = self._base_dir / project_slug / "validations"
|
| 15 |
project_dir.mkdir(parents=True, exist_ok=True)
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 18 |
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 19 |
event = {
|
| 20 |
"event_id": str(uuid4()),
|
| 21 |
"timestamp": datetime.now(UTC).isoformat(),
|
| 22 |
"project_slug": project_slug,
|
|
|
|
|
|
|
|
|
|
| 23 |
**item.model_dump(),
|
| 24 |
}
|
| 25 |
with events_file.open("a", encoding="utf-8") as handle:
|
| 26 |
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 27 |
|
| 28 |
-
current_file = project_dir / "current.json"
|
| 29 |
-
if current_file.exists():
|
| 30 |
-
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 31 |
-
else:
|
| 32 |
-
current_payload = {}
|
| 33 |
-
|
| 34 |
current_payload[item.detection_key] = {
|
| 35 |
"status": item.status,
|
| 36 |
"corrected_species": item.corrected_species,
|
| 37 |
"notes": item.notes,
|
| 38 |
"validator": item.validator,
|
| 39 |
"updated_at": event["timestamp"],
|
|
|
|
| 40 |
}
|
| 41 |
current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
|
|
|
|
| 42 |
|
| 43 |
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 44 |
project_dir = self._base_dir / project_slug / "validations"
|
|
|
|
| 6 |
from src.domain.models import Validation
|
| 7 |
|
| 8 |
|
| 9 |
+
class OptimisticLockError(Exception):
|
| 10 |
+
def __init__(self, detection_key: str, expected_version: int, current_version: int) -> None:
|
| 11 |
+
self.detection_key = detection_key
|
| 12 |
+
self.expected_version = expected_version
|
| 13 |
+
self.current_version = current_version
|
| 14 |
+
super().__init__(
|
| 15 |
+
f"Optimistic lock conflict for detection_key={detection_key}: "
|
| 16 |
+
f"expected_version={expected_version}, current_version={current_version}"
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
class AppendOnlyValidationRepository:
|
| 21 |
def __init__(self, base_dir: str) -> None:
|
| 22 |
self._base_dir = Path(base_dir)
|
| 23 |
|
| 24 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 25 |
project_dir = self._base_dir / project_slug / "validations"
|
| 26 |
project_dir.mkdir(parents=True, exist_ok=True)
|
| 27 |
|
| 28 |
+
current_file = project_dir / "current.json"
|
| 29 |
+
if current_file.exists():
|
| 30 |
+
current_payload = json.loads(current_file.read_text(encoding="utf-8"))
|
| 31 |
+
else:
|
| 32 |
+
current_payload = {}
|
| 33 |
+
|
| 34 |
+
current_item = current_payload.get(item.detection_key, {})
|
| 35 |
+
current_version = int(current_item.get("version", 0))
|
| 36 |
+
expected = expected_version if expected_version is not None else current_version
|
| 37 |
+
if expected != current_version:
|
| 38 |
+
raise OptimisticLockError(
|
| 39 |
+
detection_key=item.detection_key,
|
| 40 |
+
expected_version=expected,
|
| 41 |
+
current_version=current_version,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
new_version = current_version + 1
|
| 45 |
+
|
| 46 |
event_date = datetime.now(UTC).strftime("%Y%m%d")
|
| 47 |
events_file = project_dir / f"events-{event_date}.jsonl"
|
| 48 |
event = {
|
| 49 |
"event_id": str(uuid4()),
|
| 50 |
"timestamp": datetime.now(UTC).isoformat(),
|
| 51 |
"project_slug": project_slug,
|
| 52 |
+
"expected_version": expected,
|
| 53 |
+
"previous_version": current_version,
|
| 54 |
+
"new_version": new_version,
|
| 55 |
**item.model_dump(),
|
| 56 |
}
|
| 57 |
with events_file.open("a", encoding="utf-8") as handle:
|
| 58 |
handle.write(json.dumps(event, ensure_ascii=True) + "\n")
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
current_payload[item.detection_key] = {
|
| 61 |
"status": item.status,
|
| 62 |
"corrected_species": item.corrected_species,
|
| 63 |
"notes": item.notes,
|
| 64 |
"validator": item.validator,
|
| 65 |
"updated_at": event["timestamp"],
|
| 66 |
+
"version": new_version,
|
| 67 |
}
|
| 68 |
current_file.write_text(json.dumps(current_payload, indent=2), encoding="utf-8")
|
| 69 |
+
return new_version
|
| 70 |
|
| 71 |
def list_events(self, project_slug: str) -> list[dict[str, object]]:
|
| 72 |
project_dir = self._base_dir / project_slug / "validations"
|
src/repositories/contracts.py
CHANGED
|
@@ -28,7 +28,7 @@ class DetectionRepository(Protocol):
|
|
| 28 |
|
| 29 |
|
| 30 |
class ValidationRepository(Protocol):
|
| 31 |
-
def save_validation(self, project_slug: str, item: Validation) ->
|
| 32 |
|
| 33 |
|
| 34 |
class AuthRepository(Protocol):
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
class ValidationRepository(Protocol):
|
| 31 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int: ...
|
| 32 |
|
| 33 |
|
| 34 |
class AuthRepository(Protocol):
|
src/repositories/in_memory_validation_repository.py
CHANGED
|
@@ -2,14 +2,24 @@ from collections import defaultdict
|
|
| 2 |
from typing import DefaultDict
|
| 3 |
|
| 4 |
from src.domain.models import Validation
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
class InMemoryValidationRepository:
|
| 8 |
def __init__(self) -> None:
|
| 9 |
self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
-
def save_validation(self, project_slug: str, item: Validation) -> None:
|
| 12 |
self._by_project[project_slug].append(item)
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
def list_validations(self, project_slug: str) -> list[Validation]:
|
| 15 |
return list(self._by_project.get(project_slug, []))
|
|
|
|
| 2 |
from typing import DefaultDict
|
| 3 |
|
| 4 |
from src.domain.models import Validation
|
| 5 |
+
from src.repositories.append_only_validation_repository import OptimisticLockError
|
| 6 |
|
| 7 |
|
| 8 |
class InMemoryValidationRepository:
|
| 9 |
def __init__(self) -> None:
|
| 10 |
self._by_project: DefaultDict[str, list[Validation]] = defaultdict(list)
|
| 11 |
+
self._versions: DefaultDict[str, dict[str, int]] = defaultdict(dict)
|
| 12 |
+
|
| 13 |
+
def save_validation(self, project_slug: str, item: Validation, expected_version: int | None = None) -> int:
|
| 14 |
+
current_version = int(self._versions[project_slug].get(item.detection_key, 0))
|
| 15 |
+
expected = expected_version if expected_version is not None else current_version
|
| 16 |
+
if expected != current_version:
|
| 17 |
+
raise OptimisticLockError(item.detection_key, expected, current_version)
|
| 18 |
|
|
|
|
| 19 |
self._by_project[project_slug].append(item)
|
| 20 |
+
new_version = current_version + 1
|
| 21 |
+
self._versions[project_slug][item.detection_key] = new_version
|
| 22 |
+
return new_version
|
| 23 |
|
| 24 |
def list_validations(self, project_slug: str) -> list[Validation]:
|
| 25 |
return list(self._by_project.get(project_slug, []))
|
src/services/validation_service.py
CHANGED
|
@@ -1,7 +1,15 @@
|
|
|
|
|
|
|
|
| 1 |
from src.domain.models import Validation
|
| 2 |
from src.repositories.contracts import ValidationRepository
|
| 3 |
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
class ValidationService:
|
| 6 |
def __init__(self, repository: ValidationRepository) -> None:
|
| 7 |
self._repository = repository
|
|
@@ -14,7 +22,8 @@ class ValidationService:
|
|
| 14 |
validator: str,
|
| 15 |
notes: str = "",
|
| 16 |
corrected_species: str | None = None,
|
| 17 |
-
|
|
|
|
| 18 |
item = Validation(
|
| 19 |
detection_key=detection_key,
|
| 20 |
status=status,
|
|
@@ -22,5 +31,9 @@ class ValidationService:
|
|
| 22 |
notes=notes,
|
| 23 |
validator=validator,
|
| 24 |
)
|
| 25 |
-
self._repository.save_validation(
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
from src.domain.models import Validation
|
| 4 |
from src.repositories.contracts import ValidationRepository
|
| 5 |
|
| 6 |
|
| 7 |
+
@dataclass(slots=True)
|
| 8 |
+
class ValidationWriteResult:
|
| 9 |
+
item: Validation
|
| 10 |
+
new_version: int
|
| 11 |
+
|
| 12 |
+
|
| 13 |
class ValidationService:
|
| 14 |
def __init__(self, repository: ValidationRepository) -> None:
|
| 15 |
self._repository = repository
|
|
|
|
| 22 |
validator: str,
|
| 23 |
notes: str = "",
|
| 24 |
corrected_species: str | None = None,
|
| 25 |
+
expected_version: int | None = None,
|
| 26 |
+
) -> ValidationWriteResult:
|
| 27 |
item = Validation(
|
| 28 |
detection_key=detection_key,
|
| 29 |
status=status,
|
|
|
|
| 31 |
notes=notes,
|
| 32 |
validator=validator,
|
| 33 |
)
|
| 34 |
+
new_version = self._repository.save_validation(
|
| 35 |
+
project_slug=project_slug,
|
| 36 |
+
item=item,
|
| 37 |
+
expected_version=expected_version,
|
| 38 |
+
)
|
| 39 |
+
return ValidationWriteResult(item=item, new_version=new_version)
|
src/ui/app_factory.py
CHANGED
|
@@ -5,7 +5,7 @@ from typing import Protocol
|
|
| 5 |
|
| 6 |
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 7 |
from src.domain.models import Detection
|
| 8 |
-
from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository
|
| 9 |
from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
|
| 10 |
from src.services.audio_fetch_service import AudioFetchService
|
| 11 |
from src.services.detection_queue_service import DetectionQueueService
|
|
@@ -33,6 +33,7 @@ class _ValidationServiceProtocol(Protocol):
|
|
| 33 |
validator: str,
|
| 34 |
notes: str = "",
|
| 35 |
corrected_species: str | None = None,
|
|
|
|
| 36 |
) -> object: ...
|
| 37 |
|
| 38 |
|
|
@@ -122,6 +123,7 @@ def _page_to_table(
|
|
| 122 |
item.start_time,
|
| 123 |
item.end_time,
|
| 124 |
str(snapshot.get(item.detection_key, {}).get("status", "pending")),
|
|
|
|
| 125 |
]
|
| 126 |
for item in page_obj.items
|
| 127 |
]
|
|
@@ -191,6 +193,23 @@ def _extract_detection_key(rows: object, selected_index: int) -> str:
|
|
| 191 |
return detection_key
|
| 192 |
|
| 193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
def _fetch_selected_audio(
|
| 195 |
audio_service: _AudioServiceProtocol,
|
| 196 |
dataset_repo: str,
|
|
@@ -238,16 +257,25 @@ def _save_selected_validation(
|
|
| 238 |
|
| 239 |
try:
|
| 240 |
detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
|
|
|
| 241 |
_ = validation_service.validate_detection(
|
| 242 |
project_slug=project_slug,
|
| 243 |
detection_key=detection_key,
|
| 244 |
status=status_value,
|
| 245 |
validator=validator_name,
|
| 246 |
notes=notes.strip(),
|
|
|
|
| 247 |
)
|
| 248 |
if cache_key:
|
| 249 |
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 250 |
return f"Validacao salva: {detection_key} -> {status_value}", "", None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
except Exception as exc:
|
| 252 |
return f"Falha ao salvar validacao: {exc}", cache_key, None
|
| 253 |
|
|
@@ -285,6 +313,7 @@ def create_app() -> gr.Blocks:
|
|
| 285 |
"start_time",
|
| 286 |
"end_time",
|
| 287 |
"validation_status",
|
|
|
|
| 288 |
],
|
| 289 |
label="Deteccoes",
|
| 290 |
interactive=False,
|
|
|
|
| 5 |
|
| 6 |
from src.cache.ephemeral_cache_manager import EphemeralCacheManager
|
| 7 |
from src.domain.models import Detection
|
| 8 |
+
from src.repositories.append_only_validation_repository import AppendOnlyValidationRepository, OptimisticLockError
|
| 9 |
from src.repositories.in_memory_detection_repository import InMemoryDetectionRepository
|
| 10 |
from src.services.audio_fetch_service import AudioFetchService
|
| 11 |
from src.services.detection_queue_service import DetectionQueueService
|
|
|
|
| 33 |
validator: str,
|
| 34 |
notes: str = "",
|
| 35 |
corrected_species: str | None = None,
|
| 36 |
+
expected_version: int | None = None,
|
| 37 |
) -> object: ...
|
| 38 |
|
| 39 |
|
|
|
|
| 123 |
item.start_time,
|
| 124 |
item.end_time,
|
| 125 |
str(snapshot.get(item.detection_key, {}).get("status", "pending")),
|
| 126 |
+
int(snapshot.get(item.detection_key, {}).get("version", 0)),
|
| 127 |
]
|
| 128 |
for item in page_obj.items
|
| 129 |
]
|
|
|
|
| 193 |
return detection_key
|
| 194 |
|
| 195 |
|
| 196 |
+
def _extract_expected_version(rows: object, selected_index: int) -> int:
|
| 197 |
+
normalized_rows: list[list[object]]
|
| 198 |
+
|
| 199 |
+
if hasattr(rows, "values"):
|
| 200 |
+
normalized_rows = [list(item) for item in rows.values.tolist()]
|
| 201 |
+
else:
|
| 202 |
+
normalized_rows = [list(item) for item in rows] if rows else []
|
| 203 |
+
|
| 204 |
+
if not normalized_rows:
|
| 205 |
+
raise ValueError("Nenhuma deteccao carregada na tabela")
|
| 206 |
+
if selected_index < 0 or selected_index >= len(normalized_rows):
|
| 207 |
+
raise ValueError("Selecione uma deteccao valida na tabela")
|
| 208 |
+
|
| 209 |
+
value = normalized_rows[selected_index][7]
|
| 210 |
+
return int(value)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
def _fetch_selected_audio(
|
| 214 |
audio_service: _AudioServiceProtocol,
|
| 215 |
dataset_repo: str,
|
|
|
|
| 257 |
|
| 258 |
try:
|
| 259 |
detection_key = _extract_detection_key(rows=rows, selected_index=selected_index)
|
| 260 |
+
expected_version = _extract_expected_version(rows=rows, selected_index=selected_index)
|
| 261 |
_ = validation_service.validate_detection(
|
| 262 |
project_slug=project_slug,
|
| 263 |
detection_key=detection_key,
|
| 264 |
status=status_value,
|
| 265 |
validator=validator_name,
|
| 266 |
notes=notes.strip(),
|
| 267 |
+
expected_version=expected_version,
|
| 268 |
)
|
| 269 |
if cache_key:
|
| 270 |
audio_service.cleanup_after_validation(cache_key=cache_key)
|
| 271 |
return f"Validacao salva: {detection_key} -> {status_value}", "", None
|
| 272 |
+
except OptimisticLockError as exc:
|
| 273 |
+
return (
|
| 274 |
+
"Conflito de concorrencia: esta deteccao foi atualizada por outro validador "
|
| 275 |
+
f"(versao atual={exc.current_version}, esperada={exc.expected_version}). Recarregue a tabela.",
|
| 276 |
+
cache_key,
|
| 277 |
+
None,
|
| 278 |
+
)
|
| 279 |
except Exception as exc:
|
| 280 |
return f"Falha ao salvar validacao: {exc}", cache_key, None
|
| 281 |
|
|
|
|
| 313 |
"start_time",
|
| 314 |
"end_time",
|
| 315 |
"validation_status",
|
| 316 |
+
"version",
|
| 317 |
],
|
| 318 |
label="Deteccoes",
|
| 319 |
interactive=False,
|