Spaces:
Sleeping
Sleeping
File size: 5,989 Bytes
2f1a5fb 7c83d76 2f1a5fb 7c83d76 2f1a5fb 7c83d76 2f1a5fb 7c83d76 055fe47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | from __future__ import annotations
import os
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import threading
from .config import get_session_root
from .exceptions import SessionError
class SessionManager:
"""Persist and recover upload checkpoints."""
def __init__(self, session_id: str | None = None):
if session_id is None:
now = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
session_id = f"upload-{now}"
self.session_id = session_id
self.session_dir = get_session_root() / session_id
self.session_dir.mkdir(parents=True, exist_ok=True)
self._lock = threading.Lock()
@classmethod
def create_session(cls, metadata: dict[str, Any] | None = None) -> "SessionManager":
session = cls()
if metadata:
session.save_metadata(metadata)
return session
@property
def checkpoint_path(self) -> Path:
return self.session_dir / "checkpoint.json"
@property
def metadata_path(self) -> Path:
return self.session_dir / "metadata.json"
def _atomic_write_json(self, path: Path, payload: dict[str, Any]) -> None:
temp_path = path.with_suffix(path.suffix + ".tmp")
serialized = json.dumps(payload, ensure_ascii=True, indent=2) + "\n"
temp_path.write_text(serialized, encoding="utf-8")
with temp_path.open("a", encoding="utf-8") as handle:
handle.flush()
os.fsync(handle.fileno())
temp_path.replace(path)
def save_metadata(self, payload: dict[str, Any]) -> None:
self._atomic_write_json(self.metadata_path, payload)
def load_metadata(self) -> dict[str, Any]:
if not self.metadata_path.exists():
return {}
raw = self.metadata_path.read_text(encoding="utf-8")
try:
return json.loads(raw)
except json.JSONDecodeError as exc:
raise SessionError(f"Invalid metadata JSON for session '{self.session_id}'") from exc
def save_checkpoint(self, payload: dict[str, Any]) -> None:
self._atomic_write_json(self.checkpoint_path, payload)
def load_checkpoint(self) -> dict[str, Any]:
if not self.checkpoint_path.exists():
return {}
raw = self.checkpoint_path.read_text(encoding="utf-8")
try:
return json.loads(raw)
except json.JSONDecodeError as exc:
raise SessionError(f"Invalid checkpoint JSON for session '{self.session_id}'") from exc
def mark_file_done(self, *, remote_path: str, bytes_uploaded: int, status: str = "uploaded") -> dict[str, Any]:
with self._lock:
checkpoint = self.load_checkpoint()
checkpoint.setdefault("uploaded_files", [])
checkpoint.setdefault("uploaded", 0)
checkpoint.setdefault("failed", 0)
checkpoint["uploaded"] = int(checkpoint["uploaded"]) + 1
checkpoint["status"] = status
checkpoint["last_completed_file"] = remote_path
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
checkpoint["bytes_uploaded"] = int(checkpoint.get("bytes_uploaded", 0)) + int(bytes_uploaded)
checkpoint["uploaded_files"].append(remote_path)
self.save_checkpoint(checkpoint)
return checkpoint
def mark_file_failed(self, *, remote_path: str, error: str) -> dict[str, Any]:
with self._lock:
checkpoint = self.load_checkpoint()
checkpoint.setdefault("failed_files", [])
checkpoint.setdefault("uploaded", 0)
checkpoint.setdefault("failed", 0)
checkpoint["failed"] = int(checkpoint["failed"]) + 1
checkpoint["status"] = "failed"
checkpoint["last_failed_file"] = remote_path
checkpoint["last_error"] = error
checkpoint["last_update_at"] = datetime.now(UTC).isoformat()
checkpoint["failed_files"].append({"remote_path": remote_path, "error": error})
self.save_checkpoint(checkpoint)
return checkpoint
def mark_chunk_uploaded(self, *, remote_path: str, chunk_index: int, chunk_bytes: int) -> dict[str, Any]:
"""Track progress for chunked uploads of large files.
Args:
remote_path: Remote file path in dataset
chunk_index: Zero-based chunk index
chunk_bytes: Number of bytes in this chunk
Returns:
Updated checkpoint
"""
with self._lock:
checkpoint = self.load_checkpoint()
checkpoint.setdefault("chunked_uploads", {})
if remote_path not in checkpoint["chunked_uploads"]:
checkpoint["chunked_uploads"][remote_path] = {
"chunks_completed": [],
"total_bytes_uploaded": 0,
"status": "in_progress"
}
file_chunks = checkpoint["chunked_uploads"][remote_path]
file_chunks["chunks_completed"].append(chunk_index)
file_chunks["total_bytes_uploaded"] = int(file_chunks.get("total_bytes_uploaded", 0)) + chunk_bytes
file_chunks["last_chunk_at"] = datetime.now(UTC).isoformat()
self.save_checkpoint(checkpoint)
return checkpoint
def get_chunk_status(self, remote_path: str) -> dict[str, Any]:
"""Get upload status for a chunked file.
Args:
remote_path: Remote file path in dataset
Returns:
Chunk status dict with 'chunks_completed', 'total_bytes_uploaded', etc.
"""
checkpoint = self.load_checkpoint()
chunked_uploads = checkpoint.get("chunked_uploads", {})
return chunked_uploads.get(remote_path, {
"chunks_completed": [],
"total_bytes_uploaded": 0,
"status": "not_started"
})
|