"""Optional durable audio cache in a private Hugging Face Dataset. RAM and local files stay the hot cache. A disk-backed outbox batches Hub writes in a single background worker, respecting the Hub's rate-limit reset time. """ from __future__ import annotations import logging import hashlib import json import math import re from email.utils import parsedate_to_datetime from functools import lru_cache import os from pathlib import Path import tempfile import threading import time LOGGER = logging.getLogger(__name__) ROOT = Path(__file__).resolve().parent @lru_cache(maxsize=256) def _file_digest(path: str, size: int, modified: int) -> str: digest = hashlib.sha256() with open(path, "rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def content_digest(path: Path) -> str: """Portable asset identity, memoized until the local asset changes.""" stat = path.stat() return _file_digest(str(path.resolve()), stat.st_size, stat.st_mtime_ns) def remote_name(namespace: str, path: Path) -> str: # Hash filenames avoid storing input text and shard large caches on the Hub. return f"audio/{namespace}/{path.name[:2]}/{path.name}" def _token(): return os.getenv("HF_AUDIO_CACHE_TOKEN") or os.getenv("HF_TOKEN") or None def _failure_details(exc: Exception) -> str: """Expose actionable HTTP metadata without logging bodies or credentials.""" details = [type(exc).__name__] response = getattr(exc, "response", None) status = getattr(response, "status_code", None) # requests.Response is false for 4xx/5xx; do not use its truth value here. if isinstance(status, int): details.append(f"HTTP {status}") hint = { 401: "check cache token authentication", 403: "check cache token write permission", 404: "check cache repository and token access", 409: "concurrent commit conflict", 429: "Hub rate limit reached", }.get(status) if hint: details.append(hint) elif 500 <= status <= 599: details.append("Hub server error") headers = getattr(response, "headers", {}) retry_after = headers.get("Retry-After", "") if isinstance(retry_after, str) and retry_after.isascii() and retry_after.isdecimal(): details.append(f"retry after {retry_after[:10]} seconds") return "; ".join(details) def atomic_write(path: Path, data: bytes) -> None: temporary = None try: path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".tmp", delete=False) as output: temporary = Path(output.name) output.write(data) os.replace(temporary, path) finally: if temporary is not None: temporary.unlink(missing_ok=True) def read_audio(path: Path, namespace: str) -> bytes: """Read local storage first, then restore one remote entry on a miss.""" try: return path.read_bytes() except OSError: pass repo = os.getenv("HF_AUDIO_CACHE_REPO") if repo: try: from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError try: downloaded = hf_hub_download( repo_id=repo, repo_type="dataset", filename=remote_name(namespace, path), token=_token(), etag_timeout=10, ) except EntryNotFoundError: raise FileNotFoundError(path) from None data = Path(downloaded).read_bytes() try: atomic_write(path, data) except OSError: LOGGER.warning("Could not restore local %s audio cache", namespace) try: queue = _upload_queue() if queue: queue.confirm_download(remote_name(namespace, path), data) except OSError: LOGGER.warning("Could not record a confirmed %s audio backup", namespace) return data except FileNotFoundError: raise except Exception as exc: # Never include response bodies/credentials in application logs. LOGGER.warning("Remote %s cache read failed (%s); generating audio", namespace, _failure_details(exc)) raise FileNotFoundError(path) def _retry_delay(exc: Exception, failures: int = 1) -> float: """Honor both standard Retry-After and the Hub's RateLimit reset header.""" response = getattr(exc, "response", None) status = getattr(response, "status_code", None) headers = {str(k).lower(): str(v) for k, v in getattr(response, "headers", {}).items()} delays = [] value = headers.get("retry-after", "") try: delay = float(value) except ValueError: try: delay = parsedate_to_datetime(value).timestamp() - time.time() except (ValueError, TypeError, OverflowError): delay = 0 if math.isfinite(delay) and delay > 0: delays.append(delay) if status == 429: for policy in headers.get("ratelimit", "").split(","): remaining = re.search(r"(?:^|;)\s*r\s*=\s*(\d+)", policy, re.I) reset = re.search(r"(?:^|;)\s*t\s*=\s*(\d+)", policy, re.I) if reset and (remaining is None or int(remaining[1]) == 0): delays.append(float(reset[1])) if delays: return max(delays) + 1 # Avoid retrying just before the server's boundary. if status in {401, 403, 404}: return 900 return max(300 if status == 429 else 60, min(900, 60 * 2 ** min(failures - 1, 4))) class _UploadQueue: """One writer per repository; pending bytes and cooldown survive restarts. Pending is local until a successful Hub commit. It cannot survive deletion of an ephemeral container; /cache/status makes this distinction visible. """ def __init__(self, repo: str, directory: Path, *, interval: float = 60, batch_size: int = 50, max_bytes: int = 32 * 1024 * 1024): self.repo = repo self.directory = directory self.interval = max(1, interval) self.batch_size = max(1, min(100, batch_size)) self.max_bytes = max_bytes self._condition = threading.Condition(threading.RLock()) self._flush_lock = threading.Lock() self._thread = None self._stopping = False self._pending = {} self._last_error = None self._last_success = None self._failures = 0 self._next_attempt = time.time() + self.interval for folder in ("pending", "blobs", "confirmed"): (directory / folder).mkdir(parents=True, exist_ok=True) try: state = json.loads((directory / "state.json").read_text(encoding="utf-8")) self._next_attempt = max(self._next_attempt, float(state["next_attempt"])) except (OSError, ValueError, KeyError, TypeError): pass for manifest in (directory / "pending").glob("*.json"): try: entry = json.loads(manifest.read_text(encoding="utf-8")) name, digest = entry["name"], entry["digest"] identity = self._identity(name) if (manifest.stem != identity or not re.fullmatch(r"[a-f0-9]{64}", digest) or not re.fullmatch(r"audio/[a-z0-9_]+/[a-f0-9]{2}/[a-f0-9]+\.(wav|mp3)", name)): raise ValueError("invalid outbox manifest") blob = self._blob(name, digest) if not blob.is_file(): raise ValueError("missing outbox audio") self._pending[name] = digest except (OSError, ValueError, KeyError, TypeError): LOGGER.warning("Could not restore an audio backup outbox entry") @staticmethod def _identity(name): return hashlib.sha256(name.encode("utf-8")).hexdigest() def _blob(self, name, digest): identity = hashlib.sha256((name + "\0" + digest).encode()).hexdigest() return self.directory / "blobs" / f"{identity}.bin" def _manifest(self, name): return self.directory / "pending" / f"{self._identity(name)}.json" def _receipt(self, name): return self.directory / "confirmed" / f"{self._identity(name)}.json" def enqueue(self, name: str, data: bytes) -> bool: digest = hashlib.sha256(data).hexdigest() with self._condition: if self._pending.get(name) == digest: return True try: if json.loads(self._receipt(name).read_text(encoding="utf-8"))["digest"] == digest: return True except (OSError, ValueError, KeyError, TypeError): pass old = self._pending.get(name) atomic_write(self._blob(name, digest), data) atomic_write(self._manifest(name), json.dumps({"name": name, "digest": digest}).encode()) self._pending[name] = digest if old and old != digest: self._blob(name, old).unlink(missing_ok=True) self._condition.notify_all() return True # Accepted into the durable local outbox, not uploaded yet. def _finish(self, name, digest): with self._condition: atomic_write(self._receipt(name), json.dumps({"digest": digest}).encode()) if self._pending.get(name) == digest: self._manifest(name).unlink(missing_ok=True) self._pending.pop(name) self._blob(name, digest).unlink(missing_ok=True) def confirm_download(self, name, data): # A remote hit is already backed up. Do not enqueue it again later. digest = hashlib.sha256(data).hexdigest() self._finish(name, digest) def flush_once(self) -> int: """Try one bounded batch, without sleeping or bypassing cooldown.""" if not self._flush_lock.acquire(blocking=False): return 0 try: with self._condition: if not self._pending or time.time() < self._next_attempt: return 0 batch = [] total = 0 for name, digest in self._pending.items(): blob = self._blob(name, digest) size = blob.stat().st_size if batch and (len(batch) >= self.batch_size or total + size > self.max_bytes): break data = blob.read_bytes() if hashlib.sha256(data).hexdigest() != digest: raise ValueError("Invalid pending audio checksum") batch.append((name, digest, data)) total += size from huggingface_hub import HfApi, CommitOperationAdd api = HfApi(token=_token()) infos = {item.path: item for item in api.get_paths_info( self.repo, [name for name, _, _ in batch], repo_type="dataset", )} operations = [] for name, digest, data in batch: info = infos.get(name) lfs = getattr(info, "lfs", None) if lfs: matches = getattr(lfs, "sha256", None) == digest else: blob_digest = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest() matches = info is not None and getattr(info, "blob_id", None) == blob_digest if not matches: operations.append(CommitOperationAdd(path_in_repo=name, path_or_fileobj=data)) if operations: api.create_commit(repo_id=self.repo, repo_type="dataset", operations=operations, commit_message=f"Back up {len(operations)} queued audio cache files", num_threads=2) for name, digest, _ in batch: self._finish(name, digest) with self._condition: self._last_success = time.time() self._last_error = None self._failures = 0 self._next_attempt = time.time() + self.interval self._save_state() LOGGER.info("Audio backup: confirmed %s files (%s uploaded); %s pending", len(batch), len(operations), len(self._pending)) return len(batch) except Exception as exc: with self._condition: self._failures += 1 delay = _retry_delay(exc, self._failures) self._next_attempt = time.time() + delay self._last_error = _failure_details(exc) self._save_state() LOGGER.warning("Audio backup deferred (%s); retry in %.0f seconds; %s files remain in the local outbox", self._last_error, delay, len(self._pending)) return 0 finally: self._flush_lock.release() def _save_state(self): try: atomic_write(self.directory / "state.json", json.dumps({"next_attempt": self._next_attempt}).encode()) except OSError: LOGGER.warning("Could not persist audio backup retry time") def start(self): with self._condition: if self._thread is None or not self._thread.is_alive(): self._stopping = False self._thread = threading.Thread(target=self._run, name="audio-cache-backup", daemon=True) self._thread.start() def _run(self): while True: with self._condition: if self._stopping: return delay = self._next_attempt - time.time() if not self._pending or delay > 0: self._condition.wait(timeout=min(60, max(0.1, delay)) if self._pending else 60) continue self.flush_once() def stop(self, timeout=5): with self._condition: self._stopping = True self._condition.notify_all() thread = self._thread if thread is not None: thread.join(timeout=timeout) def status(self): with self._condition: return {"enabled": True, "pending_files": len(self._pending), "retry_in_seconds": math.ceil(max(0, self._next_attempt - time.time())) if self._pending else 0, "last_error": self._last_error, "last_success_unix": self._last_success} _QUEUES = {} _QUEUES_LOCK = threading.Lock() def _upload_queue(): repo = os.getenv("HF_AUDIO_CACHE_REPO") if not repo: return None root = Path(os.getenv("HF_AUDIO_CACHE_QUEUE_DIR", str(ROOT / "models/audio_cache_uploads"))) directory = root / hashlib.sha256(repo.encode()).hexdigest()[:16] key = (repo, str(directory.resolve())) with _QUEUES_LOCK: if key not in _QUEUES: _QUEUES[key] = _UploadQueue(repo, directory, interval=float(os.getenv("HF_AUDIO_CACHE_FLUSH_INTERVAL", "60")), batch_size=int(os.getenv("HF_AUDIO_CACHE_BATCH_SIZE", "50"))) queue = _QUEUES[key] queue.start() return queue def start_audio_backup_worker(): _upload_queue() def stop_audio_backup_worker(): with _QUEUES_LOCK: queues = list(_QUEUES.values()) for queue in queues: queue.stop() def audio_cache_status(): queue = _upload_queue() return queue.status() if queue else {"enabled": False, "pending_files": 0} def backup_audio(path: Path, namespace: str, data: bytes) -> bool: """Enqueue immutable bytes locally; True means queued, not Hub-confirmed.""" try: queue = _upload_queue() return queue.enqueue(remote_name(namespace, path), data) if queue else False except (OSError, ValueError): LOGGER.warning("Could not queue %s audio backup; only the source local copy is available", namespace) return False def local_cache_files(): """Yield only generated WAV/MP3 files for migration during deployment.""" zero_root = Path(os.getenv("ZEROTTS_DATA_DIR", str(ROOT / "models/zerotts"))) roots = { "edge": Path(os.getenv("EDGE_AUDIO_CACHE_DIR", str(ROOT / "models/cache"))), "zerotts": Path(os.getenv("ZEROTTS_AUDIO_CACHE_DIR", str(zero_root / "audio_cache"))), "gwen": Path(os.getenv("GWEN_AUDIO_CACHE_DIR", str(ROOT / "models/gwen/audio_cache"))), "vieneu": Path(os.getenv("VIENEU_AUDIO_CACHE_DIR", str(ROOT / "models/vieneu/audio_cache"))), "vtts": Path(os.getenv("VTTS_AUDIO_CACHE_DIR", str( Path(os.getenv("VTTS_MODEL_DIR", str(ROOT / "models/vtts"))) / "audio_cache" ))), } for provider, directory in roots.items(): for path in sorted(directory.rglob("*")): if not path.is_file() or path.is_symlink() or path.suffix not in {".wav", ".mp3"}: continue relative = path.relative_to(directory) if len(relative.parts) == 1: namespace = provider elif len(relative.parts) == 2 and relative.parts[0] in {"encoded", "tempo", "voice_color", "loudness"}: namespace = "mp3" if relative.parts[0] == "encoded" else relative.parts[0] else: continue yield remote_name(namespace, path), path