edge-tts-api / audio_cache_store.py
tranvanhung203's picture
Batch audio backups, respect Hub rate limits and persist normalized WAV cache
c898654 verified
Raw History Blame Contribute Delete
17.6 kB
"""Optional durable audio cache in a private Hugging Face Dataset.
RAM and local files stay the hot cache. A disk-backed outbox batches Hub writes
in a single background worker, respecting the Hub's rate-limit reset time.
"""
from __future__ import annotations
import logging
import hashlib
import json
import math
import re
from email.utils import parsedate_to_datetime
from functools import lru_cache
import os
from pathlib import Path
import tempfile
import threading
import time
LOGGER = logging.getLogger(__name__)
ROOT = Path(__file__).resolve().parent
@lru_cache(maxsize=256)
def _file_digest(path: str, size: int, modified: int) -> str:
digest = hashlib.sha256()
with open(path, "rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def content_digest(path: Path) -> str:
"""Portable asset identity, memoized until the local asset changes."""
stat = path.stat()
return _file_digest(str(path.resolve()), stat.st_size, stat.st_mtime_ns)
def remote_name(namespace: str, path: Path) -> str:
# Hash filenames avoid storing input text and shard large caches on the Hub.
return f"audio/{namespace}/{path.name[:2]}/{path.name}"
def _token():
return os.getenv("HF_AUDIO_CACHE_TOKEN") or os.getenv("HF_TOKEN") or None
def _failure_details(exc: Exception) -> str:
"""Expose actionable HTTP metadata without logging bodies or credentials."""
details = [type(exc).__name__]
response = getattr(exc, "response", None)
status = getattr(response, "status_code", None)
# requests.Response is false for 4xx/5xx; do not use its truth value here.
if isinstance(status, int):
details.append(f"HTTP {status}")
hint = {
401: "check cache token authentication",
403: "check cache token write permission",
404: "check cache repository and token access",
409: "concurrent commit conflict",
429: "Hub rate limit reached",
}.get(status)
if hint:
details.append(hint)
elif 500 <= status <= 599:
details.append("Hub server error")
headers = getattr(response, "headers", {})
retry_after = headers.get("Retry-After", "")
if isinstance(retry_after, str) and retry_after.isascii() and retry_after.isdecimal():
details.append(f"retry after {retry_after[:10]} seconds")
return "; ".join(details)
def atomic_write(path: Path, data: bytes) -> None:
temporary = None
try:
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".tmp", delete=False) as output:
temporary = Path(output.name)
output.write(data)
os.replace(temporary, path)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def read_audio(path: Path, namespace: str) -> bytes:
"""Read local storage first, then restore one remote entry on a miss."""
try:
return path.read_bytes()
except OSError:
pass
repo = os.getenv("HF_AUDIO_CACHE_REPO")
if repo:
try:
from huggingface_hub import hf_hub_download
from huggingface_hub.errors import EntryNotFoundError
try:
downloaded = hf_hub_download(
repo_id=repo, repo_type="dataset", filename=remote_name(namespace, path),
token=_token(), etag_timeout=10,
)
except EntryNotFoundError:
raise FileNotFoundError(path) from None
data = Path(downloaded).read_bytes()
try:
atomic_write(path, data)
except OSError:
LOGGER.warning("Could not restore local %s audio cache", namespace)
try:
queue = _upload_queue()
if queue:
queue.confirm_download(remote_name(namespace, path), data)
except OSError:
LOGGER.warning("Could not record a confirmed %s audio backup", namespace)
return data
except FileNotFoundError:
raise
except Exception as exc:
# Never include response bodies/credentials in application logs.
LOGGER.warning("Remote %s cache read failed (%s); generating audio", namespace, _failure_details(exc))
raise FileNotFoundError(path)
def _retry_delay(exc: Exception, failures: int = 1) -> float:
"""Honor both standard Retry-After and the Hub's RateLimit reset header."""
response = getattr(exc, "response", None)
status = getattr(response, "status_code", None)
headers = {str(k).lower(): str(v) for k, v in getattr(response, "headers", {}).items()}
delays = []
value = headers.get("retry-after", "")
try:
delay = float(value)
except ValueError:
try:
delay = parsedate_to_datetime(value).timestamp() - time.time()
except (ValueError, TypeError, OverflowError):
delay = 0
if math.isfinite(delay) and delay > 0:
delays.append(delay)
if status == 429:
for policy in headers.get("ratelimit", "").split(","):
remaining = re.search(r"(?:^|;)\s*r\s*=\s*(\d+)", policy, re.I)
reset = re.search(r"(?:^|;)\s*t\s*=\s*(\d+)", policy, re.I)
if reset and (remaining is None or int(remaining[1]) == 0):
delays.append(float(reset[1]))
if delays:
return max(delays) + 1 # Avoid retrying just before the server's boundary.
if status in {401, 403, 404}:
return 900
return max(300 if status == 429 else 60, min(900, 60 * 2 ** min(failures - 1, 4)))
class _UploadQueue:
"""One writer per repository; pending bytes and cooldown survive restarts.
Pending is local until a successful Hub commit. It cannot survive deletion
of an ephemeral container; /cache/status makes this distinction visible.
"""
def __init__(self, repo: str, directory: Path, *, interval: float = 60,
batch_size: int = 50, max_bytes: int = 32 * 1024 * 1024):
self.repo = repo
self.directory = directory
self.interval = max(1, interval)
self.batch_size = max(1, min(100, batch_size))
self.max_bytes = max_bytes
self._condition = threading.Condition(threading.RLock())
self._flush_lock = threading.Lock()
self._thread = None
self._stopping = False
self._pending = {}
self._last_error = None
self._last_success = None
self._failures = 0
self._next_attempt = time.time() + self.interval
for folder in ("pending", "blobs", "confirmed"):
(directory / folder).mkdir(parents=True, exist_ok=True)
try:
state = json.loads((directory / "state.json").read_text(encoding="utf-8"))
self._next_attempt = max(self._next_attempt, float(state["next_attempt"]))
except (OSError, ValueError, KeyError, TypeError):
pass
for manifest in (directory / "pending").glob("*.json"):
try:
entry = json.loads(manifest.read_text(encoding="utf-8"))
name, digest = entry["name"], entry["digest"]
identity = self._identity(name)
if (manifest.stem != identity or not re.fullmatch(r"[a-f0-9]{64}", digest)
or not re.fullmatch(r"audio/[a-z0-9_]+/[a-f0-9]{2}/[a-f0-9]+\.(wav|mp3)", name)):
raise ValueError("invalid outbox manifest")
blob = self._blob(name, digest)
if not blob.is_file():
raise ValueError("missing outbox audio")
self._pending[name] = digest
except (OSError, ValueError, KeyError, TypeError):
LOGGER.warning("Could not restore an audio backup outbox entry")
@staticmethod
def _identity(name):
return hashlib.sha256(name.encode("utf-8")).hexdigest()
def _blob(self, name, digest):
identity = hashlib.sha256((name + "\0" + digest).encode()).hexdigest()
return self.directory / "blobs" / f"{identity}.bin"
def _manifest(self, name):
return self.directory / "pending" / f"{self._identity(name)}.json"
def _receipt(self, name):
return self.directory / "confirmed" / f"{self._identity(name)}.json"
def enqueue(self, name: str, data: bytes) -> bool:
digest = hashlib.sha256(data).hexdigest()
with self._condition:
if self._pending.get(name) == digest:
return True
try:
if json.loads(self._receipt(name).read_text(encoding="utf-8"))["digest"] == digest:
return True
except (OSError, ValueError, KeyError, TypeError):
pass
old = self._pending.get(name)
atomic_write(self._blob(name, digest), data)
atomic_write(self._manifest(name), json.dumps({"name": name, "digest": digest}).encode())
self._pending[name] = digest
if old and old != digest:
self._blob(name, old).unlink(missing_ok=True)
self._condition.notify_all()
return True # Accepted into the durable local outbox, not uploaded yet.
def _finish(self, name, digest):
with self._condition:
atomic_write(self._receipt(name), json.dumps({"digest": digest}).encode())
if self._pending.get(name) == digest:
self._manifest(name).unlink(missing_ok=True)
self._pending.pop(name)
self._blob(name, digest).unlink(missing_ok=True)
def confirm_download(self, name, data):
# A remote hit is already backed up. Do not enqueue it again later.
digest = hashlib.sha256(data).hexdigest()
self._finish(name, digest)
def flush_once(self) -> int:
"""Try one bounded batch, without sleeping or bypassing cooldown."""
if not self._flush_lock.acquire(blocking=False):
return 0
try:
with self._condition:
if not self._pending or time.time() < self._next_attempt:
return 0
batch = []
total = 0
for name, digest in self._pending.items():
blob = self._blob(name, digest)
size = blob.stat().st_size
if batch and (len(batch) >= self.batch_size or total + size > self.max_bytes):
break
data = blob.read_bytes()
if hashlib.sha256(data).hexdigest() != digest:
raise ValueError("Invalid pending audio checksum")
batch.append((name, digest, data))
total += size
from huggingface_hub import HfApi, CommitOperationAdd
api = HfApi(token=_token())
infos = {item.path: item for item in api.get_paths_info(
self.repo, [name for name, _, _ in batch], repo_type="dataset",
)}
operations = []
for name, digest, data in batch:
info = infos.get(name)
lfs = getattr(info, "lfs", None)
if lfs:
matches = getattr(lfs, "sha256", None) == digest
else:
blob_digest = hashlib.sha1(f"blob {len(data)}\0".encode() + data).hexdigest()
matches = info is not None and getattr(info, "blob_id", None) == blob_digest
if not matches:
operations.append(CommitOperationAdd(path_in_repo=name, path_or_fileobj=data))
if operations:
api.create_commit(repo_id=self.repo, repo_type="dataset", operations=operations,
commit_message=f"Back up {len(operations)} queued audio cache files", num_threads=2)
for name, digest, _ in batch:
self._finish(name, digest)
with self._condition:
self._last_success = time.time()
self._last_error = None
self._failures = 0
self._next_attempt = time.time() + self.interval
self._save_state()
LOGGER.info("Audio backup: confirmed %s files (%s uploaded); %s pending",
len(batch), len(operations), len(self._pending))
return len(batch)
except Exception as exc:
with self._condition:
self._failures += 1
delay = _retry_delay(exc, self._failures)
self._next_attempt = time.time() + delay
self._last_error = _failure_details(exc)
self._save_state()
LOGGER.warning("Audio backup deferred (%s); retry in %.0f seconds; %s files remain in the local outbox",
self._last_error, delay, len(self._pending))
return 0
finally:
self._flush_lock.release()
def _save_state(self):
try:
atomic_write(self.directory / "state.json", json.dumps({"next_attempt": self._next_attempt}).encode())
except OSError:
LOGGER.warning("Could not persist audio backup retry time")
def start(self):
with self._condition:
if self._thread is None or not self._thread.is_alive():
self._stopping = False
self._thread = threading.Thread(target=self._run, name="audio-cache-backup", daemon=True)
self._thread.start()
def _run(self):
while True:
with self._condition:
if self._stopping:
return
delay = self._next_attempt - time.time()
if not self._pending or delay > 0:
self._condition.wait(timeout=min(60, max(0.1, delay)) if self._pending else 60)
continue
self.flush_once()
def stop(self, timeout=5):
with self._condition:
self._stopping = True
self._condition.notify_all()
thread = self._thread
if thread is not None:
thread.join(timeout=timeout)
def status(self):
with self._condition:
return {"enabled": True, "pending_files": len(self._pending),
"retry_in_seconds": math.ceil(max(0, self._next_attempt - time.time())) if self._pending else 0,
"last_error": self._last_error, "last_success_unix": self._last_success}
_QUEUES = {}
_QUEUES_LOCK = threading.Lock()
def _upload_queue():
repo = os.getenv("HF_AUDIO_CACHE_REPO")
if not repo:
return None
root = Path(os.getenv("HF_AUDIO_CACHE_QUEUE_DIR", str(ROOT / "models/audio_cache_uploads")))
directory = root / hashlib.sha256(repo.encode()).hexdigest()[:16]
key = (repo, str(directory.resolve()))
with _QUEUES_LOCK:
if key not in _QUEUES:
_QUEUES[key] = _UploadQueue(repo, directory,
interval=float(os.getenv("HF_AUDIO_CACHE_FLUSH_INTERVAL", "60")),
batch_size=int(os.getenv("HF_AUDIO_CACHE_BATCH_SIZE", "50")))
queue = _QUEUES[key]
queue.start()
return queue
def start_audio_backup_worker():
_upload_queue()
def stop_audio_backup_worker():
with _QUEUES_LOCK:
queues = list(_QUEUES.values())
for queue in queues:
queue.stop()
def audio_cache_status():
queue = _upload_queue()
return queue.status() if queue else {"enabled": False, "pending_files": 0}
def backup_audio(path: Path, namespace: str, data: bytes) -> bool:
"""Enqueue immutable bytes locally; True means queued, not Hub-confirmed."""
try:
queue = _upload_queue()
return queue.enqueue(remote_name(namespace, path), data) if queue else False
except (OSError, ValueError):
LOGGER.warning("Could not queue %s audio backup; only the source local copy is available", namespace)
return False
def local_cache_files():
"""Yield only generated WAV/MP3 files for migration during deployment."""
zero_root = Path(os.getenv("ZEROTTS_DATA_DIR", str(ROOT / "models/zerotts")))
roots = {
"edge": Path(os.getenv("EDGE_AUDIO_CACHE_DIR", str(ROOT / "models/cache"))),
"zerotts": Path(os.getenv("ZEROTTS_AUDIO_CACHE_DIR", str(zero_root / "audio_cache"))),
"gwen": Path(os.getenv("GWEN_AUDIO_CACHE_DIR", str(ROOT / "models/gwen/audio_cache"))),
"vieneu": Path(os.getenv("VIENEU_AUDIO_CACHE_DIR", str(ROOT / "models/vieneu/audio_cache"))),
"vtts": Path(os.getenv("VTTS_AUDIO_CACHE_DIR", str(
Path(os.getenv("VTTS_MODEL_DIR", str(ROOT / "models/vtts"))) / "audio_cache"
))),
}
for provider, directory in roots.items():
for path in sorted(directory.rglob("*")):
if not path.is_file() or path.is_symlink() or path.suffix not in {".wav", ".mp3"}:
continue
relative = path.relative_to(directory)
if len(relative.parts) == 1:
namespace = provider
elif len(relative.parts) == 2 and relative.parts[0] in {"encoded", "tempo", "voice_color", "loudness"}:
namespace = "mp3" if relative.parts[0] == "encoded" else relative.parts[0]
else:
continue
yield remote_name(namespace, path), path