Spaces:
Sleeping
Sleeping
Download app/services/storage.py from AnishDev/coderound-task-backend: direct link, hf CLI and curl.
- Browser
- Download file 12.1 kB
-
https://huggingface.co/spaces/AnishDev/coderound-task-backend/resolve/main/app/services/storage.py
- Command line
-
hf download hf://spaces/AnishDev/coderound-task-backend/app/services/storage.py
-
curl -L -o storage.py https://huggingface.co/spaces/AnishDev/coderound-task-backend/resolve/main/app/services/storage.py
12.1 kB
| """ | |
| Hybrid storage layer. | |
| Candidates (100k+ rows, 50-150 MB) β process memory only. Never Redis. | |
| Reason: remote stores (Upstash free = 10 MB max request size) cannot hold this. | |
| The raw candidate list is only needed during run_matching(); after that it can be GC'd. | |
| Everything else (JD metadata, match status, top-N results, explanations) is tiny (<1 MB | |
| total per session) β Redis if reachable, else in-process memory fallback. | |
| This means the system works with zero external dependencies for local dev. | |
| """ | |
| import json | |
| import time | |
| import logging | |
| from typing import Optional | |
| logger = logging.getLogger(__name__) | |
| # TTL constants (seconds) | |
| TTL_JD = 7 * 24 * 3600 | |
| TTL_BATCH = 7 * 24 * 3600 | |
| TTL_RESULTS = 24 * 3600 | |
| TTL_EXPLANATION = 48 * 3600 | |
| # ββ In-process memory store ββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class _MemStore: | |
| """Minimal in-process store mimicking the Redis commands we use.""" | |
| def __init__(self): | |
| self._strings: dict[str, str] = {} | |
| self._hashes: dict[str, dict[str, str]] = {} | |
| self._zsets: dict[str, dict[str, float]] = {} | |
| self._expiry: dict[str, float] = {} | |
| def _live(self, key: str) -> bool: | |
| exp = self._expiry.get(key) | |
| if exp is not None and time.time() > exp: | |
| for store in (self._strings, self._hashes, self._zsets): | |
| store.pop(key, None) | |
| self._expiry.pop(key, None) | |
| return False | |
| return True | |
| def ping(self) -> bool: | |
| return True | |
| def set(self, key: str, value: str, ex: Optional[int] = None) -> None: | |
| self._strings[key] = value | |
| if ex: | |
| self._expiry[key] = time.time() + ex | |
| def get(self, key: str) -> Optional[str]: | |
| return self._strings.get(key) if self._live(key) else None | |
| def delete(self, key: str) -> None: | |
| for store in (self._strings, self._hashes, self._zsets): | |
| store.pop(key, None) | |
| self._expiry.pop(key, None) | |
| def expire(self, key: str, seconds: int) -> None: | |
| self._expiry[key] = time.time() + seconds | |
| def hset(self, key: str, mapping: dict) -> None: | |
| if key not in self._hashes: | |
| self._hashes[key] = {} | |
| self._hashes[key].update(mapping) | |
| def hgetall(self, key: str) -> dict: | |
| return dict(self._hashes.get(key, {})) if self._live(key) else {} | |
| def keys(self, pattern: str) -> list[str]: | |
| prefix = pattern.replace("*", "") | |
| result = set() | |
| for store in (self._strings, self._hashes, self._zsets): | |
| for k in list(store.keys()): | |
| if k.startswith(prefix) and self._live(k): | |
| result.add(k) | |
| return list(result) | |
| def zadd(self, key: str, mapping: dict) -> None: | |
| if key not in self._zsets: | |
| self._zsets[key] = {} | |
| self._zsets[key].update(mapping) | |
| def zrevrange(self, key: str, start: int, stop: int, | |
| withscores: bool = False) -> list: | |
| if not self._live(key): | |
| return [] | |
| items = sorted(self._zsets.get(key, {}).items(), | |
| key=lambda x: x[1], reverse=True) | |
| sliced = items[start:] if stop == -1 else items[start: stop + 1] | |
| return sliced if withscores else [m for m, _ in sliced] | |
| def pipeline(self): | |
| return _MemPipeline(self) | |
| class _MemPipeline: | |
| """Fake pipeline that buffers calls and executes them synchronously.""" | |
| def __init__(self, store: _MemStore): | |
| self._store = store | |
| self._ops: list = [] | |
| def __getattr__(self, name): | |
| def recorder(*args, **kwargs): | |
| self._ops.append((name, args, kwargs)) | |
| return self | |
| return recorder | |
| def execute(self): | |
| results = [] | |
| for name, args, kwargs in self._ops: | |
| fn = getattr(self._store, name) | |
| results.append(fn(*args, **kwargs)) | |
| self._ops.clear() | |
| return results | |
| _mem = _MemStore() | |
| # ββ Redis client (optional) ββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _redis_client = None | |
| _redis_ok: Optional[bool] = None # None = not yet tested | |
| def _store(): | |
| """Return Redis client if reachable, else in-process _MemStore.""" | |
| global _redis_client, _redis_ok | |
| if _redis_ok is False: | |
| return _mem | |
| try: | |
| if _redis_client is None: | |
| from app.config import settings | |
| import redis as redis_lib | |
| _redis_client = redis_lib.from_url( | |
| settings.redis_url, | |
| decode_responses=True, | |
| socket_connect_timeout=3, | |
| socket_timeout=5, | |
| ssl_cert_reqs=None, | |
| ) | |
| _redis_client.ping() | |
| _redis_ok = True | |
| return _redis_client | |
| except Exception as e: | |
| if _redis_ok is not False: | |
| logger.warning(f"Redis unavailable ({e}), using in-process memory store") | |
| _redis_ok = False | |
| return _mem | |
| def ping() -> bool: | |
| try: | |
| return _store().ping() | |
| except Exception: | |
| return False | |
| def _now_iso() -> str: | |
| from datetime import datetime, timezone | |
| return datetime.now(timezone.utc).isoformat() | |
| # ββ Candidates: process memory ONLY βββββββββββββββββββββββββββββββββββββββββββ | |
| # 100k candidates β 50-150 MB JSON. Remote stores cannot handle this cheaply. | |
| # After matching, only the top-N detail hashes are persisted; this list is freed. | |
| _candidate_mem: dict[str, str] = {} # batch_id -> candidates_json | |
| _candidate_exp: dict[str, float] = {} # batch_id -> expiry unix timestamp | |
| def _cand_live(batch_id: str) -> bool: | |
| exp = _candidate_exp.get(batch_id) | |
| if exp is not None and time.time() > exp: | |
| _candidate_mem.pop(batch_id, None) | |
| _candidate_exp.pop(batch_id, None) | |
| return False | |
| return batch_id in _candidate_mem | |
| # ββ JD ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def save_jd(jd_id: str, data: dict) -> None: | |
| r = _store() | |
| key = f"jd:{jd_id}" | |
| r.hset(key, mapping={k: json.dumps(v) for k, v in data.items()}) | |
| r.expire(key, TTL_JD) | |
| def get_jd(jd_id: str) -> Optional[dict]: | |
| raw = _store().hgetall(f"jd:{jd_id}") | |
| if not raw: | |
| return None | |
| result = {} | |
| for k, v in raw.items(): | |
| try: | |
| result[k] = json.loads(v) | |
| except (json.JSONDecodeError, TypeError): | |
| result[k] = v | |
| return result | |
| def list_jds() -> list[dict]: | |
| r = _store() | |
| jds = [] | |
| for key in sorted(r.keys("jd:*"), reverse=True): | |
| jd_id = key.split(":", 1)[1] | |
| data = get_jd(jd_id) | |
| if data: | |
| jds.append(data) | |
| return jds | |
| # ββ Candidate batch βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def save_batch(batch_id: str, metadata: dict, candidates_json: str) -> None: | |
| # Metadata (tiny) β Redis / mem store | |
| r = _store() | |
| meta_key = f"batch:{batch_id}" | |
| r.hset(meta_key, mapping={ | |
| k: json.dumps(v) if not isinstance(v, str) else v | |
| for k, v in metadata.items() | |
| }) | |
| r.expire(meta_key, TTL_BATCH) | |
| # Candidates (large) β process memory only, never remote | |
| _candidate_mem[batch_id] = candidates_json | |
| _candidate_exp[batch_id] = time.time() + TTL_BATCH | |
| logger.info(f"Stored {len(candidates_json) // 1024} KB candidates for batch {batch_id} in process memory") | |
| def get_batch_metadata(batch_id: str) -> Optional[dict]: | |
| raw = _store().hgetall(f"batch:{batch_id}") | |
| if not raw: | |
| return None | |
| result = {} | |
| for k, v in raw.items(): | |
| try: | |
| result[k] = json.loads(v) | |
| except (json.JSONDecodeError, TypeError): | |
| result[k] = v | |
| return result | |
| def get_batch_candidates_json(batch_id: str) -> Optional[str]: | |
| if not _cand_live(batch_id): | |
| return None | |
| return _candidate_mem.get(batch_id) | |
| def list_batches() -> list[dict]: | |
| r = _store() | |
| batches = [] | |
| for key in sorted(r.keys("batch:*"), reverse=True): | |
| batch_id = key.split(":", 1)[1] | |
| data = get_batch_metadata(batch_id) | |
| if data: | |
| batches.append(data) | |
| return batches | |
| # ββ Match status ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def set_match_status( | |
| match_id: str, | |
| status: str, | |
| progress: Optional[float] = None, | |
| message: Optional[str] = None, | |
| ) -> None: | |
| r = _store() | |
| key = f"match_status:{match_id}" | |
| data: dict[str, str] = {"status": status, "match_id": match_id} | |
| if progress is not None: | |
| data["progress"] = str(progress) | |
| if message: | |
| data["message"] = message | |
| if status == "completed": | |
| data["completed_at"] = _now_iso() | |
| r.hset(key, mapping=data) | |
| r.expire(key, TTL_RESULTS) | |
| def get_match_status(match_id: str) -> Optional[dict]: | |
| raw = _store().hgetall(f"match_status:{match_id}") | |
| return raw if raw else None | |
| # ββ Match results βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def save_match_results(jd_id: str, batch_id: str, results: list[dict]) -> None: | |
| r = _store() | |
| pipe = r.pipeline() | |
| zset_key = f"results:{jd_id}:{batch_id}" | |
| pipe.delete(zset_key) | |
| for result in results: | |
| cid = result["candidate_id"] | |
| score = result["final_score"] | |
| pipe.zadd(zset_key, {cid: score}) | |
| detail_key = f"detail:{jd_id}:{batch_id}:{cid}" | |
| flat = {k: json.dumps(v) if not isinstance(v, str) else v | |
| for k, v in result.items()} | |
| pipe.hset(detail_key, mapping=flat) | |
| pipe.expire(detail_key, TTL_RESULTS) | |
| pipe.expire(zset_key, TTL_RESULTS) | |
| pipe.execute() | |
| def get_ranked_candidates( | |
| jd_id: str, batch_id: str, top_n: int | |
| ) -> list[tuple[str, float]]: | |
| return _store().zrevrange( | |
| f"results:{jd_id}:{batch_id}", 0, top_n - 1, withscores=True | |
| ) | |
| def get_candidate_detail( | |
| jd_id: str, batch_id: str, candidate_id: str | |
| ) -> Optional[dict]: | |
| raw = _store().hgetall(f"detail:{jd_id}:{batch_id}:{candidate_id}") | |
| if not raw: | |
| return None | |
| result = {} | |
| for k, v in raw.items(): | |
| try: | |
| result[k] = json.loads(v) | |
| except (json.JSONDecodeError, TypeError): | |
| result[k] = v | |
| return result | |
| # ββ Explanation cache βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def get_explanation(jd_id: str, batch_id: str, candidate_id: str) -> Optional[str]: | |
| return _store().get(f"explanation:{jd_id}:{batch_id}:{candidate_id}") | |
| def save_explanation(jd_id: str, batch_id: str, candidate_id: str, text: str) -> None: | |
| _store().set( | |
| f"explanation:{jd_id}:{batch_id}:{candidate_id}", | |
| text, | |
| ex=TTL_EXPLANATION, | |
| ) | |
| # ββ Active batch pointer ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def set_active_batch(jd_id: str, batch_id: str) -> None: | |
| _store().set(f"active_batch:{jd_id}", batch_id, ex=TTL_RESULTS) | |
| def get_active_batch(jd_id: str) -> Optional[str]: | |
| return _store().get(f"active_batch:{jd_id}") | |