AnishDev's picture
redis fix
8bf96b3
Raw History Blame Contribute Delete
12.1 kB
"""
Hybrid storage layer.
Candidates (100k+ rows, 50-150 MB) β†’ process memory only. Never Redis.
Reason: remote stores (Upstash free = 10 MB max request size) cannot hold this.
The raw candidate list is only needed during run_matching(); after that it can be GC'd.
Everything else (JD metadata, match status, top-N results, explanations) is tiny (<1 MB
total per session) β†’ Redis if reachable, else in-process memory fallback.
This means the system works with zero external dependencies for local dev.
"""
import json
import time
import logging
from typing import Optional
logger = logging.getLogger(__name__)
# TTL constants (seconds)
TTL_JD = 7 * 24 * 3600
TTL_BATCH = 7 * 24 * 3600
TTL_RESULTS = 24 * 3600
TTL_EXPLANATION = 48 * 3600
# ── In-process memory store ────────────────────────────────────────────────────
class _MemStore:
"""Minimal in-process store mimicking the Redis commands we use."""
def __init__(self):
self._strings: dict[str, str] = {}
self._hashes: dict[str, dict[str, str]] = {}
self._zsets: dict[str, dict[str, float]] = {}
self._expiry: dict[str, float] = {}
def _live(self, key: str) -> bool:
exp = self._expiry.get(key)
if exp is not None and time.time() > exp:
for store in (self._strings, self._hashes, self._zsets):
store.pop(key, None)
self._expiry.pop(key, None)
return False
return True
def ping(self) -> bool:
return True
def set(self, key: str, value: str, ex: Optional[int] = None) -> None:
self._strings[key] = value
if ex:
self._expiry[key] = time.time() + ex
def get(self, key: str) -> Optional[str]:
return self._strings.get(key) if self._live(key) else None
def delete(self, key: str) -> None:
for store in (self._strings, self._hashes, self._zsets):
store.pop(key, None)
self._expiry.pop(key, None)
def expire(self, key: str, seconds: int) -> None:
self._expiry[key] = time.time() + seconds
def hset(self, key: str, mapping: dict) -> None:
if key not in self._hashes:
self._hashes[key] = {}
self._hashes[key].update(mapping)
def hgetall(self, key: str) -> dict:
return dict(self._hashes.get(key, {})) if self._live(key) else {}
def keys(self, pattern: str) -> list[str]:
prefix = pattern.replace("*", "")
result = set()
for store in (self._strings, self._hashes, self._zsets):
for k in list(store.keys()):
if k.startswith(prefix) and self._live(k):
result.add(k)
return list(result)
def zadd(self, key: str, mapping: dict) -> None:
if key not in self._zsets:
self._zsets[key] = {}
self._zsets[key].update(mapping)
def zrevrange(self, key: str, start: int, stop: int,
withscores: bool = False) -> list:
if not self._live(key):
return []
items = sorted(self._zsets.get(key, {}).items(),
key=lambda x: x[1], reverse=True)
sliced = items[start:] if stop == -1 else items[start: stop + 1]
return sliced if withscores else [m for m, _ in sliced]
def pipeline(self):
return _MemPipeline(self)
class _MemPipeline:
"""Fake pipeline that buffers calls and executes them synchronously."""
def __init__(self, store: _MemStore):
self._store = store
self._ops: list = []
def __getattr__(self, name):
def recorder(*args, **kwargs):
self._ops.append((name, args, kwargs))
return self
return recorder
def execute(self):
results = []
for name, args, kwargs in self._ops:
fn = getattr(self._store, name)
results.append(fn(*args, **kwargs))
self._ops.clear()
return results
_mem = _MemStore()
# ── Redis client (optional) ────────────────────────────────────────────────────
_redis_client = None
_redis_ok: Optional[bool] = None # None = not yet tested
def _store():
"""Return Redis client if reachable, else in-process _MemStore."""
global _redis_client, _redis_ok
if _redis_ok is False:
return _mem
try:
if _redis_client is None:
from app.config import settings
import redis as redis_lib
_redis_client = redis_lib.from_url(
settings.redis_url,
decode_responses=True,
socket_connect_timeout=3,
socket_timeout=5,
ssl_cert_reqs=None,
)
_redis_client.ping()
_redis_ok = True
return _redis_client
except Exception as e:
if _redis_ok is not False:
logger.warning(f"Redis unavailable ({e}), using in-process memory store")
_redis_ok = False
return _mem
def ping() -> bool:
try:
return _store().ping()
except Exception:
return False
def _now_iso() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat()
# ── Candidates: process memory ONLY ───────────────────────────────────────────
# 100k candidates β‰ˆ 50-150 MB JSON. Remote stores cannot handle this cheaply.
# After matching, only the top-N detail hashes are persisted; this list is freed.
_candidate_mem: dict[str, str] = {} # batch_id -> candidates_json
_candidate_exp: dict[str, float] = {} # batch_id -> expiry unix timestamp
def _cand_live(batch_id: str) -> bool:
exp = _candidate_exp.get(batch_id)
if exp is not None and time.time() > exp:
_candidate_mem.pop(batch_id, None)
_candidate_exp.pop(batch_id, None)
return False
return batch_id in _candidate_mem
# ── JD ────────────────────────────────────────────────────────────────────────
def save_jd(jd_id: str, data: dict) -> None:
r = _store()
key = f"jd:{jd_id}"
r.hset(key, mapping={k: json.dumps(v) for k, v in data.items()})
r.expire(key, TTL_JD)
def get_jd(jd_id: str) -> Optional[dict]:
raw = _store().hgetall(f"jd:{jd_id}")
if not raw:
return None
result = {}
for k, v in raw.items():
try:
result[k] = json.loads(v)
except (json.JSONDecodeError, TypeError):
result[k] = v
return result
def list_jds() -> list[dict]:
r = _store()
jds = []
for key in sorted(r.keys("jd:*"), reverse=True):
jd_id = key.split(":", 1)[1]
data = get_jd(jd_id)
if data:
jds.append(data)
return jds
# ── Candidate batch ───────────────────────────────────────────────────────────
def save_batch(batch_id: str, metadata: dict, candidates_json: str) -> None:
# Metadata (tiny) β†’ Redis / mem store
r = _store()
meta_key = f"batch:{batch_id}"
r.hset(meta_key, mapping={
k: json.dumps(v) if not isinstance(v, str) else v
for k, v in metadata.items()
})
r.expire(meta_key, TTL_BATCH)
# Candidates (large) β†’ process memory only, never remote
_candidate_mem[batch_id] = candidates_json
_candidate_exp[batch_id] = time.time() + TTL_BATCH
logger.info(f"Stored {len(candidates_json) // 1024} KB candidates for batch {batch_id} in process memory")
def get_batch_metadata(batch_id: str) -> Optional[dict]:
raw = _store().hgetall(f"batch:{batch_id}")
if not raw:
return None
result = {}
for k, v in raw.items():
try:
result[k] = json.loads(v)
except (json.JSONDecodeError, TypeError):
result[k] = v
return result
def get_batch_candidates_json(batch_id: str) -> Optional[str]:
if not _cand_live(batch_id):
return None
return _candidate_mem.get(batch_id)
def list_batches() -> list[dict]:
r = _store()
batches = []
for key in sorted(r.keys("batch:*"), reverse=True):
batch_id = key.split(":", 1)[1]
data = get_batch_metadata(batch_id)
if data:
batches.append(data)
return batches
# ── Match status ──────────────────────────────────────────────────────────────
def set_match_status(
match_id: str,
status: str,
progress: Optional[float] = None,
message: Optional[str] = None,
) -> None:
r = _store()
key = f"match_status:{match_id}"
data: dict[str, str] = {"status": status, "match_id": match_id}
if progress is not None:
data["progress"] = str(progress)
if message:
data["message"] = message
if status == "completed":
data["completed_at"] = _now_iso()
r.hset(key, mapping=data)
r.expire(key, TTL_RESULTS)
def get_match_status(match_id: str) -> Optional[dict]:
raw = _store().hgetall(f"match_status:{match_id}")
return raw if raw else None
# ── Match results ─────────────────────────────────────────────────────────────
def save_match_results(jd_id: str, batch_id: str, results: list[dict]) -> None:
r = _store()
pipe = r.pipeline()
zset_key = f"results:{jd_id}:{batch_id}"
pipe.delete(zset_key)
for result in results:
cid = result["candidate_id"]
score = result["final_score"]
pipe.zadd(zset_key, {cid: score})
detail_key = f"detail:{jd_id}:{batch_id}:{cid}"
flat = {k: json.dumps(v) if not isinstance(v, str) else v
for k, v in result.items()}
pipe.hset(detail_key, mapping=flat)
pipe.expire(detail_key, TTL_RESULTS)
pipe.expire(zset_key, TTL_RESULTS)
pipe.execute()
def get_ranked_candidates(
jd_id: str, batch_id: str, top_n: int
) -> list[tuple[str, float]]:
return _store().zrevrange(
f"results:{jd_id}:{batch_id}", 0, top_n - 1, withscores=True
)
def get_candidate_detail(
jd_id: str, batch_id: str, candidate_id: str
) -> Optional[dict]:
raw = _store().hgetall(f"detail:{jd_id}:{batch_id}:{candidate_id}")
if not raw:
return None
result = {}
for k, v in raw.items():
try:
result[k] = json.loads(v)
except (json.JSONDecodeError, TypeError):
result[k] = v
return result
# ── Explanation cache ─────────────────────────────────────────────────────────
def get_explanation(jd_id: str, batch_id: str, candidate_id: str) -> Optional[str]:
return _store().get(f"explanation:{jd_id}:{batch_id}:{candidate_id}")
def save_explanation(jd_id: str, batch_id: str, candidate_id: str, text: str) -> None:
_store().set(
f"explanation:{jd_id}:{batch_id}:{candidate_id}",
text,
ex=TTL_EXPLANATION,
)
# ── Active batch pointer ──────────────────────────────────────────────────────
def set_active_batch(jd_id: str, batch_id: str) -> None:
_store().set(f"active_batch:{jd_id}", batch_id, ex=TTL_RESULTS)
def get_active_batch(jd_id: str) -> Optional[str]:
return _store().get(f"active_batch:{jd_id}")