store: merge concurrent copies on write conflict instead of overwriting
Browse files- app/store.py +42 -8
app/store.py
CHANGED
|
@@ -22,6 +22,10 @@ import httpx
|
|
| 22 |
GITHUB_API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
|
| 23 |
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
# ---------- 标注仓库 ----------
|
| 26 |
|
| 27 |
class LocalAnnotationStore:
|
|
@@ -40,6 +44,9 @@ class LocalAnnotationStore:
|
|
| 40 |
|
| 41 |
def write(self, annotator: str, name: str, data: bytes, sha: Optional[str], message: str) -> str:
|
| 42 |
p = self._path(annotator, name)
|
|
|
|
|
|
|
|
|
|
| 43 |
p.parent.mkdir(parents=True, exist_ok=True)
|
| 44 |
tmp = p.with_suffix(p.suffix + ".tmp")
|
| 45 |
tmp.write_bytes(data)
|
|
@@ -100,13 +107,8 @@ class GitHubAnnotationStore:
|
|
| 100 |
url = self._url(annotator, name)
|
| 101 |
r = self.client.put(url, headers=self.headers, json=body)
|
| 102 |
if r.status_code in (409, 422):
|
| 103 |
-
# sha 过期(别处改过文件
|
| 104 |
-
|
| 105 |
-
if cur:
|
| 106 |
-
body["sha"] = cur
|
| 107 |
-
else:
|
| 108 |
-
body.pop("sha", None)
|
| 109 |
-
r = self.client.put(url, headers=self.headers, json=body)
|
| 110 |
r.raise_for_status()
|
| 111 |
return r.json()["content"]["sha"]
|
| 112 |
|
|
@@ -299,6 +301,28 @@ class GitHubPapers:
|
|
| 299 |
|
| 300 |
# ---------- 写回缓存 ----------
|
| 301 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
class DocCache:
|
| 303 |
"""(annotator, paper) → 标注文档, 内存里改, 后台线程每 flush_seconds 秒把脏文档写回存储。
|
| 304 |
每次改动都立即在内存生效(界面零延迟); 同步状态通过 status() 暴露给前端的同步指示。"""
|
|
@@ -356,7 +380,17 @@ class DocCache:
|
|
| 356 |
n = len(msgs)
|
| 357 |
message = f"{annotator} · {paper} · {n} change{'s' if n != 1 else ''}\n\n" + "\n".join(msgs[:50])
|
| 358 |
try:
|
| 359 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
except Exception as e: # 保持脏, 下一轮重试
|
| 361 |
self.last_error[annotator] = f"{type(e).__name__}: {e}"[:300]
|
| 362 |
continue
|
|
|
|
| 22 |
GITHUB_API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
|
| 23 |
|
| 24 |
|
| 25 |
+
class ConflictError(Exception):
|
| 26 |
+
"""存储里的文件已被别处改过(sha 对不上)。由 DocCache 读回合并后重写, 不直接覆盖。"""
|
| 27 |
+
|
| 28 |
+
|
| 29 |
# ---------- 标注仓库 ----------
|
| 30 |
|
| 31 |
class LocalAnnotationStore:
|
|
|
|
| 44 |
|
| 45 |
def write(self, annotator: str, name: str, data: bytes, sha: Optional[str], message: str) -> str:
|
| 46 |
p = self._path(annotator, name)
|
| 47 |
+
cur = str(p.stat().st_mtime_ns) if p.exists() else None
|
| 48 |
+
if cur != sha:
|
| 49 |
+
raise ConflictError(name)
|
| 50 |
p.parent.mkdir(parents=True, exist_ok=True)
|
| 51 |
tmp = p.with_suffix(p.suffix + ".tmp")
|
| 52 |
tmp.write_bytes(data)
|
|
|
|
| 107 |
url = self._url(annotator, name)
|
| 108 |
r = self.client.put(url, headers=self.headers, json=body)
|
| 109 |
if r.status_code in (409, 422):
|
| 110 |
+
# sha 过期(别处改过文件, 如另一个服务器实例): 不覆盖, 交给 DocCache 合并后重写
|
| 111 |
+
raise ConflictError(name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
r.raise_for_status()
|
| 113 |
return r.json()["content"]["sha"]
|
| 114 |
|
|
|
|
| 301 |
|
| 302 |
# ---------- 写回缓存 ----------
|
| 303 |
|
| 304 |
+
def merge_docs(mine: dict, theirs: dict) -> None:
|
| 305 |
+
"""把存储里的版本(theirs)合并进内存版本(mine), 原地修改 mine。
|
| 306 |
+
记录 / 字段按 updated_ts 取较新的一份; 两边都有的保留较新; 只一边有的保留。events、papers_refs_seen 取并集。"""
|
| 307 |
+
def newer(a, b):
|
| 308 |
+
return a if (a or {}).get("updated_ts", 0) >= (b or {}).get("updated_ts", 0) else b
|
| 309 |
+
|
| 310 |
+
for uid, rec in (theirs.get("records") or {}).items():
|
| 311 |
+
mine.setdefault("records", {})[uid] = newer(mine["records"].get(uid), rec) if uid in mine["records"] else rec
|
| 312 |
+
for uid, flds in (theirs.get("fields") or {}).items():
|
| 313 |
+
mf = mine.setdefault("fields", {}).setdefault(uid, {})
|
| 314 |
+
for fid, f in flds.items():
|
| 315 |
+
mf[fid] = newer(mf.get(fid), f) if fid in mf else f
|
| 316 |
+
seen = {(e.get("ts"), e.get("kind"), e.get("detail")) for e in mine.get("events", [])}
|
| 317 |
+
extra = [e for e in theirs.get("events", []) if (e.get("ts"), e.get("kind"), e.get("detail")) not in seen]
|
| 318 |
+
if extra:
|
| 319 |
+
mine["events"] = sorted(mine.get("events", []) + extra, key=lambda e: e.get("ts", 0))
|
| 320 |
+
refs = mine.setdefault("papers_refs_seen", [])
|
| 321 |
+
refs += [r for r in theirs.get("papers_refs_seen", []) if r not in refs]
|
| 322 |
+
mine["events"].append({"ts": time.time(), "kind": "merged_concurrent_copy",
|
| 323 |
+
"detail": f"{len(extra)} events from a concurrently written copy"})
|
| 324 |
+
|
| 325 |
+
|
| 326 |
class DocCache:
|
| 327 |
"""(annotator, paper) → 标注文档, 内存里改, 后台线程每 flush_seconds 秒把脏文档写回存储。
|
| 328 |
每次改动都立即在内存生效(界面零延迟); 同步状态通过 status() 暴露给前端的同步指示。"""
|
|
|
|
| 380 |
n = len(msgs)
|
| 381 |
message = f"{annotator} · {paper} · {n} change{'s' if n != 1 else ''}\n\n" + "\n".join(msgs[:50])
|
| 382 |
try:
|
| 383 |
+
try:
|
| 384 |
+
sha = self.store.write(annotator, self.path_for(paper), data, self.shas.get(key), message)
|
| 385 |
+
except ConflictError:
|
| 386 |
+
# 别处写过这份文件: 读回、按字段合并进内存版本, 再写一次(不丢任何一边的改动)
|
| 387 |
+
remote, rsha = self.store.read(annotator, self.path_for(paper))
|
| 388 |
+
with self.lock:
|
| 389 |
+
if remote:
|
| 390 |
+
merge_docs(self.docs[key], json.loads(remote))
|
| 391 |
+
self.shas[key] = rsha
|
| 392 |
+
data = json.dumps(self.docs[key], ensure_ascii=False, indent=1).encode("utf-8")
|
| 393 |
+
sha = self.store.write(annotator, self.path_for(paper), data, rsha, message + "\n(merged)")
|
| 394 |
except Exception as e: # 保持脏, 下一轮重试
|
| 395 |
self.last_error[annotator] = f"{type(e).__name__}: {e}"[:300]
|
| 396 |
continue
|