roz322 commited on
Commit
b0111e0
·
verified ·
1 Parent(s): 3f2c936

store: merge concurrent copies on write conflict instead of overwriting

Browse files
Files changed (1) hide show
  1. app/store.py +42 -8
app/store.py CHANGED
@@ -22,6 +22,10 @@ import httpx
22
  GITHUB_API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
23
 
24
 
 
 
 
 
25
  # ---------- 标注仓库 ----------
26
 
27
  class LocalAnnotationStore:
@@ -40,6 +44,9 @@ class LocalAnnotationStore:
40
 
41
  def write(self, annotator: str, name: str, data: bytes, sha: Optional[str], message: str) -> str:
42
  p = self._path(annotator, name)
 
 
 
43
  p.parent.mkdir(parents=True, exist_ok=True)
44
  tmp = p.with_suffix(p.suffix + ".tmp")
45
  tmp.write_bytes(data)
@@ -100,13 +107,8 @@ class GitHubAnnotationStore:
100
  url = self._url(annotator, name)
101
  r = self.client.put(url, headers=self.headers, json=body)
102
  if r.status_code in (409, 422):
103
- # sha 过期(别处改过文件): 取最新 sha 重试一次; 服务器是唯一写入方, 以内存版本为准
104
- _, cur = self.read(annotator, name)
105
- if cur:
106
- body["sha"] = cur
107
- else:
108
- body.pop("sha", None)
109
- r = self.client.put(url, headers=self.headers, json=body)
110
  r.raise_for_status()
111
  return r.json()["content"]["sha"]
112
 
@@ -299,6 +301,28 @@ class GitHubPapers:
299
 
300
  # ---------- 写回缓存 ----------
301
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
302
  class DocCache:
303
  """(annotator, paper) → 标注文档, 内存里改, 后台线程每 flush_seconds 秒把脏文档写回存储。
304
  每次改动都立即在内存生效(界面零延迟); 同步状态通过 status() 暴露给前端的同步指示。"""
@@ -356,7 +380,17 @@ class DocCache:
356
  n = len(msgs)
357
  message = f"{annotator} · {paper} · {n} change{'s' if n != 1 else ''}\n\n" + "\n".join(msgs[:50])
358
  try:
359
- sha = self.store.write(annotator, self.path_for(paper), data, self.shas.get(key), message)
 
 
 
 
 
 
 
 
 
 
360
  except Exception as e: # 保持脏, 下一轮重试
361
  self.last_error[annotator] = f"{type(e).__name__}: {e}"[:300]
362
  continue
 
22
  GITHUB_API = os.environ.get("GITHUB_API_URL", "https://api.github.com")
23
 
24
 
25
+ class ConflictError(Exception):
26
+ """存储里的文件已被别处改过(sha 对不上)。由 DocCache 读回合并后重写, 不直接覆盖。"""
27
+
28
+
29
  # ---------- 标注仓库 ----------
30
 
31
  class LocalAnnotationStore:
 
44
 
45
  def write(self, annotator: str, name: str, data: bytes, sha: Optional[str], message: str) -> str:
46
  p = self._path(annotator, name)
47
+ cur = str(p.stat().st_mtime_ns) if p.exists() else None
48
+ if cur != sha:
49
+ raise ConflictError(name)
50
  p.parent.mkdir(parents=True, exist_ok=True)
51
  tmp = p.with_suffix(p.suffix + ".tmp")
52
  tmp.write_bytes(data)
 
107
  url = self._url(annotator, name)
108
  r = self.client.put(url, headers=self.headers, json=body)
109
  if r.status_code in (409, 422):
110
+ # sha 过期(别处改过文件, 如另一个服务器实例): 不覆盖, 交给 DocCache 合并后重写
111
+ raise ConflictError(name)
 
 
 
 
 
112
  r.raise_for_status()
113
  return r.json()["content"]["sha"]
114
 
 
301
 
302
  # ---------- 写回缓存 ----------
303
 
304
+ def merge_docs(mine: dict, theirs: dict) -> None:
305
+ """把存储里的版本(theirs)合并进内存版本(mine), 原地修改 mine。
306
+ 记录 / 字段按 updated_ts 取较新的一份; 两边都有的保留较新; 只一边有的保留。events、papers_refs_seen 取并集。"""
307
+ def newer(a, b):
308
+ return a if (a or {}).get("updated_ts", 0) >= (b or {}).get("updated_ts", 0) else b
309
+
310
+ for uid, rec in (theirs.get("records") or {}).items():
311
+ mine.setdefault("records", {})[uid] = newer(mine["records"].get(uid), rec) if uid in mine["records"] else rec
312
+ for uid, flds in (theirs.get("fields") or {}).items():
313
+ mf = mine.setdefault("fields", {}).setdefault(uid, {})
314
+ for fid, f in flds.items():
315
+ mf[fid] = newer(mf.get(fid), f) if fid in mf else f
316
+ seen = {(e.get("ts"), e.get("kind"), e.get("detail")) for e in mine.get("events", [])}
317
+ extra = [e for e in theirs.get("events", []) if (e.get("ts"), e.get("kind"), e.get("detail")) not in seen]
318
+ if extra:
319
+ mine["events"] = sorted(mine.get("events", []) + extra, key=lambda e: e.get("ts", 0))
320
+ refs = mine.setdefault("papers_refs_seen", [])
321
+ refs += [r for r in theirs.get("papers_refs_seen", []) if r not in refs]
322
+ mine["events"].append({"ts": time.time(), "kind": "merged_concurrent_copy",
323
+ "detail": f"{len(extra)} events from a concurrently written copy"})
324
+
325
+
326
  class DocCache:
327
  """(annotator, paper) → 标注文档, 内存里改, 后台线程每 flush_seconds 秒把脏文档写回存储。
328
  每次改动都立即在内存生效(界面零延迟); 同步状态通过 status() 暴露给前端的同步指示。"""
 
380
  n = len(msgs)
381
  message = f"{annotator} · {paper} · {n} change{'s' if n != 1 else ''}\n\n" + "\n".join(msgs[:50])
382
  try:
383
+ try:
384
+ sha = self.store.write(annotator, self.path_for(paper), data, self.shas.get(key), message)
385
+ except ConflictError:
386
+ # 别处写过这份文件: 读回、按字段合并进内存版本, 再写一次(不丢任何一边的改动)
387
+ remote, rsha = self.store.read(annotator, self.path_for(paper))
388
+ with self.lock:
389
+ if remote:
390
+ merge_docs(self.docs[key], json.loads(remote))
391
+ self.shas[key] = rsha
392
+ data = json.dumps(self.docs[key], ensure_ascii=False, indent=1).encode("utf-8")
393
+ sha = self.store.write(annotator, self.path_for(paper), data, rsha, message + "\n(merged)")
394
  except Exception as e: # 保持脏, 下一轮重试
395
  self.last_error[annotator] = f"{type(e).__name__}: {e}"[:300]
396
  continue