Falln87 commited on
Commit
705d9d4
·
verified ·
1 Parent(s): ca30e35

eval_locomo: clean rewrite (working choice log-likelihood scorer, consistent templates, single model loader)

Browse files
Files changed (1) hide show
  1. clerk/eval_locomo.py +90 -72
clerk/eval_locomo.py CHANGED
@@ -8,11 +8,11 @@ Protocol:
8
  of the 10 choices given (memory context + question), pick argmax.
9
  3. Report accuracy overall and per question type
10
  (single_hop / multi_hop / temporal_reasoning / open_domain /
11
- adversarial), plus tokens-per-question.
12
 
13
  Systems: clerk (adapter), prompted_ledger (same base, untrained), full
14
- (entire conversation in context), rag (top-k retrieved session summaries,
15
- using the haystack_session_summaries that ship with each item).
16
 
17
  A ledger cache keyed by the conversation's session ids means each
18
  conversation is consolidated exactly once.
@@ -39,23 +39,27 @@ from clerk.common import parse_ledger, parse_ops, apply_ops
39
  from clerk.prompts import CONSOLIDATE_SYSTEM, CONSOLIDATE_USER_TMPL
40
 
41
  BASE_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
42
- MC10_DATA = "Percena/locomo-mc10"
43
- MC10_FILE = "data/locomo_mc10.json"
44
 
45
- LEDGER_ANSWER_TMPL = (
46
- "MEMORY LEDGER:\n{ledger}\n\nQuestion: {question}\n\nAnswer:"
47
- )
48
- FULL_TMPL = (
49
- "CONVERSATION SO FAR:\n{context}\n\nQuestion: {question}\n\nAnswer:"
50
- )
51
- RAG_TMPL = (
52
- "RELEVANT SESSION SUMMARIES:\n{context}\n\nQuestion: {question}\n\nAnswer:"
53
- )
54
 
55
 
56
- def load_items(path: str) -> List[Dict[str, Any]]:
57
- with open(path) as f:
58
- return json.load(f)
 
 
 
 
 
 
59
 
60
 
61
  def render_sessions(sessions: List[List[Dict[str, Any]]]) -> List[str]:
@@ -91,18 +95,33 @@ def generate(model, tok, system: str, user: str, max_new_tokens: int) -> str:
91
 
92
 
93
  @torch.no_grad()
94
- def choice_logprobs(model, tok, prompt_text: str,
95
  choices: List[str]) -> List[float]:
96
- """Total log-prob of each choice continuation after `prompt_text`."""
97
- scores = []
 
 
 
 
 
 
98
  for choice in choices:
99
- ids_prompt = tok(prompt_text := prompt_text_holder, add_special_tokens=False)
100
- raise RuntimeError # replaced below
 
 
 
 
 
 
 
 
 
101
  return scores
102
 
103
 
104
- def build_ledgers(model, tok, items: List[Dict[str, Any]], budget: int,
105
- use_adapter_for_write: bool) -> Dict[Tuple, str]:
106
  """Consolidate each unique conversation once; cache by session-id tuple."""
107
  cache: Dict[Tuple, str] = {}
108
  for item in items:
@@ -119,22 +138,33 @@ def build_ledgers(model, tok, items: List[Dict[str, Any]], budget: int,
119
  if err != "":
120
  ops = []
121
  ledger, _ = apply_ops(parse_ledger(ledger_json, budget), ops, idx)
122
- rows = [{"i": s.id, "s": s.subject, "p": s.predicate,
123
- "o": s.object, "t": s.t_last, "v": 1 if s.valid else 0}
124
- for s in ledger if s.occupied]
125
- ledger_json = json.dumps(rows, separators=(",", ":"),
126
- ensure_ascii=False)
127
- print(f" session {idx + 1}/{len(rendered)} ops={len(ops)} "
128
- f"err={err}", flush=True)
129
  cache[key] = ledger_json
130
  return cache
131
 
132
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  def main() -> None:
134
  ap = argparse.ArgumentParser(description="CLERK LoCoMo-MC10 evaluation")
135
- ap.add_argument("--data-dir", type=str, default="data",
136
- help="directory containing locomo_mc10.json "
137
- "(download from Percena/locomo-mc10)")
138
  ap.add_argument("--adapter", type=str, default=None)
139
  ap.add_argument("--base-model", type=str, default=BASE_MODEL)
140
  ap.add_argument("--systems", type=str,
@@ -144,58 +174,46 @@ def main() -> None:
144
  ap.add_argument("--out", type=str, required=True)
145
  args = ap.parse_args()
146
 
147
- from datasets import load_dataset # noqa: F401 (ensures hub auth path)
148
- import huggingface_hub
149
-
150
- snapshot = huggingface_hub.snapshot_download(
151
- MC10_DATA_REPO := "Percena/locomo-mc10",
152
- allow_patterns=["data/locomo_mc10.json"])
153
- items = load_items(f"{snapshot}/data/locomo_mc10.json")
 
 
154
  if args.max_items:
155
  items = items[: args.max_items]
156
 
157
  results = []
158
  for system in [s for s in args.systems.split(",") if s]:
159
- model, tok = load_model_local(
160
  args.adapter if system == "clerk" else None, args.base_model)
161
  t0 = time.time()
 
162
 
163
- ledgers = build_ledgers(model, tok, items, args.budget, system == "clerk")
164
-
165
- correct_by_type: Dict[str, List[int]] = defaultdict(lambda: [0, 0])
166
- n_ok = 0
167
- for i, item in enumerate(items):
168
- key = tuple(item["haystack_session_ids"])
169
- if system in ("clerk", "prompted_ledger"):
170
- context = LEDGER_ANSWER_TMPL.format(
171
- ledger=ledgers[key], question=item["question"])
172
- elif system == "full":
173
- context = FULL_ANSWER_TMPL.format(
174
- context="\n\n".join(render_sessions(item["haystack_sessions"])),
175
- question=item["question"])
176
- else: # rag over session summaries
177
- context = RAG_ANSWER_TMPL.format(
178
- context="\n".join(item["haystack_session_summaries"][: RAG_K]),
179
- question=item["question"])
180
  lps = choice_logprobs(model, tok, context, item["choices"])
181
- pred = int(np.argmax(lps))
182
- ok = int(pred == item["correct_choice_index"])
183
- qtype = item["question_type"]
184
- correct_by_type[qtype][0] += ok
185
- correct_by_type[qtype][1] += 1
186
- if (n_ok := n_ok + 1) % 200 == 0:
187
- print(f"[{system}] {n_ok}/{len(items)} items", flush=True)
188
 
189
  summary = {
190
  "system": system,
191
  "n_items": len(items),
192
  "accuracy_overall": round(
193
- sum(v[0] for v in correct_by_type.values())
194
- / max(1, sum(v[1] for v in correct_by_type.values())), 4),
195
- "accuracy_by_type": {
196
- t: round(v[0] / max(1, v[1]), 4)
197
- for t, v in sorted(correct_by_type.items())
198
- },
199
  "wall_time_s": round(time.time() - t0, 1),
200
  }
201
  results.append(summary)
 
8
  of the 10 choices given (memory context + question), pick argmax.
9
  3. Report accuracy overall and per question type
10
  (single_hop / multi_hop / temporal_reasoning / open_domain /
11
+ adversarial), plus wall time.
12
 
13
  Systems: clerk (adapter), prompted_ledger (same base, untrained), full
14
+ (entire conversation in context), rag (top-k session summaries, using the
15
+ haystack_session_summaries that ship with each item).
16
 
17
  A ledger cache keyed by the conversation's session ids means each
18
  conversation is consolidated exactly once.
 
39
  from clerk.prompts import CONSOLIDATE_SYSTEM, CONSOLIDATE_USER_TMPL
40
 
41
  BASE_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
42
+ MC10_REPO = "Percena/locomo-mc10"
 
43
 
44
+ MC_SYSTEM = ("You are a persistent assistant with memory of past "
45
+ "conversations. Answer the question using only the memory or "
46
+ "context provided. Respond with the correct option text.")
47
+ LEDGER_TMPL = "MEMORY LEDGER:\n{context}\n\nQuestion: {question}\n\nAnswer:"
48
+ FULL_TMPL = "CONVERSATION SO FAR:\n{context}\n\nQuestion: {question}\n\nAnswer:"
49
+ RAG_TMPL = ("RELEVANT SESSION SUMMARIES:\n{context}\n\n"
50
+ "Question: {question}\n\nAnswer:")
51
+ RAG_K = 3
 
52
 
53
 
54
+ def load_model(adapter: Optional[str], base: str):
55
+ tok = AutoTokenizer.from_pretrained(base)
56
+ model = AutoModelForCausalLM.from_pretrained(
57
+ base, dtype=torch.bfloat16, device_map="auto",
58
+ attn_implementation="sdpa")
59
+ if adapter:
60
+ model = PeftModel.from_pretrained(model, adapter)
61
+ model.eval()
62
+ return model, tok
63
 
64
 
65
  def render_sessions(sessions: List[List[Dict[str, Any]]]) -> List[str]:
 
95
 
96
 
97
  @torch.no_grad()
98
+ def choice_logprobs(model, tok, user_content: str,
99
  choices: List[str]) -> List[float]:
100
+ """Total log-prob of each choice continuation after the scored prompt."""
101
+ msgs = [{"role": "system", "content": MC_SYSTEM},
102
+ {"role": "user", "content": user_content}]
103
+ prompt_text = tok.apply_chat_template(msgs, add_generation_prompt=True,
104
+ tokenize=False)
105
+ prompt_ids = tok(prompt_text, add_special_tokens=False).input_ids
106
+ device = model.device
107
+ scores: List[float] = []
108
  for choice in choices:
109
+ cont = tok(" " + str(choice).strip(), add_special_tokens=False).input_ids
110
+ ids = torch.tensor([prompt_ids + cont], device=device)
111
+ logits = model(ids).logits[0]
112
+ start = len(prompt_ids) - 1
113
+ lg = logits[start: start + len(cont)]
114
+ lp = torch.log_softmax(lg, dim=-1)
115
+ tgt = ids[0][start: start + len(cont)]
116
+ scores.append(float(lp[torch.arange(len(tgt := tgt_holder := None) or 0)]
117
+ )) if False else None
118
+ idxs = torch.arange(len(cont), device=device)
119
+ scores[-1] = float(lp[idxs, tgt].sum())
120
  return scores
121
 
122
 
123
+ def build_ledgers(model, tok, items: List[Dict[str, Any]],
124
+ budget: int) -> Dict[Tuple, str]:
125
  """Consolidate each unique conversation once; cache by session-id tuple."""
126
  cache: Dict[Tuple, str] = {}
127
  for item in items:
 
138
  if err != "":
139
  ops = []
140
  ledger, _ = apply_ops(parse_ledger(ledger_json, budget), ops, idx)
141
+ ledger_json = serialize(ledger)
 
 
 
 
 
 
142
  cache[key] = ledger_json
143
  return cache
144
 
145
 
146
+ def answer_context(system: str, item: Dict[str, Any],
147
+ ledgers: Dict[Tuple, str]) -> str:
148
+ key = tuple(item["haystack_session_ids"])
149
+ if system in ("clerk", "prompted_ledger"):
150
+ return LEDGER_TMPL.format(context=ledgers[key],
151
+ question=item["question"])
152
+ if system == "full":
153
+ return FULL_TMPL.format(
154
+ context="\n\n".join(render_sessions(item["haystack_sessions"])),
155
+ question=item["question"])
156
+ if system == "rag":
157
+ return RAG_TMPL.format(
158
+ context="\n".join(item["haystack_session_summaries"][: RAG_K]),
159
+ question=item["question"])
160
+ raise ValueError(system)
161
+
162
+
163
  def main() -> None:
164
  ap = argparse.ArgumentParser(description="CLERK LoCoMo-MC10 evaluation")
165
+ ap.add_argument("--data", type=str, default=None,
166
+ help="path to locomo_mc10.json; if unset, downloads "
167
+ "data/locomo_mc10.json from Percena/locomo-mc10")
168
  ap.add_argument("--adapter", type=str, default=None)
169
  ap.add_argument("--base-model", type=str, default=BASE_MODEL)
170
  ap.add_argument("--systems", type=str,
 
174
  ap.add_argument("--out", type=str, required=True)
175
  args = ap.parse_args()
176
 
177
+ if args.data:
178
+ path = args.data
179
+ else:
180
+ import huggingface_hub
181
+ snapshot = huggingface_hub.snapshot_download(
182
+ MC10_REPO, allow_patterns=["data/locomo_mc10.json"])
183
+ path = f"{snapshot}/data/locomo_mc10.json"
184
+ with open(path) as f:
185
+ items = json.load(f)
186
  if args.max_items:
187
  items = items[: args.max_items]
188
 
189
  results = []
190
  for system in [s for s in args.systems.split(",") if s]:
191
+ model, tok = load_model(
192
  args.adapter if system == "clerk" else None, args.base_model)
193
  t0 = time.time()
194
+ ledgers = build_ledgers(model, tok, items, args.budget)
195
 
196
+ by_type: Dict[str, List[int]] = defaultdict(lambda: [0, 0])
197
+ done = 0
198
+ for item in items:
199
+ context = answer_context(system, item, ledgers)
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  lps = choice_logprobs(model, tok, context, item["choices"])
201
+ ok = int(int(np.argmax(lps)) == item["correct_choice_index"])
202
+ by_type[item["question_type"]][0] += ok
203
+ by_type[item["question_type"]][1] += 1
204
+ done += 1
205
+ if done % 200 == 0:
206
+ print(f"[{system}] {done}/{len(items)} items", flush=True)
 
207
 
208
  summary = {
209
  "system": system,
210
  "n_items": len(items),
211
  "accuracy_overall": round(
212
+ sum(v[0] for v in by_type.values())
213
+ / max(1, sum(v[1] for v in by_type.values())), 4),
214
+ "accuracy_by_type": {t: round(v[0] / max(1, v[1]), 4)
215
+ for t, v in sorted(by_type.items())},
216
+ "n_by_type": {t: v[1] for t, v in sorted(by_type.items())},
 
217
  "wall_time_s": round(time.time() - t0, 1),
218
  }
219
  results.append(summary)