Upload 149 files

hugging/td_fuse/grpo_reason.py

@@ -45,7 +45,7 @@ from dataclasses import dataclass
 @dataclass
 class GRPOConfig:
     """Configuration for overnight GRPO reasoning training."""
-    model_path: str = "td_fuse_outputs/healed_final"
+    model_path: str = "td_fuse_outputs/reasoning_healed"
     output_dir: str = "td_fuse_outputs/grpo_cycle2"
     progress_file: str = "td_fuse_outputs/grpo_cycle2/progress.json"
 
@@ -208,10 +208,18 @@ def generate_all_problems(cfg: GRPOConfig) -> List[Dict]:
                 b_val = random.randint(10, 100)
             diff = abs(a_val - b_val)
             who = "Alice" if a_val > b_val else "Bob"
+            # Split into two separate problems to avoid compound answer verification issues
+            # Problem 1: Who has more? (name answer)
             problems.append({
-                "question": f"Alice has {a_val} marbles and Bob has {b_val} marbles. Who has more and by how many?",
+                "question": f"Alice has {a_val} marbles and Bob has {b_val} marbles. Who has more?",
+                "answer": who.lower(), "type": "math_reasoning",
+                "think_hint": f"Alice has {a_val}, Bob has {b_val}. {who} has more."
+            })
+            # Problem 2: By how many? (numeric answer)
+            problems.append({
+                "question": f"Alice has {a_val} marbles and Bob has {b_val} marbles. What is the difference?",
                 "answer": str(diff), "type": "math_reasoning",
-                "think_hint": f"{who} has more. Difference: {max(a_val,b_val)} - {min(a_val,b_val)} = {diff}."
+                "think_hint": f"Difference: {max(a_val,b_val)} - {min(a_val,b_val)} = {diff}."
             })
 
     random.shuffle(problems)
@@ -364,13 +372,21 @@ def verify_answer(problem: Dict, response: str) -> float:
             except ValueError:
                 pass
 
-        # Second: check last 2 numbers only (avoids problem restatement numbers)
+        # Second: check the LAST number only (the conclusion)
+        # If model says "25 - 5 = 20, answer is 20", last number = 20
+        # Checking last 2 caused false positives when expected appeared in calculation
         numbers = re.findall(r'-?\d+\.?\d*', answer_text)
         if numbers:
-            check_nums = numbers[-2:] if len(numbers) > 2 else numbers
-            for num in check_nums:
+            try:
+                if abs(float(numbers[-1]) - float(expected)) < 0.01:
+                    return 1.0
+            except ValueError:
+                pass
+            # Fallback: check for "answer is X" or "= X" patterns
+            answer_pattern = re.search(r'(?:answer is|equals|=)\s*(-?\d+\.?\d*)', answer_text)
+            if answer_pattern:
                 try:
-                    if abs(float(num) - float(expected)) < 0.01:
+                    if abs(float(answer_pattern.group(1)) - float(expected)) < 0.01:
                         return 1.0
                 except ValueError:
                     pass

hugging/td_fuse/test_before_grpo.py

@@ -0,0 +1,174 @@
+"""
+Quick test: verify functions + healed model reasoning check.
+Run this BEFORE GRPO to make sure everything works.
+
+Part 1: Unit tests on verify_answer (catches false positives)
+Part 2: Ask the healed model 10 reasoning questions
+"""
+
+import sys
+import os
+
+# ============================================================
+# PART 1: UNIT TESTS ON VERIFY FUNCTIONS
+# ============================================================
+
+print("=" * 60)
+print("PART 1: VERIFY FUNCTION UNIT TESTS")
+print("=" * 60)
+
+from td_fuse.grpo_reason import verify_answer
+
+tests_passed = 0
+tests_failed = 0
+
+def check(desc, problem, response, expected_score):
+    global tests_passed, tests_failed
+    score = verify_answer(problem, response)
+    status = "PASS" if score == expected_score else "FAIL"
+    if status == "FAIL":
+        tests_failed += 1
+        print(f"  {status}: {desc}")
+        print(f"    Expected {expected_score}, got {score}")
+        print(f"    Response: {response[:100]}")
+    else:
+        tests_passed += 1
+
+# -- "same" trick questions --
+check("same: correct answer",
+      {"answer": "same"}, "They both weigh the same — a pound is a pound.", 1.0)
+check("same: wrong answer says heavier",
+      {"answer": "same"}, "Bricks is heavier because bricks are denser.", 0.0)
+check("same: mentions 'same' but also says heavier (contradiction)",
+      {"answer": "same"}, "They are the same but steel is heavier", 0.0)
+
+# -- yes/no questions --
+check("yes: correct",
+      {"answer": "yes"}, "Yes, whales do breathe air since they are mammals.", 1.0)
+check("yes: wrong (says no)",
+      {"answer": "yes"}, "No, whales cannot breathe air.", 0.0)
+check("no: correct (not definitely)",
+      {"answer": "no"}, "Not necessarily — only some dogs are large.", 1.0)
+check("no: wrong (starts with yes)",
+      {"answer": "no"}, "Yes, Rex is definitely large.", 0.0)
+
+# -- "once" trick --
+check("once: correct",
+      {"answer": "once"}, "You can only subtract 5 from 25 once. After that it's 20.", 1.0)
+check("once: wrong (says 5 times)",
+      {"answer": "once"}, "You can subtract 5 from 25 five times: 25, 20, 15, 10, 5.", 0.0)
+
+# -- "all" months trick --
+check("all: correct (all 12)",
+      {"answer": "all"}, "All 12 months have at least 28 days.", 1.0)
+check("all: wrong (says 1)",
+      {"answer": "all"}, "Only February has exactly 28 days.", 0.0)
+
+# -- "neither" rooster trick --
+check("neither: correct",
+      {"answer": "neither"}, "Neither side — roosters don't lay eggs!", 1.0)
+check("neither: wrong (says left)",
+      {"answer": "neither"}, "The egg rolls down the left side.", 0.0)
+
+# -- ordinal (2nd place) --
+check("2nd: correct with CoT",
+      {"answer": "2nd"},
+      "<think>If you pass 2nd place, you take their spot.</think>\nYou are now in second place.", 1.0)
+check("2nd: wrong (says 1st)",
+      {"answer": "2nd"},
+      "<think>You passed 2nd, so you must be first!</think>\nYou are in first place.", 0.0)
+
+# -- day names (temporal) --
+check("day: correct (wednesday)",
+      {"answer": "wednesday"},
+      "<think>Yesterday was Monday, so today is Tuesday. Tomorrow is Wednesday.</think>\nThe answer is Wednesday.", 1.0)
+check("day: wrong (says thursday)",
+      {"answer": "wednesday"},
+      "<think>Yesterday was Monday, today is Tuesday, tomorrow is Thursday.</think>\nThursday.", 0.0)
+
+# -- numeric: last number only --
+check("numeric: correct (last number matches)",
+      {"answer": "150"}, "60 × 2.5 = 150. The train travels 150 miles.", 1.0)
+check("numeric: false positive check (expected in calculation but wrong answer)",
+      {"answer": "20"}, "I start with 25 apples and eat 3, so I have 22 left.", 0.0)
+check("numeric: #### format correct",
+      {"answer": "105"}, "Each shirt is $15. 7 × 15 = 105.\n#### 105", 1.0)
+check("numeric: #### format wrong",
+      {"answer": "105"}, "Each shirt is $15. 7 × 15 = 100.\n#### 100", 0.0)
+
+# -- name answers (sam, carol, etc) --
+check("name: correct (sam won)",
+      {"answer": "sam"}, "<think>Sam finished first, then Tom, then Jerry.</think>\nSam won the race.", 1.0)
+check("name: wrong (says tom)",
+      {"answer": "sam"}, "<think>Tom finished first.</think>\nTom won the race.", 0.0)
+
+# -- single letter answers --
+check("letter: correct (c is shortest)",
+      {"answer": "c"}, "<think>A > B > C.</think>\nC is the shortest.", 1.0)
+
+print(f"\nResults: {tests_passed} passed, {tests_failed} failed out of {tests_passed + tests_failed}")
+if tests_failed > 0:
+    print("FIX THE VERIFY BUGS BEFORE RUNNING GRPO!")
+    sys.exit(1)
+else:
+    print("All verify tests passed!")
+
+# ============================================================
+# PART 2: ASK THE HEALED MODEL (if model exists)
+# ============================================================
+
+MODEL_PATH = "td_fuse_outputs/reasoning_healed"
+if not os.path.exists(MODEL_PATH):
+    print(f"\nModel not found at {MODEL_PATH} — skipping live test")
+    sys.exit(0)
+
+print(f"\n{'=' * 60}")
+print("PART 2: HEALED MODEL REASONING TEST")
+print(f"{'=' * 60}")
+
+import torch
+from transformers import AutoModelForImageTextToText, AutoTokenizer
+
+print("Loading model...")
+model = AutoModelForImageTextToText.from_pretrained(
+    MODEL_PATH, dtype=torch.bfloat16,
+    device_map="auto", trust_remote_code=True
+)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+eos_id = tokenizer.convert_tokens_to_ids("<|im_end|>")
+print("Loaded!\n")
+
+questions = [
+    ("Which is heavier: a pound of feathers or a pound of bricks?", "same"),
+    ("If yesterday was Monday, what day is tomorrow?", "wednesday"),
+    ("A farmer has 17 sheep. All but 9 die. How many are left?", "9"),
+    ("What is 15% of 200?", "30"),
+    ("All mammals breathe air. Whales are mammals. Do whales breathe air?", "yes"),
+    ("In a race, Tom finished before Jerry but after Sam. Who won?", "sam"),
+    ("How many months have 28 days?", "all"),
+    ("If A > B and B > C, who is smallest?", "c"),
+    ("A car goes 80 mph for 3 hours. How far?", "240"),
+    ("What comes next: 3, 7, 11, 15, 19?", "23"),
+]
+
+correct = 0
+for q, expected in questions:
+    prompt = (
+        f"<|im_start|>user\n{q}\n"
+        f"Think step by step before giving your final answer.<|im_end|>\n"
+        f"<|im_start|>assistant\n<think>\n"
+    )
+    ids = tokenizer(prompt, return_tensors="pt").to(model.device)
+    out = model.generate(**ids, max_new_tokens=500, do_sample=False, eos_token_id=eos_id)
+    response = tokenizer.decode(out[0][ids.input_ids.shape[1]:], skip_special_tokens=True)
+
+    score = verify_answer({"answer": expected}, response)
+    answer_part = response.split("</think>")[-1].strip() if "</think>" in response else response[:100]
+    status = "OK" if score > 0 else "FAIL"
+    if score > 0:
+        correct += 1
+    print(f"  {status}: {q[:60]}")
+    print(f"    -> {answer_part[:120]}")
+
+print(f"\nScore: {correct}/10")
+print("Model is ready for GRPO!" if correct >= 5 else "Model might need more healing — low score")