Upload 163 files

hugging/td_fuse/td_data_finder.py

@@ -1,5 +1,12 @@
 """
-TD Dataset Finder v1 — Search Everywhere for Training Data
+TD Dataset Finder v2 — Search Everywhere for Training Data
+
+v2 changes (March 2026):
+- Known-good fallback datasets tried FIRST (reliable, parseable)
+- HuggingFace search only used if fallbacks don't have enough
+- More fallback datasets added per category (4-5 each)
+- Retention floor increased to 20% (prevents catastrophic forgetting)
+- Stall detection in fallback downloads (skip after 500 unparseable items)
 
 Takes the weakness report from td_weakness.py and finds training data
 to fix those weaknesses. Searches multiple sources in priority order.
@@ -7,10 +14,11 @@ to fix those weaknesses. Searches multiple sources in priority order.
 This is Step 3 of the self-improvement loop.
 
 Sources (in order of priority):
-  1. HuggingFace Datasets API — biggest, easiest to download
-  2. GitHub — repos with training data, benchmarks, problem sets
-  3. Kaggle — competitions and datasets
-  4. (Future) Web scraping — Stack Overflow, math forums, etc.
+  1. Known-good fallback datasets (curated, tested, parseable)
+  2. HuggingFace Datasets API search (if fallbacks aren't enough)
+  3. GitHub — repos with training data, benchmarks, problem sets
+  4. Kaggle — competitions and datasets
+  5. (Future) Web scraping — Stack Overflow, math forums, etc.
 
 Input:
   - WeaknessReport from td_weakness.py
@@ -374,29 +382,39 @@ FALLBACK_DATASETS = {
         ("openai/gsm8k", "main"),          # 8.8K train, question/answer with ####
         ("lighteval/MATH", "all"),           # 12.5K, competition math
         ("microsoft/orca-math-word-problems-200k", None),  # 200K word problems
+        ("AI-MO/NuminaMath-CoT", None),     # 860K chain-of-thought math
+        ("TIGER-Lab/MathInstruct", None),   # 262K math instruction pairs
     ],
     "code": [
         ("openai/openai_humaneval", None),   # 164 problems — small but high quality
         ("google-research-datasets/mbpp", None),  # FIX: full name (short "mbpp" fails)
         ("codeparrot/apps", "introductory"), # Competitive programming
+        ("m-a-p/CodeFeedback-Filtered-Instruction", None),  # 157K code instructions
     ],
     "reasoning": [
         ("allenai/ai2_arc", "ARC-Challenge"),  # 2.6K challenge set
         ("Rowan/hellaswag", None),              # 40K commonsense
         ("allenai/openbookqa", "main"),         # FIX: replaced openai_prm800k (not a public dataset)
+        ("allenai/winogrande", "winogrande_xl"),  # 40K commonsense reasoning
+        ("tasksource/bigbench", None),          # Diverse reasoning tasks
     ],
     "creativity": [
-        ("teknium/OpenHermes-2.5", None),   # 1M instruction pairs
+        ("teknium/OpenHermes-2.5", None),   # 1M instruction pairs — diverse, includes creative
         ("Open-Orca/OpenOrca", None),        # 4.2M conversations
+        ("tatsu-lab/alpaca", None),           # 52K instructions — many creative tasks
+        ("WizardLMTeam/WizardLM_evol_instruct_V2_196k", None),  # 196K evolved instructions
     ],
     "knowledge": [
         ("TIGER-Lab/MMLU-Pro", None),        # MMLU harder version
         ("cais/mmlu", "all"),                 # Classic MMLU
         ("truthfulqa/truthful_qa", "generation"),  # 817 truthfulness questions
+        ("sail/regmix-data-sample", None),   # Diverse knowledge mix
     ],
     "instruction_following": [
         ("HuggingFaceH4/ultrafeedback_binarized", None),  # 61K preference pairs
         ("tatsu-lab/alpaca", None),           # 52K instructions
+        ("Intel/orca_dpo_pairs", None),       # 12K DPO pairs — strong instruction following
+        ("argilla/ultrafeedback-binarized-preferences", None),  # Alternative format
     ],
 }
 
@@ -445,39 +463,16 @@ def find_training_data(
 
         print(f"\n--- Searching for: {category} ({needed} samples needed) ---")
 
-        # Step 1: Search HuggingFace
-        print(f"  Searching HuggingFace...")
-        hf_results = search_huggingface(keywords, hf_tags, category)
-
-        if hf_results:
-            print(f"  Found {len(hf_results)} datasets on HuggingFace:")
-            for r in hf_results[:3]:
-                print(f"    - {r.name} (relevance: {r.relevance_score:.2f})")
-
-        # Step 2: Download from best matches
         category_samples = []
-        for result in hf_results:
-            if len(category_samples) >= needed:
-                break
 
-            remaining = needed - len(category_samples)
-            samples = download_huggingface_dataset(
-                dataset_name=result.name,
-                category=category,
-                num_samples=remaining,
-            )
-            category_samples.extend(samples)
-            if samples:
-                all_sources.append(result)
-
-        # Step 3: If not enough, try fallback datasets
-        if len(category_samples) < needed:
-            print(f"  Only got {len(category_samples)}/{needed}. Trying fallback datasets...")
-            fallbacks = FALLBACK_DATASETS.get(category, [])
+        # Step 1: Try KNOWN-GOOD fallback datasets FIRST (reliable, parseable)
+        fallbacks = FALLBACK_DATASETS.get(category, [])
+        if fallbacks:
+            print(f"  Trying {len(fallbacks)} known-good datasets for {category}...")
             try:
                 from datasets import load_dataset  # Import once outside loop
             except ImportError:
-                print("    Warning: datasets library not installed — cannot try fallbacks")
+                print("    Warning: datasets library not installed — cannot load datasets")
                 fallbacks = []  # Skip the fallback loop
             for ds_name, config in fallbacks:
                 if len(category_samples) >= needed:
@@ -491,6 +486,7 @@ def find_training_data(
                         ds = load_dataset(ds_name, streaming=True, split="train")
 
                     count = 0
+                    stall = 0
                     for item in ds:
                         if count >= remaining:
                             break
@@ -498,22 +494,56 @@ def find_training_data(
                         if sample:
                             category_samples.append(sample)
                             count += 1
+                            stall = 0
+                        else:
+                            stall += 1
+                            if stall > 500:
+                                break
 
                     if count > 0:
-                        print(f"    Got {count} from fallback: {ds_name}")
+                        print(f"    Got {count} from: {ds_name}")
                         all_sources.append(DatasetResult(
                             name=ds_name,
-                            source="huggingface_fallback",
+                            source="huggingface_known",
                             url=f"https://huggingface.co/datasets/{ds_name}",
-                            description=f"Fallback dataset for {category}",
+                            description=f"Known-good dataset for {category}",
                             size_estimate=count,
-                            relevance_score=0.5,
+                            relevance_score=0.8,
                             category=category,
                         ))
                 except Exception as e:
-                    print(f"    Fallback {ds_name} failed: {e}")
+                    print(f"    {ds_name} failed: {e}")
+                    continue
+
+        # Step 2: If still not enough, search HuggingFace for more
+        if len(category_samples) < needed:
+            print(f"  Got {len(category_samples)}/{needed} from known datasets. Searching HuggingFace...")
+            hf_results = search_huggingface(keywords, hf_tags, category)
+
+            if hf_results:
+                print(f"  Found {len(hf_results)} datasets on HuggingFace:")
+                for r in hf_results[:3]:
+                    print(f"    - {r.name} (relevance: {r.relevance_score:.2f})")
+
+            # Download from best matches
+            for result in hf_results:
+                if len(category_samples) >= needed:
+                    break
+
+                # Skip datasets we already used as fallbacks
+                if any(result.name == fb[0] for fb in fallbacks):
                     continue
 
+                remaining = needed - len(category_samples)
+                samples = download_huggingface_dataset(
+                    dataset_name=result.name,
+                    category=category,
+                    num_samples=remaining,
+                )
+                category_samples.extend(samples)
+                if samples:
+                    all_sources.append(result)
+
         # Track what we got
         if category_samples:
             all_samples.extend(category_samples)

hugging/td_fuse/td_weakness.py

@@ -478,11 +478,12 @@ def _allocate_data(
     ALL_CATEGORIES = ["math", "code", "reasoning", "creativity",
                       "knowledge", "instruction_following"]
 
-    # FLOOR: 10% of budget split equally across ALL categories
+    # FLOOR: 20% of budget split equally across ALL categories
     # This prevents catastrophic forgetting on categories above target
-    floor_pct = 0.10
+    # Increased from 10% to 20% — model at 95%+ needs more retention data
+    floor_pct = 0.20
     floor_total = int(total_budget * floor_pct)
-    floor_per_cat = max(floor_total // len(ALL_CATEGORIES), 50)
+    floor_per_cat = max(floor_total // len(ALL_CATEGORIES), 100)
 
     # Start with floor for every category
     allocation = {cat: floor_per_cat for cat in ALL_CATEGORIES}