Auto-load tokenizer for merge rules

generate_imatrix.py

@@ -260,9 +260,26 @@ class SimpleTokenizer:
     """Minimal BPE tokenizer from GGUF metadata, with HPC acceleration."""
 
     def __init__(self, model):
+        self.model_path = model._f.name
         self.tokens = model.kv.get('tokenizer.ggml.tokens', [])
         self.vocab_size = len(self.tokens)
         merges_raw = model.kv.get('tokenizer.ggml.merges', [])
+        
+        # Override for models (like Mistral v0.3) where gguf merges are missing
+        if not merges_raw:
+            try:
+                import json, os
+                model_dir = os.path.dirname(os.path.abspath(self.model_path))
+                tok_path = os.path.join(model_dir, 'tokenizer.json')
+                if os.path.exists(tok_path):
+                    with open(tok_path, 'r') as f:
+                        tok_data = json.load(f)
+                        merges_raw = tok_data.get('model', {}).get('merges', [])
+                    if merges_raw:
+                        print(f"  Injected {len(merges_raw)} merges from local tokenizer.json!")
+            except Exception as e:
+                pass
+
         self.bos_id = model.kv.get('tokenizer.ggml.bos_token_id', 2)
         self.eos_id = model.kv.get('tokenizer.ggml.eos_token_id', 1)