| import json, os, sys |
| from pathlib import Path |
|
|
| BASE = Path(r"C:\Users\Admin\Quillan-Ronin") |
| MODEL_DIR = BASE / "Quillan-v4.2-model" / "quillan_custom" |
| DATA_DIR = BASE / "training_data" |
|
|
| |
| print("=== TRAINING DATA INVENTORY ===") |
| jsonl_files = list(DATA_DIR.glob("*.jsonl")) |
| pt_files = list(DATA_DIR.glob("*.pt")) |
|
|
| for f in sorted(jsonl_files, key=lambda x: x.stat().st_size, reverse=True): |
| sz = f.stat().st_size / 1024 / 1024 |
| |
| with open(f, 'r', encoding='utf-8', errors='ignore') as fh: |
| lines = sum(1 for _ in fh) |
| print(f" {f.name}: {sz:.1f} MB, {lines:,} samples") |
|
|
| print(f"\n=== MODEL CHECKPOINT ===") |
| print(f" Path: {MODEL_DIR}") |
| print(f" SafeTensors: {(MODEL_DIR / 'model.safetensors').exists()}") |
| print(f" Config: {(MODEL_DIR / 'config.json').exists()}") |
| print(f" Tokenizer: {(MODEL_DIR / 'tokenizer.json').exists()}") |
|
|
| |
| with open(MODEL_DIR / "tokenizer.json", 'r', encoding='utf-8') as f: |
| tok = json.load(f) |
| added = tok.get('added_tokens', []) |
| base_vocab = len(tok.get('model', {}).get('vocab', {})) |
| print(f" Base vocab: {base_vocab}") |
| print(f" Added tokens: {len(added)}") |
| print(f" Total vocab: {base_vocab + len(added)}") |
|
|
