Quillan-Ronin / scripts /clean_training_data.py

Upload batch 15/16

7500ff7 verified 2 days ago

4.78 kB

"""
Quillan-Ronin Training Data Cleaner v2.0
==========================================
INTENTIONALLY KEEPS:
  - media_metadata source entirely (trains the MoE expert router)
  - tool_use_excellence, coding_excellence (high-quality signal)
  - Quillan system prompts, templates, persona files
  - Python/JS/code files with actual logic
  - Downloads, misc sources
REMOVES only genuine noise leaked from full_pc_crawl / quillan_ronin:
  - npm/yarn lockfile JSON entries (sha512 integrity + requiresBuild)
  - Raw timestamped log lines (INFO/DEBUG/ERROR spam)
  - Pure copyright/license header-only files (MS boilerplate)
  - TextMate/VS Code grammar token pattern files
Source-aware logic:
  - media_metadata  → ALWAYS KEEP (intentional router training signal)
  - tool/coding     → ALWAYS KEEP
  - quillan_ronin   → filter out npm locks + log spam, keep rest
  - full_pc_crawl   → filter out copyright-only + grammar files, keep code
  - unknown         → apply noise filter
Output: domain_general_routing_clean.jsonl
"""
import json, re
from pathlib import Path

SRC = Path("C:/Users/Admin/Quillan-Ronin/training_data/domain_splits/domain_general_routing.jsonl")
DST = Path("C:/Users/Admin/Quillan-Ronin/training_data/domain_splits/domain_general_routing_clean.jsonl")

# ── Sources to always keep in full ──────────────────────────────────────────
ALWAYS_KEEP_SOURCES = {
    "media_metadata",           # intentional MoE router training
    "tool_use_excellence",
    "coding_excellence",
    "quillan_knowledge",
    "downloads",
}

# ── Noise patterns — only applied to non-protected sources ──────────────────
NOISE_RE = [
    # npm lockfile: has both requiresBuild AND sha512 integrity hash
    re.compile(r'"requiresBuild".*"integrity"\s*:\s*"sha512-', re.DOTALL),
    # Bare sha512 hash lines (no useful text around them)
    re.compile(r'^"[A-Za-z0-9_\-]+"\s*:\s*\{\s*"checkedAt"\s*:\s*\d+.*"integrity"\s*:\s*"sha512-'),
    # Timestamped log spam: 2026-05-14 12:34:56,789 [INFO] Something
    re.compile(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\[(?:INFO|DEBUG|WARNING|ERROR|CRITICAL)\]'),
    # Pure MS copyright header files (no real code below)
    re.compile(r'Copyright \(c\) Microsoft Corporation\. All rights reserved\.\s*$', re.MULTILINE),
    # TextMate/VS Code grammar scope names only
    re.compile(r'"name"\s*:\s*"(?:comment|punctuation|keyword|entity|storage|string|variable|support|meta)\.[a-z.]+\.[a-z.]+"'),
]

MIN_LEN = 30  # anything shorter is noise


def is_noise(text: str, source: str) -> bool:
    """Return True if this entry should be removed."""
    if source in ALWAYS_KEEP_SOURCES:
        return False  # never filter protected sources

    if len(text.strip()) < MIN_LEN:
        return True

    # Apply each noise pattern — only needs one match to flag
    for pat in NOISE_RE:
        if pat.search(text[:500]):
            return True

    return False


def main():
    total = kept = removed = 0
    removed_by_source: dict[str, int] = {}
    kept_by_source: dict[str, int] = {}

    with open(SRC, "r", encoding="utf-8", errors="replace") as fin, \
         open(DST, "w", encoding="utf-8") as fout:

        for raw_line in fin:
            raw_line = raw_line.strip()
            if not raw_line:
                continue
            total += 1

            try:
                d = json.loads(raw_line)
            except json.JSONDecodeError:
                removed += 1
                continue

            text   = d.get("text", "")
            source = d.get("meta", {}).get("source", "unknown")

            if is_noise(text, source):
                removed += 1
                removed_by_source[source] = removed_by_source.get(source, 0) + 1
            else:
                kept += 1
                kept_by_source[source] = kept_by_source.get(source, 0) + 1
                fout.write(raw_line + "\n")

            if total % 20000 == 0:
                print(f"  {total:,} processed — kept {kept:,}, removed {removed:,}", flush=True)

    print(f"\n=== Clean Complete ===")
    print(f"Total:   {total:,}")
    print(f"Kept:    {kept:,} ({100*kept/max(total,1):.1f}%)")
    print(f"Removed: {removed:,} ({100*removed/max(total,1):.1f}%)")

    print(f"\nKept by source:")
    for src, n in sorted(kept_by_source.items(), key=lambda x: -x[1]):
        print(f"  {n:>7,}  {src}")

    print(f"\nRemoved by source:")
    for src, n in sorted(removed_by_source.items(), key=lambda x: -x[1]):
        print(f"  {n:>7,}  {src}")

    size_mb = DST.stat().st_size / 1e6
    print(f"\nOutput: {DST.name}  ({size_mb:.1f} MB)")


if __name__ == "__main__":
    main()

	"""
	Quillan-Ronin Training Data Cleaner v2.0
	==========================================
	INTENTIONALLY KEEPS:
	- media_metadata source entirely (trains the MoE expert router)
	- tool_use_excellence, coding_excellence (high-quality signal)
	- Quillan system prompts, templates, persona files
	- Python/JS/code files with actual logic
	- Downloads, misc sources

	REMOVES only genuine noise leaked from full_pc_crawl / quillan_ronin:
	- npm/yarn lockfile JSON entries (sha512 integrity + requiresBuild)
	- Raw timestamped log lines (INFO/DEBUG/ERROR spam)
	- Pure copyright/license header-only files (MS boilerplate)
	- TextMate/VS Code grammar token pattern files

	Source-aware logic:
	- media_metadata → ALWAYS KEEP (intentional router training signal)
	- tool/coding → ALWAYS KEEP
	- quillan_ronin → filter out npm locks + log spam, keep rest
	- full_pc_crawl → filter out copyright-only + grammar files, keep code
	- unknown → apply noise filter

	Output: domain_general_routing_clean.jsonl
	"""
	import json, re
	from pathlib import Path

	SRC = Path("C:/Users/Admin/Quillan-Ronin/training_data/domain_splits/domain_general_routing.jsonl")
	DST = Path("C:/Users/Admin/Quillan-Ronin/training_data/domain_splits/domain_general_routing_clean.jsonl")

	# ── Sources to always keep in full ──────────────────────────────────────────
	ALWAYS_KEEP_SOURCES = {
	"media_metadata", # intentional MoE router training
	"tool_use_excellence",
	"coding_excellence",
	"quillan_knowledge",
	"downloads",
	}

	# ── Noise patterns — only applied to non-protected sources ──────────────────
	NOISE_RE = [
	# npm lockfile: has both requiresBuild AND sha512 integrity hash
	re.compile(r'"requiresBuild"."integrity"\s:\s*"sha512-', re.DOTALL),
	# Bare sha512 hash lines (no useful text around them)
	re.compile(r'^"[A-Za-z0-9_\-]+"\s:\s\{\s"checkedAt"\s:\s\d+."integrity"\s:\s"sha512-'),
	# Timestamped log spam: 2026-05-14 12:34:56,789 [INFO] Something
	re.compile(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\[(?:INFO\|DEBUG\|WARNING\|ERROR\|CRITICAL)\]'),
	# Pure MS copyright header files (no real code below)
	re.compile(r'Copyright \(c\) Microsoft Corporation\. All rights reserved\.\s*$', re.MULTILINE),
	# TextMate/VS Code grammar scope names only
	re.compile(r'"name"\s:\s"(?:comment\|punctuation\|keyword\|entity\|storage\|string\|variable\|support\|meta)\.[a-z.]+\.[a-z.]+"'),
	]

	MIN_LEN = 30 # anything shorter is noise


	def is_noise(text: str, source: str) -> bool:
	"""Return True if this entry should be removed."""
	if source in ALWAYS_KEEP_SOURCES:
	return False # never filter protected sources

	if len(text.strip()) < MIN_LEN:
	return True

	# Apply each noise pattern — only needs one match to flag
	for pat in NOISE_RE:
	if pat.search(text[:500]):
	return True

	return False


	def main():
	total = kept = removed = 0
	removed_by_source: dict[str, int] = {}
	kept_by_source: dict[str, int] = {}

	with open(SRC, "r", encoding="utf-8", errors="replace") as fin, \
	open(DST, "w", encoding="utf-8") as fout:

	for raw_line in fin:
	raw_line = raw_line.strip()
	if not raw_line:
	continue
	total += 1

	try:
	d = json.loads(raw_line)
	except json.JSONDecodeError:
	removed += 1
	continue

	text = d.get("text", "")
	source = d.get("meta", {}).get("source", "unknown")

	if is_noise(text, source):
	removed += 1
	removed_by_source[source] = removed_by_source.get(source, 0) + 1
	else:
	kept += 1
	kept_by_source[source] = kept_by_source.get(source, 0) + 1
	fout.write(raw_line + "\n")

	if total % 20000 == 0:
	print(f" {total:,} processed — kept {kept:,}, removed {removed:,}", flush=True)

	print(f"\n=== Clean Complete ===")
	print(f"Total: {total:,}")
	print(f"Kept: {kept:,} ({100*kept/max(total,1):.1f}%)")
	print(f"Removed: {removed:,} ({100*removed/max(total,1):.1f}%)")

	print(f"\nKept by source:")
	for src, n in sorted(kept_by_source.items(), key=lambda x: -x[1]):
	print(f" {n:>7,} {src}")

	print(f"\nRemoved by source:")
	for src, n in sorted(removed_by_source.items(), key=lambda x: -x[1]):
	print(f" {n:>7,} {src}")

	size_mb = DST.stat().st_size / 1e6
	print(f"\nOutput: {DST.name} ({size_mb:.1f} MB)")


	if __name__ == "__main__":
	main()