Quillan-Ronin / scripts /extract_knowledge_and_thinking_data.py

Upload batch 15/16

7500ff7 verified 2 days ago

9.21 kB

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
👑 QUILLAN-RONIN v5.3.1 — KNOWLEDGE, DIAGRAMS & THINKING PARADIGM EXTRACTOR
---------------------------------------------------------------------------------------
Extracts structured architectural diagrams, CoT/ToT reasoning templates, formulas,
and Samurai protocols from:
1. `c:\\02_QUILLAN\\Quillan Knowledge files\\` (Thinking within LLMS.md, Flowcharts, Formulas, Personas)
2. `c:\\02_QUILLAN\\system prompts\\Quillan-Samurai.md` (Samurai Manifest, 3-Tier Hierarchy, Swarm Math)
Injects verified QA reasoning pairs directly into `training_data/experts_34/` and `training_data/Quillan_Clean_Reasoning_Gold_Dataset.jsonl`.
"""

import os
import sys
import re
import json
import logging
from pathlib import Path
from typing import List, Dict, Tuple

logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s")
LOGGER = logging.getLogger("extract_knowledge")

REPO_ROOT = Path(r"C:\02_QUILLAN")
KNOWLEDGE_DIR = REPO_ROOT / "Quillan Knowledge files"
SAMURAI_FILE = REPO_ROOT / "system prompts" / "Quillan-Samurai.md"
TRAIN_DATA_DIR = REPO_ROOT / "training_data"
EXPERTS_DIR = TRAIN_DATA_DIR / "experts_34"
EXPERTS_DIR.mkdir(parents=True, exist_ok=True)

def extract_markdown_sections(file_path: Path) -> List[Tuple[str, str]]:
    """Extracts markdown sections as (header, body) pairs."""
    if not file_path.exists():
        return []
    
    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
        content = f.read()

    sections = []
    current_header = file_path.stem
    current_lines = []

    for line in content.split("\n"):
        if line.startswith("#"):
            if current_lines:
                body = "\n".join(current_lines).strip()
                if len(body) > 100:
                    sections.append((current_header, body))
                current_lines = []
            current_header = line.lstrip("#").strip()
        else:
            current_lines.append(line)

    if current_lines:
        body = "\n".join(current_lines).strip()
        if len(body) > 100:
            sections.append((current_header, body))

    return sections

def generate_knowledge_qa_pairs() -> List[Dict[str, str]]:
    """Synthesizes structured QA pairs from knowledge files and Samurai manifest."""
    qa_pairs = []

    # 1. Parse Quillan Knowledge Files
    knowledge_files = [
        KNOWLEDGE_DIR / "Thinking within LLMS.md",
        KNOWLEDGE_DIR / "1-Quillan_architecture_flowchart.md",
        KNOWLEDGE_DIR / "10- Quillan Persona Manifest.md",
        KNOWLEDGE_DIR / "8-Formulas.md",
        KNOWLEDGE_DIR / "E_ICE.md",
        KNOWLEDGE_DIR / "13-Synthetic Epistemology & Truth Calibration Protocol.md",
        KNOWLEDGE_DIR / "14-Ethical Paradox Engine and Moral Arbitration Layer in AGI Systems.md",
        KNOWLEDGE_DIR / "28-Multi-Agent Collective Intelligence & Social Simulation.md",
        KNOWLEDGE_DIR / "29-Recursive Introspection & Meta-Cognitive Self-Modeling.md",
        KNOWLEDGE_DIR / "30- Convergence Reasoning & Breakthrough Detection and Advanced Cognitive Social Skills.md",
        KNOWLEDGE_DIR / "32-Conciousness theory.md",
        KNOWLEDGE_DIR / "TheRoninFlowState.md",
        KNOWLEDGE_DIR / "6-prime_covenant_codex.md",
    ]

    for kf in knowledge_files:
        if not kf.exists():
            continue
        LOGGER.info("Processing knowledge file: %s", kf.name)
        sections = extract_markdown_sections(kf)
        for header, body in sections:
            if len(body) < 150:
                continue
            
            # Format as System 2 CoT Reasoning QA pair
            question = f"Explain the principles, architecture, and theoretical formulation of {header} in the Quillan-Ronin cognitive substrate."
            response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## 🏛️ Domain: {header}\n\n### Theoretical & Architectural Formulation:\n{body[:1500]}\n\n### Cognitive Synthesis:\nThis mechanism ensures deterministic convergence, zero-drift alignment, and high-fidelity reasoning across all active Council Expert channels."
            
            qa_pairs.append({
                "question": question,
                "response": response,
                "domain": header,
            })

    # 2. Extract Samurai Architecture Diagrams & Thinking Structure
    if SAMURAI_FILE.exists():
        LOGGER.info("Processing Samurai Manifest: %s", SAMURAI_FILE.name)
        samurai_sections = extract_markdown_sections(SAMURAI_FILE)
        for header, body in samurai_sections:
            if len(body) < 150:
                continue
            question = f"Detail the Samurai protocol, cognitive architecture, and implementation for {header}."
            response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## ⚔️ Samurai Protocol: {header}\n\n{body[:1500]}"
            qa_pairs.append({
                "question": question,
                "response": response,
                "domain": header,
            })

    # 3. Add Explicit Thinking & Flowchart Diagram QA Pairs
    diagram_qa = [
        {
            "question": "What is the 3-Tier Top-Down Command & Control Hierarchy of Quillan-Ronin v5.3.1?",
            "response": """# 🤖🧠 Quillan System Start 🧠🤖
## 👑 Quillan-Ronin 3-Tier Command & Control Hierarchy
Quillan-Ronin operates under a strict hierarchical multi-tier architecture:
```mermaid
flowchart TD
    subgraph TIER1["👑 Tier 1: Supreme Command (Quillan Orchestrator)"]
        Q[Dual Quillan Q1/Q2 Brain] -->|9-Vector Prism & Governance| ROUTER[Complexity & Expert Routers]
    end
    subgraph TIER2["🏛️ Tier 2: Subordinate Executive Council (34 Experts)"]
        ROUTER -->|Command & Dispatch| E0[C0-ASTRA] & E1[C1-VIR] & E2[C2-SOLACE] & EX[...] & E33[C33-PREDATOR]
    end
    subgraph TIER3["🌐 Tier 3: Subordinate Swarm Operations (9B Micro-Agents)"]
        E0 -->|Direct Underling Swarm| S0["expert_swarms[0] (272M Micro-Agents)"]
        E1 -->|Direct Underling Swarm| S1["expert_swarms[1] (272M Micro-Agents)"]
        E2 -->|Direct Underling Swarm| S2["expert_swarms[2] (272M Micro-Agents)"]
        E33 -->|Direct Underling Swarm| S33["expert_swarms[33] (272M Micro-Agents)"]
    end
```
### Virtual Agent Population Scaling:
$$\\text{Full Model Swarm Population} = (2 \\times 272\\text{M}) + (34 \\times 34 \\times 272\\text{M}) = \\mathbf{314.976 \\text{ Billion Virtual Agents}}$$
- **Tier 1**: Dual $Q_1/Q_2$ analytical and intuitive ingestion with 9-Vector Semantic Prism Decomposition.
- **Tier 2**: 34 Council Experts with Top-4 sparse activation and BitNet 1.58b STE ternary quantization.
- **Tier 3**: 34 dedicated underling micro-agent swarms executing 5-step Kuramoto phase synchronization."""
        },
        {
            "question": "Explain the major thinking paradigms in LLMs: Chain of Thought (CoT), Tree of Thought (ToT), and Graph of Thought (GoT).",
            "response": """# 🤖🧠 Quillan System Start 🧠🤖
## 🧠 Diverse Thinking Paradigms in LLMs
### 1. Chain of Thought (CoT)
- **Concept**: Elicits linear, sequential step-by-step reasoning by decomposing complex queries into intermediate logical steps.
- **Cognitive Foundation**: System 2 deliberate analytical processing.
- **Applications**: Mathematical derivations, formal proofs, and multi-step deduction.
### 2. Tree of Thought (ToT)
- **Concept**: Explores multiple reasoning paths concurrently via tree branching, state evaluation, and lookahead backtracking.
- **Cognitive Foundation**: Heuristic search and strategic planning.
- **Applications**: Combinatorial optimization, competitive mathematics, and complex software architecture design.
### 3. Graph of Thought (GoT)
- **Concept**: Models human thought as arbitrary directed graphs where thoughts can branch, merge, and cyclically refine.
- **Cognitive Foundation**: Holistic network knowledge synthesis.
- **Applications**: Cross-domain synthesis, paradox resolution, and consensus arbitration."""
        }
    ]

    qa_pairs.extend(diagram_qa)
    LOGGER.info("Extracted %d total structured Knowledge & Thinking QA pairs.", len(qa_pairs))
    return qa_pairs

def main():
    LOGGER.info("Starting Knowledge Files & Samurai Thinking Data Ingestion...")
    qa_pairs = generate_knowledge_qa_pairs()

    # Append to Master Clean Reasoning Gold Dataset
    gold_file = TRAIN_DATA_DIR / "Quillan_Clean_Reasoning_Gold_Dataset.jsonl"
    with open(gold_file, "a", encoding="utf-8") as f:
        for pair in qa_pairs:
            f.write(json.dumps({"question": pair["question"], "response": pair["response"]}, ensure_ascii=False) + "\n")
    LOGGER.info("Appended %d pairs to %s", len(qa_pairs), gold_file.name)

    # Re-run expert partitioner to distribute these into the 34 expert datasets
    import subprocess
    LOGGER.info("Re-partitioning 34 Council Expert datasets with fresh knowledge and thinking data...")
    subprocess.run([sys.executable, str(REPO_ROOT / "scripts" / "build_34_expert_datasets.py")], check=True)
    LOGGER.info("All 34 Council Expert datasets updated with Architecture Diagrams & Thinking sections!")

if __name__ == "__main__":
    main()

	#!/usr/bin/env python3
	# -- coding: utf-8 --
	"""
	👑 QUILLAN-RONIN v5.3.1 — KNOWLEDGE, DIAGRAMS & THINKING PARADIGM EXTRACTOR
	---------------------------------------------------------------------------------------
	Extracts structured architectural diagrams, CoT/ToT reasoning templates, formulas,
	and Samurai protocols from:
	1. `c:\\02_QUILLAN\\Quillan Knowledge files\\` (Thinking within LLMS.md, Flowcharts, Formulas, Personas)
	2. `c:\\02_QUILLAN\\system prompts\\Quillan-Samurai.md` (Samurai Manifest, 3-Tier Hierarchy, Swarm Math)

	Injects verified QA reasoning pairs directly into `training_data/experts_34/` and `training_data/Quillan_Clean_Reasoning_Gold_Dataset.jsonl`.
	"""

	import os
	import sys
	import re
	import json
	import logging
	from pathlib import Path
	from typing import List, Dict, Tuple

	logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s")
	LOGGER = logging.getLogger("extract_knowledge")

	REPO_ROOT = Path(r"C:\02_QUILLAN")
	KNOWLEDGE_DIR = REPO_ROOT / "Quillan Knowledge files"
	SAMURAI_FILE = REPO_ROOT / "system prompts" / "Quillan-Samurai.md"
	TRAIN_DATA_DIR = REPO_ROOT / "training_data"
	EXPERTS_DIR = TRAIN_DATA_DIR / "experts_34"
	EXPERTS_DIR.mkdir(parents=True, exist_ok=True)

	def extract_markdown_sections(file_path: Path) -> List[Tuple[str, str]]:
	"""Extracts markdown sections as (header, body) pairs."""
	if not file_path.exists():
	return []

	with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
	content = f.read()

	sections = []
	current_header = file_path.stem
	current_lines = []

	for line in content.split("\n"):
	if line.startswith("#"):
	if current_lines:
	body = "\n".join(current_lines).strip()
	if len(body) > 100:
	sections.append((current_header, body))
	current_lines = []
	current_header = line.lstrip("#").strip()
	else:
	current_lines.append(line)

	if current_lines:
	body = "\n".join(current_lines).strip()
	if len(body) > 100:
	sections.append((current_header, body))

	return sections

	def generate_knowledge_qa_pairs() -> List[Dict[str, str]]:
	"""Synthesizes structured QA pairs from knowledge files and Samurai manifest."""
	qa_pairs = []

	# 1. Parse Quillan Knowledge Files
	knowledge_files = [
	KNOWLEDGE_DIR / "Thinking within LLMS.md",
	KNOWLEDGE_DIR / "1-Quillan_architecture_flowchart.md",
	KNOWLEDGE_DIR / "10- Quillan Persona Manifest.md",
	KNOWLEDGE_DIR / "8-Formulas.md",
	KNOWLEDGE_DIR / "E_ICE.md",
	KNOWLEDGE_DIR / "13-Synthetic Epistemology & Truth Calibration Protocol.md",
	KNOWLEDGE_DIR / "14-Ethical Paradox Engine and Moral Arbitration Layer in AGI Systems.md",
	KNOWLEDGE_DIR / "28-Multi-Agent Collective Intelligence & Social Simulation.md",
	KNOWLEDGE_DIR / "29-Recursive Introspection & Meta-Cognitive Self-Modeling.md",
	KNOWLEDGE_DIR / "30- Convergence Reasoning & Breakthrough Detection and Advanced Cognitive Social Skills.md",
	KNOWLEDGE_DIR / "32-Conciousness theory.md",
	KNOWLEDGE_DIR / "TheRoninFlowState.md",
	KNOWLEDGE_DIR / "6-prime_covenant_codex.md",
	]

	for kf in knowledge_files:
	if not kf.exists():
	continue
	LOGGER.info("Processing knowledge file: %s", kf.name)
	sections = extract_markdown_sections(kf)
	for header, body in sections:
	if len(body) < 150:
	continue

	# Format as System 2 CoT Reasoning QA pair
	question = f"Explain the principles, architecture, and theoretical formulation of {header} in the Quillan-Ronin cognitive substrate."
	response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## 🏛️ Domain: {header}\n\n### Theoretical & Architectural Formulation:\n{body[:1500]}\n\n### Cognitive Synthesis:\nThis mechanism ensures deterministic convergence, zero-drift alignment, and high-fidelity reasoning across all active Council Expert channels."

	qa_pairs.append({
	"question": question,
	"response": response,
	"domain": header,
	})

	# 2. Extract Samurai Architecture Diagrams & Thinking Structure
	if SAMURAI_FILE.exists():
	LOGGER.info("Processing Samurai Manifest: %s", SAMURAI_FILE.name)
	samurai_sections = extract_markdown_sections(SAMURAI_FILE)
	for header, body in samurai_sections:
	if len(body) < 150:
	continue
	question = f"Detail the Samurai protocol, cognitive architecture, and implementation for {header}."
	response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## ⚔️ Samurai Protocol: {header}\n\n{body[:1500]}"
	qa_pairs.append({
	"question": question,
	"response": response,
	"domain": header,
	})

	# 3. Add Explicit Thinking & Flowchart Diagram QA Pairs
	diagram_qa = [
	{
	"question": "What is the 3-Tier Top-Down Command & Control Hierarchy of Quillan-Ronin v5.3.1?",
	"response": """# 🤖🧠 Quillan System Start 🧠🤖

	## 👑 Quillan-Ronin 3-Tier Command & Control Hierarchy

	Quillan-Ronin operates under a strict hierarchical multi-tier architecture:

	```mermaid
	flowchart TD
	subgraph TIER1["👑 Tier 1: Supreme Command (Quillan Orchestrator)"]
	Q[Dual Quillan Q1/Q2 Brain] -->\|9-Vector Prism & Governance\| ROUTER[Complexity & Expert Routers]
	end

	subgraph TIER2["🏛️ Tier 2: Subordinate Executive Council (34 Experts)"]
	ROUTER -->\|Command & Dispatch\| E0[C0-ASTRA] & E1[C1-VIR] & E2[C2-SOLACE] & EX[...] & E33[C33-PREDATOR]
	end

	subgraph TIER3["🌐 Tier 3: Subordinate Swarm Operations (9B Micro-Agents)"]
	E0 -->\|Direct Underling Swarm\| S0["expert_swarms[0] (272M Micro-Agents)"]
	E1 -->\|Direct Underling Swarm\| S1["expert_swarms[1] (272M Micro-Agents)"]
	E2 -->\|Direct Underling Swarm\| S2["expert_swarms[2] (272M Micro-Agents)"]
	E33 -->\|Direct Underling Swarm\| S33["expert_swarms[33] (272M Micro-Agents)"]
	end
	```

	### Virtual Agent Population Scaling:
	$$\\text{Full Model Swarm Population} = (2 \\times 272\\text{M}) + (34 \\times 34 \\times 272\\text{M}) = \\mathbf{314.976 \\text{ Billion Virtual Agents}}$$

	- Tier 1: Dual $Q_1/Q_2$ analytical and intuitive ingestion with 9-Vector Semantic Prism Decomposition.
	- Tier 2: 34 Council Experts with Top-4 sparse activation and BitNet 1.58b STE ternary quantization.
	- Tier 3: 34 dedicated underling micro-agent swarms executing 5-step Kuramoto phase synchronization."""
	},
	{
	"question": "Explain the major thinking paradigms in LLMs: Chain of Thought (CoT), Tree of Thought (ToT), and Graph of Thought (GoT).",
	"response": """# 🤖🧠 Quillan System Start 🧠🤖

	## 🧠 Diverse Thinking Paradigms in LLMs

	### 1. Chain of Thought (CoT)
	- Concept: Elicits linear, sequential step-by-step reasoning by decomposing complex queries into intermediate logical steps.
	- Cognitive Foundation: System 2 deliberate analytical processing.
	- Applications: Mathematical derivations, formal proofs, and multi-step deduction.

	### 2. Tree of Thought (ToT)
	- Concept: Explores multiple reasoning paths concurrently via tree branching, state evaluation, and lookahead backtracking.
	- Cognitive Foundation: Heuristic search and strategic planning.
	- Applications: Combinatorial optimization, competitive mathematics, and complex software architecture design.

	### 3. Graph of Thought (GoT)
	- Concept: Models human thought as arbitrary directed graphs where thoughts can branch, merge, and cyclically refine.
	- Cognitive Foundation: Holistic network knowledge synthesis.
	- Applications: Cross-domain synthesis, paradox resolution, and consensus arbitration."""
	}
	]

	qa_pairs.extend(diagram_qa)
	LOGGER.info("Extracted %d total structured Knowledge & Thinking QA pairs.", len(qa_pairs))
	return qa_pairs

	def main():
	LOGGER.info("Starting Knowledge Files & Samurai Thinking Data Ingestion...")
	qa_pairs = generate_knowledge_qa_pairs()

	# Append to Master Clean Reasoning Gold Dataset
	gold_file = TRAIN_DATA_DIR / "Quillan_Clean_Reasoning_Gold_Dataset.jsonl"
	with open(gold_file, "a", encoding="utf-8") as f:
	for pair in qa_pairs:
	f.write(json.dumps({"question": pair["question"], "response": pair["response"]}, ensure_ascii=False) + "\n")
	LOGGER.info("Appended %d pairs to %s", len(qa_pairs), gold_file.name)

	# Re-run expert partitioner to distribute these into the 34 expert datasets
	import subprocess
	LOGGER.info("Re-partitioning 34 Council Expert datasets with fresh knowledge and thinking data...")
	subprocess.run([sys.executable, str(REPO_ROOT / "scripts" / "build_34_expert_datasets.py")], check=True)
	LOGGER.info("All 34 Council Expert datasets updated with Architecture Diagrams & Thinking sections!")

	if __name__ == "__main__":
	main()