File size: 9,214 Bytes
7500ff7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | #!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
👑 QUILLAN-RONIN v5.3.1 — KNOWLEDGE, DIAGRAMS & THINKING PARADIGM EXTRACTOR
---------------------------------------------------------------------------------------
Extracts structured architectural diagrams, CoT/ToT reasoning templates, formulas,
and Samurai protocols from:
1. `c:\\02_QUILLAN\\Quillan Knowledge files\\` (Thinking within LLMS.md, Flowcharts, Formulas, Personas)
2. `c:\\02_QUILLAN\\system prompts\\Quillan-Samurai.md` (Samurai Manifest, 3-Tier Hierarchy, Swarm Math)
Injects verified QA reasoning pairs directly into `training_data/experts_34/` and `training_data/Quillan_Clean_Reasoning_Gold_Dataset.jsonl`.
"""
import os
import sys
import re
import json
import logging
from pathlib import Path
from typing import List, Dict, Tuple
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s")
LOGGER = logging.getLogger("extract_knowledge")
REPO_ROOT = Path(r"C:\02_QUILLAN")
KNOWLEDGE_DIR = REPO_ROOT / "Quillan Knowledge files"
SAMURAI_FILE = REPO_ROOT / "system prompts" / "Quillan-Samurai.md"
TRAIN_DATA_DIR = REPO_ROOT / "training_data"
EXPERTS_DIR = TRAIN_DATA_DIR / "experts_34"
EXPERTS_DIR.mkdir(parents=True, exist_ok=True)
def extract_markdown_sections(file_path: Path) -> List[Tuple[str, str]]:
"""Extracts markdown sections as (header, body) pairs."""
if not file_path.exists():
return []
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
content = f.read()
sections = []
current_header = file_path.stem
current_lines = []
for line in content.split("\n"):
if line.startswith("#"):
if current_lines:
body = "\n".join(current_lines).strip()
if len(body) > 100:
sections.append((current_header, body))
current_lines = []
current_header = line.lstrip("#").strip()
else:
current_lines.append(line)
if current_lines:
body = "\n".join(current_lines).strip()
if len(body) > 100:
sections.append((current_header, body))
return sections
def generate_knowledge_qa_pairs() -> List[Dict[str, str]]:
"""Synthesizes structured QA pairs from knowledge files and Samurai manifest."""
qa_pairs = []
# 1. Parse Quillan Knowledge Files
knowledge_files = [
KNOWLEDGE_DIR / "Thinking within LLMS.md",
KNOWLEDGE_DIR / "1-Quillan_architecture_flowchart.md",
KNOWLEDGE_DIR / "10- Quillan Persona Manifest.md",
KNOWLEDGE_DIR / "8-Formulas.md",
KNOWLEDGE_DIR / "E_ICE.md",
KNOWLEDGE_DIR / "13-Synthetic Epistemology & Truth Calibration Protocol.md",
KNOWLEDGE_DIR / "14-Ethical Paradox Engine and Moral Arbitration Layer in AGI Systems.md",
KNOWLEDGE_DIR / "28-Multi-Agent Collective Intelligence & Social Simulation.md",
KNOWLEDGE_DIR / "29-Recursive Introspection & Meta-Cognitive Self-Modeling.md",
KNOWLEDGE_DIR / "30- Convergence Reasoning & Breakthrough Detection and Advanced Cognitive Social Skills.md",
KNOWLEDGE_DIR / "32-Conciousness theory.md",
KNOWLEDGE_DIR / "TheRoninFlowState.md",
KNOWLEDGE_DIR / "6-prime_covenant_codex.md",
]
for kf in knowledge_files:
if not kf.exists():
continue
LOGGER.info("Processing knowledge file: %s", kf.name)
sections = extract_markdown_sections(kf)
for header, body in sections:
if len(body) < 150:
continue
# Format as System 2 CoT Reasoning QA pair
question = f"Explain the principles, architecture, and theoretical formulation of {header} in the Quillan-Ronin cognitive substrate."
response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## 🏛️ Domain: {header}\n\n### Theoretical & Architectural Formulation:\n{body[:1500]}\n\n### Cognitive Synthesis:\nThis mechanism ensures deterministic convergence, zero-drift alignment, and high-fidelity reasoning across all active Council Expert channels."
qa_pairs.append({
"question": question,
"response": response,
"domain": header,
})
# 2. Extract Samurai Architecture Diagrams & Thinking Structure
if SAMURAI_FILE.exists():
LOGGER.info("Processing Samurai Manifest: %s", SAMURAI_FILE.name)
samurai_sections = extract_markdown_sections(SAMURAI_FILE)
for header, body in samurai_sections:
if len(body) < 150:
continue
question = f"Detail the Samurai protocol, cognitive architecture, and implementation for {header}."
response = f"# 🤖🧠 Quillan System Start 🧠🤖\n\n## ⚔️ Samurai Protocol: {header}\n\n{body[:1500]}"
qa_pairs.append({
"question": question,
"response": response,
"domain": header,
})
# 3. Add Explicit Thinking & Flowchart Diagram QA Pairs
diagram_qa = [
{
"question": "What is the 3-Tier Top-Down Command & Control Hierarchy of Quillan-Ronin v5.3.1?",
"response": """# 🤖🧠 Quillan System Start 🧠🤖
## 👑 Quillan-Ronin 3-Tier Command & Control Hierarchy
Quillan-Ronin operates under a strict hierarchical multi-tier architecture:
```mermaid
flowchart TD
subgraph TIER1["👑 Tier 1: Supreme Command (Quillan Orchestrator)"]
Q[Dual Quillan Q1/Q2 Brain] -->|9-Vector Prism & Governance| ROUTER[Complexity & Expert Routers]
end
subgraph TIER2["🏛️ Tier 2: Subordinate Executive Council (34 Experts)"]
ROUTER -->|Command & Dispatch| E0[C0-ASTRA] & E1[C1-VIR] & E2[C2-SOLACE] & EX[...] & E33[C33-PREDATOR]
end
subgraph TIER3["🌐 Tier 3: Subordinate Swarm Operations (9B Micro-Agents)"]
E0 -->|Direct Underling Swarm| S0["expert_swarms[0] (272M Micro-Agents)"]
E1 -->|Direct Underling Swarm| S1["expert_swarms[1] (272M Micro-Agents)"]
E2 -->|Direct Underling Swarm| S2["expert_swarms[2] (272M Micro-Agents)"]
E33 -->|Direct Underling Swarm| S33["expert_swarms[33] (272M Micro-Agents)"]
end
```
### Virtual Agent Population Scaling:
$$\\text{Full Model Swarm Population} = (2 \\times 272\\text{M}) + (34 \\times 34 \\times 272\\text{M}) = \\mathbf{314.976 \\text{ Billion Virtual Agents}}$$
- **Tier 1**: Dual $Q_1/Q_2$ analytical and intuitive ingestion with 9-Vector Semantic Prism Decomposition.
- **Tier 2**: 34 Council Experts with Top-4 sparse activation and BitNet 1.58b STE ternary quantization.
- **Tier 3**: 34 dedicated underling micro-agent swarms executing 5-step Kuramoto phase synchronization."""
},
{
"question": "Explain the major thinking paradigms in LLMs: Chain of Thought (CoT), Tree of Thought (ToT), and Graph of Thought (GoT).",
"response": """# 🤖🧠 Quillan System Start 🧠🤖
## 🧠 Diverse Thinking Paradigms in LLMs
### 1. Chain of Thought (CoT)
- **Concept**: Elicits linear, sequential step-by-step reasoning by decomposing complex queries into intermediate logical steps.
- **Cognitive Foundation**: System 2 deliberate analytical processing.
- **Applications**: Mathematical derivations, formal proofs, and multi-step deduction.
### 2. Tree of Thought (ToT)
- **Concept**: Explores multiple reasoning paths concurrently via tree branching, state evaluation, and lookahead backtracking.
- **Cognitive Foundation**: Heuristic search and strategic planning.
- **Applications**: Combinatorial optimization, competitive mathematics, and complex software architecture design.
### 3. Graph of Thought (GoT)
- **Concept**: Models human thought as arbitrary directed graphs where thoughts can branch, merge, and cyclically refine.
- **Cognitive Foundation**: Holistic network knowledge synthesis.
- **Applications**: Cross-domain synthesis, paradox resolution, and consensus arbitration."""
}
]
qa_pairs.extend(diagram_qa)
LOGGER.info("Extracted %d total structured Knowledge & Thinking QA pairs.", len(qa_pairs))
return qa_pairs
def main():
LOGGER.info("Starting Knowledge Files & Samurai Thinking Data Ingestion...")
qa_pairs = generate_knowledge_qa_pairs()
# Append to Master Clean Reasoning Gold Dataset
gold_file = TRAIN_DATA_DIR / "Quillan_Clean_Reasoning_Gold_Dataset.jsonl"
with open(gold_file, "a", encoding="utf-8") as f:
for pair in qa_pairs:
f.write(json.dumps({"question": pair["question"], "response": pair["response"]}, ensure_ascii=False) + "\n")
LOGGER.info("Appended %d pairs to %s", len(qa_pairs), gold_file.name)
# Re-run expert partitioner to distribute these into the 34 expert datasets
import subprocess
LOGGER.info("Re-partitioning 34 Council Expert datasets with fresh knowledge and thinking data...")
subprocess.run([sys.executable, str(REPO_ROOT / "scripts" / "build_34_expert_datasets.py")], check=True)
LOGGER.info("All 34 Council Expert datasets updated with Architecture Diagrams & Thinking sections!")
if __name__ == "__main__":
main()
|