stt4 / backend /data /knowledge_index.json
vynor's picture
Deploy QORIN CPU Research Platform & Inference Backend
1bd0d2b verified
Raw History Blame Contribute Delete
5.2 kB
[
{
"chunk_id": "doc-model-1",
"doc_title": "QORIN Model Architecture Specifications",
"doc_source": "web/src/data/modelsData.ts",
"content": "QORIN-16M contains 16,768,128 parameters (23,305,344 checkpoint tensors including tied embeddings), 6 layers, 384 hidden dimension, 6 attention heads, standard GELU feed-forward network with 1024 intermediate dimension, 512 context length, trained on 150M tokens. It confirmed the syntax-execution capability bottleneck.",
"section": "QORIN-16M Proof of Concept",
"token_count": 50,
"metadata": {
"type": "architecture_spec"
}
},
{
"chunk_id": "doc-model-2",
"doc_title": "QORIN Model Architecture Specifications",
"doc_source": "web/src/data/modelsData.ts",
"content": "QORIN-50M contains 48,934,912 parameters (58,083,840 checkpoint tensors), 12 layers, 512 hidden dimension, 8 query heads, 4 KV heads (2:1 Grouped-Query Attention ratio), SwiGLU activation with 1376 intermediate dimension, 512 context length. In iso-data ablation on 250M tokens, GQA halved KV-cache memory while preserving 99.4% perplexity.",
"section": "QORIN-50M Iso-Data Architecture Champion",
"token_count": 58,
"metadata": {
"type": "architecture_spec"
}
},
{
"chunk_id": "doc-model-3",
"doc_title": "QORIN Model Architecture Specifications",
"doc_source": "web/src/data/modelsData.ts",
"content": "QORIN-100M contains 104,807,424 parameters (120,029,440 checkpoint tensors), 14 layers, 768 hidden dimension, 12 query heads, 4 KV heads (3:1 Grouped-Query Attention ratio), SwiGLU activation with 2048 intermediate dimension. In the 4K context pilot, context length was extended from 512 to 4096 tokens using linear RoPE frequency scaling (s = 8.0). Passkey needle-in-a-haystack retrieval collapsed to 4.0% across 4K tokens despite stable perplexity.",
"section": "QORIN-100M Baseline & 4K Context Extension",
"token_count": 80,
"metadata": {
"type": "architecture_spec"
}
},
{
"chunk_id": "doc-model-4",
"doc_title": "QORIN Model Architecture Specifications",
"doc_source": "web/src/data/modelsData.ts",
"content": "QORIN-500M contains 514,389,504 parameters (569,964,032 checkpoint tensors), 20 layers, 1536 hidden dimension, 24 query heads, 4 KV heads (6:1 Grouped-Query Attention ratio), SwiGLU activation with 3840 intermediate dimension, 2048 context length, trained on a 5B-token corpus across 101 binary shards with token-weighted loss.",
"section": "QORIN-500M Pretraining Campaign Target",
"token_count": 55,
"metadata": {
"type": "architecture_spec"
}
},
{
"chunk_id": "doc-hw-5",
"doc_title": "Hardware & GPU Duty Cycle Specification",
"doc_source": "modern_gpt/reports/phase1c_pretraining/gpu_cycle_log.jsonl",
"content": "QORIN pretraining and fine-tuning execute on a single physical consumer laptop machine with an NVIDIA GeForce RTX 5060 Laptop GPU (8GB GDDR7 dedicated VRAM). Under PyTorch 2.16 with CUDA and BF16 mixed-precision, sustained throughput is ~5,054 tokens/sec during 500M pretraining (micro-batch size 1, gradient accumulation 16, sequence length 2048). Hardware duty cycle enforces 105 minutes active training followed by 15 minutes idle cooldown to maintain junction temp below 88\u00b0C.",
"section": "Hardware Environment",
"token_count": 89,
"metadata": {
"type": "hardware_telemetry"
}
},
{
"chunk_id": "doc-data-6",
"doc_title": "5B Token Pretraining Mixture & Licensing Compliance",
"doc_source": "web/src/data/datasetsData.ts",
"content": "The 5.0-Billion-token pretraining mixture spans 7 curated domains: FineWeb-Edu (40.0%, ODC-By 1.0), Cosmopedia v2 (15.0%, Apache-2.0), StarCoder Python & Polyglot (15.0%, Permissive open source), Standard Ebooks & Curated Literature (10.0%, Public Domain/CC0), Wikipedia Clean English (7.5%, CC-BY-SA-4.0), OpenWebMath (7.5%, ODC-By/Apache-2.0), and UltraChat 200k (strictly capped at 5.0%, MIT). The isolated validation split consists of 20,025,153 tokens with 0.00% training contamination verified by 25,137 document SHA-256 hashes.",
"section": "Pretraining Corpus",
"token_count": 85,
"metadata": {
"type": "data_manifest"
}
},
{
"chunk_id": "doc-sft-7",
"doc_title": "SFT Failure Forensics: Sycophancy & Greeting Collapse",
"doc_source": "modern_gpt/reports/phase1b_dataset/audit_sft_v3_results.json",
"content": "In SFT 4K v3, the model achieved an all-time low validation loss (0.765 nats) but suffered catastrophic conversational collapse. Forensic inspection revealed that unmasked conversational dialogue turns in UltraChat (80.3% of the dataset) caused severe template lock-in. The unmasked greeting density was uncovered at exactly 0.026% (9 records out of 34,700), causing the model to loop assistant tags instead of returning greetings. This motivated SFT 4K v4 with strict dialogue capping and token-weighted loss.",
"section": "SFT Alignment Forensics",
"token_count": 96,
"metadata": {
"type": "forensics"
}
}
]