File size: 2,573 Bytes
fac8bb8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Shared, deterministic model/data definitions. Copyright (c) 2026 sraivante."""
from __future__ import annotations
import hashlib
import json
import re
import unicodedata
from datetime import datetime, timezone
from pathlib import Path
SEED = 20261017
CONTEXT = 1024
VOCAB = 33069
SPECIALS = ["<|pad|>", "<|endoftext|>", "<|im_end|>", "<|im_start|>", "<think>", "</think>"]
MAX_SOURCE_BYTES = 20_000_000_000
MAX_FILE_BYTES = 7_000_000_000
def now():
return datetime.now(timezone.utc).isoformat()
def sha(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for chunk in iter(lambda: f.read(8 * 1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def dump(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(path.name + '.partial')
tmp.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
tmp.replace(path)
def prompt_key(prompt):
# All identical normalized questions share a split, including different solutions.
s = re.sub(r'\s+', ' ', unicodedata.normalize('NFKC', str(prompt))).strip().casefold()
return hashlib.sha256(s.encode('utf-8')).digest()
def split_for(prompt):
bucket = int.from_bytes(prompt_key(prompt)[:8], 'big') % 1000
return 'validation' if bucket < 5 else ('test' if bucket < 10 else 'train')
def render_row(row):
prompt = str(row.get('prompt') or '').strip()
answer = str(row.get('answer') or '').strip()
thought = str(row.get('think') or '').strip()
if not prompt or not (answer or thought):
return None
# Preserve source Unicode, math, code indentation and newlines. No ASCII filtering.
reasoning = '<think>\n' + thought + '\n</think>\n' if thought else ''
return ('<|im_start|>user\n' + prompt + '<|im_end|>\n'
+ '<|im_start|>assistant\n' + reasoning + answer + '<|im_end|>\n<|endoftext|>')
def model_config():
from transformers import Qwen3Config
return Qwen3Config(
vocab_size=VOCAB, hidden_size=512, intermediate_size=1536,
num_hidden_layers=14, num_attention_heads=8, num_key_value_heads=4,
head_dim=128, max_position_embeddings=CONTEXT,
hidden_act='silu', rms_norm_eps=1e-6, rope_theta=1_000_000.0,
attention_bias=False, attention_dropout=0.0, tie_word_embeddings=True,
use_sliding_window=False, sliding_window=None, max_window_layers=14,
pad_token_id=0, bos_token_id=1, eos_token_id=2, use_cache=False,
architectures=['Qwen3ForCausalLM'],
)
|