Delete pattern-matched folders (batch 7/11)

C6p5e18_410m_alpha0.050_beta0.045/step_110/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ca6a336965f7f0101fbe4a39dc86b8b0152cda660a0d1170fd0ec68373f68b13
-size 822421664

C6p5e18_410m_alpha0.050_beta0.045/step_110/optimizer_states/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ca6a336965f7f0101fbe4a39dc86b8b0152cda660a0d1170fd0ec68373f68b13
-size 822421664

C6p5e18_410m_alpha0.050_beta0.045/step_110/optimizer_states/optimizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:97ebb0e90e0ac1c0ff9780a3956596d8253591ccdc35c60301cc558c295b79ee
-size 1645034059

C6p5e18_410m_alpha0.050_beta0.045/step_110/optimizer_states/random_states_0.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b45b275223c5ffd4fe4c0aeb90e5e53b21df53ec545f716ef7f45f95e23a3949
-size 15081

C6p5e18_410m_alpha0.050_beta0.045/step_110/optimizer_states/scheduler.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0183971c5d0a20e92ba8c8db76425ea86478f65d9a0e5fe494f3c85f26cc9c20
-size 1465

C6p5e18_410m_alpha0.050_beta0.045/step_110/optimizer_states/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.050_beta0.045/step_110/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null
-}

C6p5e18_410m_alpha0.050_beta0.045/step_110/tokenizer.py

@@ -1,818 +0,0 @@
-"""
-Auto-generated self-contained HF tokenizer.
-Do NOT edit manually -- regenerate via training.hf_tokenizer_utils.save_hf_tokenizer().
-"""
-from __future__ import annotations
-
-# --- BaseTokenizer (inlined) ---
-# base_tokenizer.py
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
-
-class BaseTokenizer(ABC):
-    """Minimal interface for tokenizers used in pretraining."""
-
-    # ---- required ----
-    @abstractmethod
-    def encode(self, text: str) -> List[int]:
-        """Convert text/PGN to token IDs."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def decode(self, ids: List[int]) -> str:
-        """Convert token IDs back to text/PGN."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def get_vocab(self) -> Dict[str, int]:
-        """Return token -> id mapping (if available)."""
-        raise NotImplementedError
-
-    def bos_id(self) -> Optional[int]: return None
-    def eos_id(self) -> Optional[int]: return None
-    def pad_id(self) -> Optional[int]: return None
-    def get_vocab_size(self) -> int: return len(self.get_vocab())
-
-    def __call__(self, text: str) -> List[int]:
-        """Alias for encode()."""
-        return self.encode(text)
-
-# --- Concrete tokenizer (inlined) ---
-# lan_tokenizer_sft.py
-"""
-LAN Tokenizer with SFT support (CoT format with <T> and <sep> tokens).
-
-This extends the base LAN tokenizer with SFT-specific functionality:
-- <T> token for marking thinking/CoT content
-- <sep> token for separating prompt from response
-"""
-from typing import List, Dict, Optional, Tuple
-import io
-import chess, chess.pgn
-from tokenizers import Tokenizer
-from tokenizers.models import WordLevel
-from tokenizers.pre_tokenizers import WhitespaceSplit
-_RESULT = {"1-0", "0-1", "1/2-1/2", "*"}
-FILES = "abcdefgh"
-RANKS = "12345678"
-SQUARES = [f+r for f in FILES for r in RANKS]
-PROMOS = "QRBN"
-DIGITS = set("0123456789")
-
-# SFT special tokens for CoT format
-T_TOKEN = "<T>"
-T_END_TOKEN = "</T>"
-SEP_TOKEN = "<sep>"
-
-# Environment interaction / reward special tokens
-CALL_ENV_TOKEN = "<call_env>"
-VERIFY_TOKEN = "<verify>"
-REWARD_POS_TOKEN = "<+1>"
-REWARD_NEG_TOKEN = "<-1>"
-REWARD_ZERO_TOKEN = "<0>"
-ENV_TOKENS = [CALL_ENV_TOKEN]
-REWARD_TOKENS = [VERIFY_TOKEN, REWARD_POS_TOKEN, REWARD_NEG_TOKEN, REWARD_ZERO_TOKEN]
-
-def _vocab_with_sft(
-    include_move_numbers: bool,
-    keep_result: bool,
-    bos: str,
-    eos: str,
-    unk: str,
-    include_env_tokens: bool = False,
-    include_reward_tokens: bool = False,
-) -> Dict[str, int]:
-    """Create vocabulary including SFT special tokens."""
-    base = [bos, eos, unk]
-    ops = ["x", "=", "+", "#", "O-O", "O-O-O", ".", "..."]
-    toks = base + list("KQRBNP") + SQUARES + list(PROMOS) + ops
-    if include_move_numbers:
-        toks += list("0123456789")
-    if keep_result:
-        toks += sorted(_RESULT)
-
-    # Add SFT special tokens for CoT format
-    sft_tokens = [T_TOKEN, T_END_TOKEN, SEP_TOKEN]
-    toks += sft_tokens
-
-    # Add environment / reward tokens when requested
-    if include_env_tokens:
-        toks += ENV_TOKENS
-    if include_reward_tokens:
-        toks += REWARD_TOKENS
-
-    return {t: i for i, t in enumerate(dict.fromkeys(toks))}
-
-
-class LanTokenizerSFT(BaseTokenizer):
-    """
-    LAN Tokenizer with SFT capabilities.
-
-    This tokenizer extends the base LAN tokenizer with:
-    - <T> token for marking thinking/CoT boundaries
-    - <sep> token for separating candidate trajectories
-
-    CoT Format: {prompt} <T> <sep> {traj1} <sep> {traj2} <sep> ... <sep> {trajN} <sep> <T> {answer}
-
-    Where:
-    - {prompt}: The game history/board state (PGN moves)
-    - {trajN}: Candidate reasoning trajectories
-    - {answer}: The final best move
-    """
-
-    # Special tokens for CoT format
-    T = T_TOKEN
-    T_END = T_END_TOKEN
-    SEP = SEP_TOKEN
-
-    # Environment / reward tokens (class-level constants for easy access)
-    CALL_ENV = CALL_ENV_TOKEN   # "<call_env>"
-    VERIFY = VERIFY_TOKEN       # "<verify>"
-    REWARD_POS = REWARD_POS_TOKEN  # "<+1>"
-    REWARD_NEG = REWARD_NEG_TOKEN  # "<-1>"
-    REWARD_ZERO = REWARD_ZERO_TOKEN  # "<0>"
-    ENV_TOKENS = ENV_TOKENS     # full list
-
-    def __init__(self, config: Optional[dict] = None):
-        """
-        Args:
-            config: Configuration dict with tokenizer settings.
-                include_env_tokens (bool): add <call_env>, <verify>, <+1>, <-1>, <0>
-                    to the vocabulary.  Default: False.
-        """
-        config = config or {}
-
-        include_move_numbers = config.get("include_move_numbers", False)
-        include_black_tripledots = config.get("include_black_tripledots", False)
-        bos = config.get("bos", "<bos>")
-        eos = config.get("eos", "<eos>")
-        unk = config.get("unk", "<unk>")
-        keep_result = config.get("keep_result", False)
-        include_env_tokens = config.get("include_env_tokens", False)
-        include_reward_tokens = config.get("include_reward_tokens", False)
-
-        self._bos = bos
-        self._eos = eos
-        self._unk = unk
-        self._keep_res = keep_result
-        self._include_nums = include_move_numbers
-        self._include_black_ellipses = include_black_tripledots
-        self._include_env_tokens = include_env_tokens
-        self._include_reward_tokens = include_reward_tokens
-
-        # Create vocabulary with SFT tokens
-        tok2id = _vocab_with_sft(
-            include_move_numbers, keep_result, bos, eos, unk,
-            include_env_tokens=include_env_tokens,
-            include_reward_tokens=include_reward_tokens,
-        )
-        self._tok2id = tok2id
-
-        # Initialize tokenizer
-        self.tk = Tokenizer(WordLevel(vocab=tok2id, unk_token=self._unk))
-        self.tk.pre_tokenizer = WhitespaceSplit()
-    
-    def _pgn_to_tokens(self, text: str) -> Optional[List[str]]:
-        """Convert PGN text to tokens."""
-        import os, contextlib
-        with open(os.devnull, "w") as devnull, contextlib.redirect_stderr(devnull):
-            g = chess.pgn.read_game(io.StringIO(text))
-        if g is None:
-            return None
-        
-        b, out, n = g.board(), [], 1
-        for mv in g.mainline_moves():
-            if b.turn == chess.WHITE and self._include_nums:
-                out += list(str(n)) + (
-                    ["..."] if self._include_black_ellipses and b.fullmove_number < n else ["."]
-                )
-            
-            if b.is_castling(mv):
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                b.pop()
-                out.append("O-O" if chess.square_file(mv.to_square) == 6 else "O-O-O")
-                if suf:
-                    out.append(suf)
-                b.push(mv)
-            else:
-                piece = b.piece_at(mv.from_square).symbol().upper()
-                frm = chess.square_name(mv.from_square)
-                to = chess.square_name(mv.to_square)
-                is_cap = b.is_capture(mv)
-                promo = mv.promotion
-                
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                
-                # Emit LAN tokens
-                out.append(piece)
-                out.append(frm)
-                if is_cap:
-                    out.append("x")
-                out.append(to)
-                if promo:
-                    out += ["=", chess.piece_symbol(promo).upper()]
-                if suf:
-                    out.append(suf)
-            
-            if b.turn == chess.WHITE:
-                n += 1
-        
-        res = g.headers.get("Result")
-        if self._keep_res and res in _RESULT:
-            out.append(res)
-        
-        return out
-    
-    def _lan_move_to_tokens(self, move: str) -> List[str]:
-        """
-        Convert a single LAN move to tokens.
-        
-        LAN format: [Piece][from_square][x]?[to_square][=Promo]?[+#]?
-        
-        Examples:
-            "Ng1f3" -> ["N", "g1", "f3"]
-            "Nd4xe6" -> ["N", "d4", "x", "e6"]
-            "Pe2e4" -> ["P", "e2", "e4"]
-            "Pe4xd5" -> ["P", "e4", "x", "d5"]
-            "O-O" -> ["O-O"]
-            "O-O-O" -> ["O-O-O"]
-            "Pe7e8=Q" -> ["P", "e7", "e8", "=", "Q"]
-            "Ng1f3+" -> ["N", "g1", "f3", "+"]
-        """
-        # Handle castling
-        if move in {"O-O", "O-O-O"}:
-            return [move]
-        if move.rstrip("+#") in {"O-O", "O-O-O"}:
-            base = move.rstrip("+#")
-            suffix = move[len(base):]
-            return [base] + ([suffix] if suffix else [])
-        
-        out = []
-        i = 0
-        n = len(move)
-        
-        # Get piece letter (required in LAN format)
-        if i < n and move[i] in "KQRBNP":
-            out.append(move[i])
-            i += 1
-        else:
-            # No piece letter - might be malformed, return as-is
-            return [move]
-        
-        # Get from square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle capture
-        if i < n and move[i] == "x":
-            out.append("x")
-            i += 1
-        
-        # Get to square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle promotion
-        if i < n and move[i] == "=":
-            out.append("=")
-            i += 1
-            if i < n and move[i] in PROMOS:
-                out.append(move[i])
-                i += 1
-        
-        # Handle check/checkmate
-        if i < n and move[i] in "+#":
-            out.append(move[i])
-            i += 1
-        
-        return out
-    
-    def _active_env_tokens(self) -> set:
-        """Return the set of env tokens that are active for this instance."""
-        return set(ENV_TOKENS) if self._include_env_tokens else set()
-
-    def _cot_to_tokens(self, text: str) -> List[str]:
-        """
-        Convert CoT formatted text to tokens.
-        Handles special tokens and LAN moves.
-        """
-        env_toks = self._active_env_tokens()
-        out = []
-        for token in text.split():
-            if token in {self.T, self.T_END, self.SEP} or token in env_toks:
-                # Keep special tokens as-is
-                out.append(token)
-            elif token in _RESULT:
-                # Game result
-                out.append(token)
-            elif token and token[0].isdigit() and "." in token:
-                # Move number like "1." or "15..."
-                # Split into digits and dots
-                num_part = token.rstrip(".")
-                dot_part = token[len(num_part):]
-                out.extend(list(num_part))
-                if dot_part:
-                    out.append("..." if len(dot_part) > 1 else ".")
-            elif token and all(c.isdigit() for c in token):
-                # Pure number - tokenize each digit
-                out.extend(list(token))
-            else:
-                # LAN move - tokenize it
-                out.extend(self._lan_move_to_tokens(token))
-        return out
-    
-    def encode(self, text: str) -> List[int]:
-        """
-        Encode text to token IDs.
-        
-        Args:
-            text: Text to encode (can be PGN or CoT formatted)
-        
-        Returns:
-            List of token IDs
-        """
-        # Check if this is CoT-formatted text (contains special tokens)
-        sft_special = (
-            [self.T, self.T_END, self.SEP]
-            + (ENV_TOKENS if self._include_env_tokens else [])
-        )
-        is_cot_format = any(token in text for token in sft_special)
-        
-        if is_cot_format:
-            t_idx = text.index(self.T)
-            prompt_part = text[:t_idx].strip()
-            rest_part = text[t_idx:]  # starts with <T>
-
-            pgn_tokens = self._pgn_to_tokens(prompt_part) if prompt_part else None
-            if pgn_tokens is None:
-                pgn_tokens = self._cot_to_tokens(prompt_part) if prompt_part else []
-            rest_tokens = self._cot_to_tokens(rest_part)
-            tokens = [self._bos] + pgn_tokens + rest_tokens + [self._eos]
-        else:
-            pgn_tokens = self._pgn_to_tokens(text)
-            if pgn_tokens is not None and len(pgn_tokens) > 0:
-                tokens = [self._bos] + pgn_tokens + [self._eos]
-            else:
-                # Not valid PGN — treat each word as a LAN move
-                lan_tokens = []
-                for word in text.split():
-                    lan_tokens.extend(self._lan_move_to_tokens(word))
-                tokens = [self._bos] + lan_tokens + [self._eos]
-        
-        return self.tk.encode(" ".join(tokens)).ids
-    
-    def decode(self, ids: List[int]) -> str:
-        """
-        Decode token IDs to text.
-        
-        Args:
-            ids: List of token IDs
-        
-        Returns:
-            Decoded text
-        """
-        toks = [t for t in self.tk.decode(ids).split() if t not in {self._bos, self._eos}]
-        
-        # Otherwise, use LAN decoding logic
-        out: List[str] = []
-        i, n = 0, len(toks)
-        
-        while i < n:
-            t = toks[i]
-            
-            if t in {self.T, self.T_END, self.SEP} or t in _RESULT or t in self._active_env_tokens():
-                out.append(t)
-                i += 1
-                continue
-            
-            if t and all(ch in DIGITS for ch in t):
-                j = i
-                num = []
-                while j < n and all(ch in DIGITS for ch in toks[j]):
-                    num.append(toks[j])
-                    j += 1
-                dots = ""
-                if j < n and toks[j] in {".", "..."}:
-                    dots = toks[j]
-                    j += 1
-                out.append("".join(num) + dots)
-                i = j
-                continue
-            
-            if t in {"O-O", "O-O-O"}:
-                j = i + 1
-                suf = toks[j] if j < n and toks[j] in {"+", "#"} else ""
-                if suf:
-                    j += 1
-                out.append(t + suf)
-                i = j
-                continue
-            
-            if t in set("KQRBNP"):
-                piece = t
-                j = i + 1
-                frm = toks[j] if j < n else ""
-                j += 1
-                cap = ""
-                if j < n and toks[j] == "x":
-                    cap = "x"
-                    j += 1
-                to = toks[j] if j < n else ""
-                j += 1
-                promo = ""
-                if j + 1 <= n - 1 and toks[j] == "=" and toks[j + 1] in set(PROMOS):
-                    promo = "=" + toks[j + 1]
-                    j += 2
-                suf = ""
-                if j < n and toks[j] in {"+", "#"}:
-                    suf = toks[j]
-                    j += 1
-                lan = f"{piece}{frm}{cap}{to}{promo}{suf}"
-                out.append(lan)
-                i = j
-                continue
-            
-            out.append(t)
-            i += 1
-        
-        return " ".join(out)
-    
-    def get_vocab(self) -> Dict[str, int]:
-        """Get token-to-id vocabulary mapping."""
-        return self._tok2id
-    
-    def bos_id(self) -> Optional[int]:
-        """Get BOS token ID."""
-        return self._tok2id[self._bos]
-    
-    def eos_id(self) -> Optional[int]:
-        """Get EOS token ID."""
-        return self._tok2id[self._eos]
-    
-    def pad_id(self) -> Optional[int]:
-        """Get PAD token ID (uses BOS as pad by default)."""
-        return self._tok2id.get("<pad>", self.bos_id())
-    
-    def get_vocab_size(self) -> int:
-        """Get vocabulary size."""
-        return len(self._tok2id)
-    
-    def t_id(self) -> int:
-        """Get <T> token ID."""
-        return self._tok2id[self.T]
-    
-    def sep_id(self) -> int:
-        """Get <sep> token ID."""
-        return self._tok2id[self.SEP]
-    
-    def t_end_id(self) -> int:
-        """Get </T> token ID."""
-        return self._tok2id[self.T_END]
-    
-    # ------------------------------------------------------------------
-    # Environment / reward token accessors
-    # ------------------------------------------------------------------
-
-    def _require_env_tokens(self) -> None:
-        if not self._include_env_tokens:
-            raise ValueError(
-                "Environment tokens are not enabled. "
-                "Pass include_env_tokens=True in the config."
-            )
-
-    def call_env_id(self) -> int:
-        """Get <call_env> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[CALL_ENV_TOKEN]
-
-    def verify_id(self) -> int:
-        """Get <verify> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[VERIFY_TOKEN]
-
-    def reward_pos_id(self) -> int:
-        """Get <+1> (positive reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_POS_TOKEN]
-
-    def reward_neg_id(self) -> int:
-        """Get <-1> (negative reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_NEG_TOKEN]
-
-    def reward_zero_id(self) -> int:
-        """Get <0> (zero reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_ZERO_TOKEN]
-
-    def reward_id(self, value) -> int:
-        """
-        Get reward token ID by numeric value.
-
-        Args:
-            value: 1, -1, or 0  (or the strings "+1", "-1", "0")
-
-        Returns:
-            Token ID for the corresponding reward token.
-        """
-        self._require_env_tokens()
-        mapping = {1: REWARD_POS_TOKEN, -1: REWARD_NEG_TOKEN, 0: REWARD_ZERO_TOKEN,
-                   "+1": REWARD_POS_TOKEN, "-1": REWARD_NEG_TOKEN, "0": REWARD_ZERO_TOKEN}
-        if value not in mapping:
-            raise ValueError(f"reward value must be one of 1, -1, 0 (or '+1', '-1', '0'), got {value!r}")
-        return self._tok2id[mapping[value]]
-
-    def env_token_ids(self) -> Dict[str, int]:
-        """Get mapping of all env/reward special tokens to their IDs."""
-        self._require_env_tokens()
-        return {tok: self._tok2id[tok] for tok in ENV_TOKENS}
-    
-    def extract_parts(self, text: str) -> Tuple[Optional[str], Optional[List[str]], str]:
-        """
-        Extract prompt, trajectories and answer from BoN CoT formatted text.
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            prompt: The prompt/context (or None if not present)
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        if self.T not in text:
-            return None, None, text
-        
-        try:
-            # Split by <T> to get prompt, thinking section, and answer
-            t_parts = text.split(self.T)
-            if len(t_parts) < 3:
-                return None, None, text
-            
-            # t_parts[0] is prompt (before first <T>)
-            # t_parts[1] is the thinking section with trajectories
-            # t_parts[2] is the answer
-            prompt = t_parts[0].strip() if t_parts[0].strip() else None
-            thinking_section = t_parts[1].strip()
-            answer = t_parts[2].strip()
-            
-            # Split thinking section by <sep> to get trajectories
-            trajectories = [t.strip() for t in thinking_section.split(self.SEP) if t.strip()]
-            
-            return prompt, trajectories, answer
-        except (ValueError, IndexError):
-            return None, None, text
-    
-    def extract_thinking_and_answer(self, text: str) -> Tuple[Optional[List[str]], str]:
-        """
-        Extract trajectories and answer from BoN CoT formatted text (ignores prompt).
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        _, trajectories, answer = self.extract_parts(text)
-        return trajectories, answer
-    
-    def get_sft_special_tokens(self) -> List[str]:
-        """Get list of SFT special tokens (including env/reward tokens if enabled)."""
-        toks = [self.T, self.T_END, self.SEP]
-        if self._include_env_tokens:
-            toks += ENV_TOKENS
-        return toks
-
-    def get_sft_token_ids(self) -> Dict[str, int]:
-        """Get mapping of SFT special tokens to their IDs."""
-        result = {
-            self.T: self._tok2id[self.T],
-            self.T_END: self._tok2id[self.T_END],
-            self.SEP: self._tok2id[self.SEP],
-        }
-        if self._include_env_tokens:
-            for tok in ENV_TOKENS:
-                result[tok] = self._tok2id[tok]
-        return result
-    
-    def parse_cot_line(self, line: str) -> Tuple[Optional[List[str]], Optional[str]]:
-        """
-        Parse a CoT data line in format: <T> <sep> ... <sep> <T> {answer}
-        
-        Args:
-            line: A line from the CoT data file
-        
-        Returns:
-            trajectories: List of trajectory strings
-            answer: The final answer/move
-        """
-        line = line.strip()
-        if not line or not line.startswith(self.T):
-            return None, None
-        
-        return self.extract_thinking_and_answer(line)
-
-# ============================================================
-# HuggingFace-compatible wrapper (auto-generated)
-# ============================================================
-import json as _json
-from pathlib import Path as _Path
-from transformers import PreTrainedTokenizer
-import torch
-from transformers.tokenization_utils_base import BatchEncoding
-
-from huggingface_hub import hf_hub_download
-
-class HFTokenizerWrapper(PreTrainedTokenizer):
-    def __init__(self, model_max_length=2048, **kwargs):
-        # These are usually provided by from_pretrained
-        repo_id = kwargs.get("name_or_path") or kwargs.get("_name_or_path")
-        revision = kwargs.get("revision", None)
-
-        if not repo_id or "/" not in str(repo_id):
-            # Fallback: user may pass repo_id explicitly
-            repo_id = kwargs.get("repo_id", None)
-        if not repo_id:
-            raise ValueError("Cannot infer repo_id; pass repo_id=... or ensure name_or_path is set.")
-
-        import os
-        if os.path.isdir(repo_id):
-            vocab_path = os.path.join(repo_id, "vocab.json")
-            cfg_path   = os.path.join(repo_id, "tokenizer_config.json")
-        else:
-            vocab_path = hf_hub_download(repo_id=repo_id, filename="vocab.json", revision=revision)
-            cfg_path   = hf_hub_download(repo_id=repo_id, filename="tokenizer_config.json", revision=revision)
-
-        with open(vocab_path, "r", encoding="utf-8") as _f:
-            saved_vocab = _json.load(_f)
-        with open(cfg_path, "r", encoding="utf-8") as _f:
-            _tok_cfg = _json.load(_f)
-
-        lan_config = _tok_cfg.get("lan_config", {})
-        lan_class_name = _tok_cfg.get("lan_tokenizer_class", "LanTokenizerSFT")
-
-        _cls = globals()[lan_class_name]
-        custom_tokenizer = _cls(config=lan_config)
-
-        # Override vocab with the saved vocab
-        custom_tokenizer._tok2id = saved_vocab
-        from tokenizers import Tokenizer as _TkTokenizer
-        from tokenizers.models import WordLevel as _WordLevel
-        from tokenizers.pre_tokenizers import WhitespaceSplit as _WhitespaceSplit
-        custom_tokenizer.tk = _TkTokenizer(_WordLevel(vocab=saved_vocab, unk_token=custom_tokenizer._unk))
-        custom_tokenizer.tk.pre_tokenizer = _WhitespaceSplit()
-
-        self.custom_tokenizer = custom_tokenizer
-        self._vocab = dict(saved_vocab)
-        self._id_to_token = {i: t for t, i in self._vocab.items()}
-
-        bos_token = _tok_cfg.get("bos_token")
-        eos_token = _tok_cfg.get("eos_token")
-        pad_token = _tok_cfg.get("pad_token")
-        unk_token = _tok_cfg.get("unk_token")
-        env_token = _tok_cfg.get("env_token")
-        if "env_id" in _tok_cfg:
-            env_token = self._id_to_token[_tok_cfg.get("env_id")]
-        else:
-            env_token = _tok_cfg.get("env_token")
-        self.env_token = env_token
-
-        for _key in ("bos_token","eos_token","pad_token","unk_token","env_token",
-                     "model_max_length","name_or_path","lan_config",
-                     "lan_tokenizer_class","tokenizer_class","auto_map","use_fast",
-                     "revision","repo_id"):
-            kwargs.pop(_key, None)
-
-        super().__init__(
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            unk_token=unk_token,
-            model_max_length=model_max_length,
-            **kwargs,
-        )
-
-    # ---- PreTrainedTokenizer interface ----
-
-    @property
-    def vocab_size(self):
-        return len(self._vocab)
-
-    def get_vocab(self):
-        return dict(self._vocab)
-
-    def _tokenize(self, text):
-        return []  # we override encode/decode directly
-
-    def _convert_token_to_id(self, token):
-        return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
-
-    def _convert_id_to_token(self, index):
-        return self._id_to_token.get(index, self.unk_token or "")
-
-    def convert_tokens_to_string(self, tokens):
-        ids = [self._convert_token_to_id(t) for t in tokens]
-        return self.custom_tokenizer.decode(ids)
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        if token_ids_1 is None:
-            return token_ids_0
-        return token_ids_0 + token_ids_1
-
-    def encode(self, text, add_special_tokens=True, **kwargs):
-        ids = self.custom_tokenizer.encode(text)
-        if add_special_tokens:
-            return ids[:-1]  # strip trailing EOS; vLLM adds its own
-        if (len(ids) >= 2
-                and self.bos_token_id is not None
-                and self.eos_token_id is not None
-                and ids[0] == self.bos_token_id
-                and ids[-1] == self.eos_token_id):
-            return ids[1:-1]
-        return ids
-
-    def decode(self, token_ids, skip_special_tokens=True, **kwargs):
-        import numpy as np
-        if isinstance(token_ids, torch.Tensor):
-            token_ids = token_ids.detach().cpu().tolist()
-        elif isinstance(token_ids, np.ndarray):
-            token_ids = token_ids.tolist()
-        return self.custom_tokenizer.decode(token_ids)
-
-    def save_vocabulary(self, save_directory, filename_prefix=None):
-        save_directory = _Path(save_directory)
-        save_directory.mkdir(parents=True, exist_ok=True)
-        vocab_file = save_directory / (
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json"
-        )
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            _json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (str(vocab_file),)
-
-    def __call__(
-        self,
-        text,
-        text_pair=None,
-        add_special_tokens=True,
-        truncation=False,
-        max_length=None,
-        padding=False,
-        return_tensors=None,
-        **kwargs,
-    ):
-        if text_pair is not None:
-            raise ValueError("text_pair not supported for this tokenizer.")
-
-        # Normalize to batch
-        is_batched = isinstance(text, (list, tuple))
-        texts = list(text) if is_batched else [text]
-
-        input_ids = [self.encode(t, add_special_tokens=add_special_tokens) for t in texts]
-
-        # Truncation
-        if truncation and max_length is not None:
-            if self.truncation_side == "left":
-                input_ids = [ids[-max_length:] for ids in input_ids]
-            else:
-                input_ids = [ids[:max_length] for ids in input_ids]
-
-        # Attention masks (pre-padding)
-        attention_mask = [[1] * len(ids) for ids in input_ids]
-
-        # Padding
-        if padding:
-            if padding == "max_length":
-                if max_length is None:
-                    raise ValueError("padding='max_length' requires max_length.")
-                pad_to = max_length
-            else:
-                pad_to = max(len(ids) for ids in input_ids) if input_ids else 0
-
-            pad_id = self.pad_token_id
-            if pad_id is None:
-                pad_id = self.bos_token_id if self.bos_token_id is not None else 0
-
-            for i, ids in enumerate(input_ids):
-                pad_len = pad_to - len(ids)
-                if pad_len > 0:
-                    input_ids[i] = ids + [pad_id] * pad_len
-                    attention_mask[i] = attention_mask[i] + [0] * pad_len
-
-        data = {"input_ids": input_ids, "attention_mask": attention_mask}
-
-        # Unbatch if single example and no tensor return
-        if not is_batched and return_tensors is None:
-            data = {"input_ids": data["input_ids"][0], "attention_mask": data["attention_mask"][0]}
-
-        # Tensors
-        if return_tensors == "pt":
-            data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()}
-
-        return BatchEncoding(data, tensor_type=None)
-
-
-__all__ = ["HFTokenizerWrapper"]

C6p5e18_410m_alpha0.050_beta0.045/step_110/tokenizer_config.json

@@ -1,26 +0,0 @@
-{
-  "tokenizer_class": "HFTokenizerWrapper",
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenizer.HFTokenizerWrapper",
-      null
-    ]
-  },
-  "model_max_length": 2048,
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null,
-  "use_fast": false,
-  "lan_config": {
-    "name": "LanTokenizerSFT",
-    "include_move_numbers": false,
-    "include_black_tripledots": false,
-    "bos": "<bos>",
-    "eos": "<eos>",
-    "unk": "<unk>",
-    "keep_result": false
-  },
-  "lan_tokenizer_class": "LanTokenizerSFT"
-}

C6p5e18_410m_alpha0.050_beta0.045/step_110/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.050_beta0.045/step_110/vocab.json

@@ -1,86 +0,0 @@
-{
-  "<bos>": 0,
-  "<eos>": 1,
-  "<unk>": 2,
-  "K": 3,
-  "Q": 4,
-  "R": 5,
-  "B": 6,
-  "N": 7,
-  "P": 8,
-  "a1": 9,
-  "a2": 10,
-  "a3": 11,
-  "a4": 12,
-  "a5": 13,
-  "a6": 14,
-  "a7": 15,
-  "a8": 16,
-  "b1": 17,
-  "b2": 18,
-  "b3": 19,
-  "b4": 20,
-  "b5": 21,
-  "b6": 22,
-  "b7": 23,
-  "b8": 24,
-  "c1": 25,
-  "c2": 26,
-  "c3": 27,
-  "c4": 28,
-  "c5": 29,
-  "c6": 30,
-  "c7": 31,
-  "c8": 32,
-  "d1": 33,
-  "d2": 34,
-  "d3": 35,
-  "d4": 36,
-  "d5": 37,
-  "d6": 38,
-  "d7": 39,
-  "d8": 40,
-  "e1": 41,
-  "e2": 42,
-  "e3": 43,
-  "e4": 44,
-  "e5": 45,
-  "e6": 46,
-  "e7": 47,
-  "e8": 48,
-  "f1": 49,
-  "f2": 50,
-  "f3": 51,
-  "f4": 52,
-  "f5": 53,
-  "f6": 54,
-  "f7": 55,
-  "f8": 56,
-  "g1": 57,
-  "g2": 58,
-  "g3": 59,
-  "g4": 60,
-  "g5": 61,
-  "g6": 62,
-  "g7": 63,
-  "g8": 64,
-  "h1": 65,
-  "h2": 66,
-  "h3": 67,
-  "h4": 68,
-  "h5": 69,
-  "h6": 70,
-  "h7": 71,
-  "h8": 72,
-  "x": 73,
-  "=": 74,
-  "+": 75,
-  "#": 76,
-  "O-O": 77,
-  "O-O-O": 78,
-  ".": 79,
-  "...": 80,
-  "<T>": 81,
-  "</T>": 82,
-  "<sep>": 83
-}

C6p5e18_410m_alpha0.200_beta0.045/config_snapshot/sft_config_multi_path_2048.yaml

@@ -1,100 +0,0 @@
-# SFT Training Configuration Example
-#
-# This configuration file is for Supervised Fine-Tuning (SFT) on a pretrained model.
-# It uses chat format with thinking tokens for chain-of-thought reasoning.
-
-model:
-  pretrained_model: "/scratch/js15262/LLM-Pretraining/checkpoints/chess_gpt2_pretrained_medium_8B_tokens/hf_Qwen3_0.6B_LanTokenizer_data7025_LNAHNAENA_bs4_lr0.001_warmup1300_shards/final_step6609/hf_model"  
-  block_size: 2048 
-
-tokenizer:
-  name: "LanTokenizerSFT"  # Use SFT-enabled tokenizer
-  include_move_numbers: false
-  include_black_tripledots: false
-  bos: "<bos>"
-  eos: "<eos>"
-  unk: "<unk>"
-  keep_result: false
-
-data:
-  # Path to SFT training data (JSONL format)
-  # Each line should be: {"prompt": "...", "response": "...", "system": "..."}
-  # Can be a folder path - will find all .json and .jsonl files inside
-  data_name: "cot_data_qwen_puzzles_no_verify_single_move_with_human"
-  train_files:
-      # - "/scratch/js15262/LLM-Pretraining/data/sft/cot_data_puzzle_seq_qwen/hf_model_seed0_evalminimax_candpolicy_nc8_nt10_nta8_d20_dmrandom_wt300_pm5_inj"
-      # - "/scratch/js15262/LLM-Pretraining/data/sft/cot_data_puzzle_seq_qwen/hf_model_seed1_evalminimax_candpolicy_nc8_nt10_nta8_d20_dmrandom_wt300_pm5_inj"
-    # - "/scratch/js15262/LLM-Pretraining/data/sft/cot_data_puzzle_seq_qwen/hf_model_seed2_evalminimax_candpolicy_nc10_nt10_nta10_d20_dmrandom_wt300_pm5_inj"
-    # - "/scratch/js15262/LLM-Pretraining/data/sft/cot_data_qwen/hf_model_seed2_evalstockfish_direct_max_candpolicy_nc10_nt5_d5_temp1.0_sf0.1"
-    # - "/scratch/js15262/LLM-Pretraining/data/sft/cot_data_qwen/hf_model_seed3_evalstockfish_direct_max_candpolicy_nc10_nt5_d5_temp1.0_sf0.1"
-  
-  # Hold out last N files from train_files for evaluation (like pretrain config)
-  eval_holdout: 2
-
-  
-  # Optional: additional evaluation data (can be empty if using eval_holdout)
-  # eval_files:
-  #   - "/scratch/js15262/LLM-Pretraining/data/chess/cot_data/some_specific_file.json"
-  
-  # SFT-specific settings
-  sft:
-    mask_prompt: true              # Mask prompt tokens in loss (only train on response)
-    cot_field: "cot_by_method.trajectory_sep.cot_format_first_move_no_labels"
-
-training:
-  epochs: 4
-  batch_size: 4                    # Smaller batch size for SFT (longer sequences)
-  eval_batch_size: 2
-  
-  # Optimizer settings
-  optimizer:
-    name: "adamw"
-    lr: 5.0e-5                     # Lower learning rate for fine-tuning
-    weight_decay: 0.01
-    betas: [0.9, 0.95]
-  
-  # Scheduler settings
-  scheduler:
-    name: "cosine"
-    warmup_steps: 50              # Fewer warmup steps for fine-tuning
-    eta_min: 1.0e-5
-  
-  # Training optimizations
-  gradient_accumulation_steps: 32  # Effective batch size = 8 * 4 = 32
-  max_grad_norm: 1.0               # Gradient clipping
-  
-  # Regularization
-  label_smoothing: 0.0             # Optional: label smoothing (0.0 = disabled)
-  
-  # Data loading
-  num_workers: 4
-  prefetch_factor: 2
-  persistent_workers: true
-  
-  # Logging and checkpointing
-  save_dir: "checkpoints/sft"
-  save_interval: 100               # Save checkpoint every N steps
-  log_interval: 10                 # Log every N steps
-  eval_interval: 50               # Evaluate every N steps
-  eval_max_steps: 50               # Number of eval batches
-  
-  # Model compilation (PyTorch 2.0+)
-  # compile_model: false             # Set to true for additional speedup
-  # compile_mode: "reduce-overhead"
-  
-  # Load pretrained weights
-  pretrained_weights: "/scratch/js15262/LLM-Pretraining/checkpoints/chess_gpt2_pretrained_medium_8B_tokens/hf_Qwen3_0.6B_LanTokenizer_data7025_LNAHNAENA_bs4_lr0.001_warmup1300_shards/final_step6609/model.safetensors"
-  
-  # Resume from checkpoint (optional)
-  # resume: "checkpoints/sft/run_name/step1000"
-
-logging:
-  backend: "wandb"
-  project: "chess-llm-sft-qwen-200m-single-turn"
-  entity: "jingyanshen-new-york-university"                     # Your wandb entity/username
-  notes: "SFT with thinking tokens"
-  tags:
-    - "sft"
-    - "thinking"
-    - "chess"
-

C6p5e18_410m_alpha0.200_beta0.045/final/config.json

@@ -1,64 +0,0 @@
-{
-  "architectures": [
-    "Qwen3ForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "dtype": "bfloat16",
-  "eos_token_id": 1,
-  "head_dim": 128,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_types": [
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention"
-  ],
-  "max_position_embeddings": 2048,
-  "max_window_layers": 28,
-  "model_type": "qwen3",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 4,
-  "pad_token_id": 0,
-  "rms_norm_eps": 1e-06,
-  "rope_parameters": {
-    "factor": 2.0,
-    "original_max_position_embeddings": 1024,
-    "type": "yarn"
-  },
-  "sliding_window": null,
-  "tie_word_embeddings": true,
-  "transformers_version": "5.3.0",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 84
-}

C6p5e18_410m_alpha0.200_beta0.045/final/generation_config.json

@@ -1,9 +0,0 @@
-{
-  "bos_token_id": 0,
-  "do_sample": true,
-  "eos_token_id": 1,
-  "max_new_tokens": 1024,
-  "temperature": 1.0,
-  "top_k": 50,
-  "transformers_version": "5.3.0"
-}

C6p5e18_410m_alpha0.200_beta0.045/final/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a864bb33decacb0fd02126c0c47b7576a3b5129830ef6b089487f009f7511efb
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/final/optimizer_states/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a864bb33decacb0fd02126c0c47b7576a3b5129830ef6b089487f009f7511efb
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/final/optimizer_states/optimizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aac3284ed0372ec7cf556121eed9f601f73f2064d672d56ac1800f531ffe64b7
-size 1645034059

C6p5e18_410m_alpha0.200_beta0.045/final/optimizer_states/random_states_0.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:85f3b361e9f4b1b254befe3769b50c204af81c5009a350ec4ce1e8e0d577d63d
-size 15081

C6p5e18_410m_alpha0.200_beta0.045/final/optimizer_states/scheduler.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0183971c5d0a20e92ba8c8db76425ea86478f65d9a0e5fe494f3c85f26cc9c20
-size 1465

C6p5e18_410m_alpha0.200_beta0.045/final/optimizer_states/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.200_beta0.045/final/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null
-}

C6p5e18_410m_alpha0.200_beta0.045/final/tokenizer.py

@@ -1,818 +0,0 @@
-"""
-Auto-generated self-contained HF tokenizer.
-Do NOT edit manually -- regenerate via training.hf_tokenizer_utils.save_hf_tokenizer().
-"""
-from __future__ import annotations
-
-# --- BaseTokenizer (inlined) ---
-# base_tokenizer.py
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
-
-class BaseTokenizer(ABC):
-    """Minimal interface for tokenizers used in pretraining."""
-
-    # ---- required ----
-    @abstractmethod
-    def encode(self, text: str) -> List[int]:
-        """Convert text/PGN to token IDs."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def decode(self, ids: List[int]) -> str:
-        """Convert token IDs back to text/PGN."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def get_vocab(self) -> Dict[str, int]:
-        """Return token -> id mapping (if available)."""
-        raise NotImplementedError
-
-    def bos_id(self) -> Optional[int]: return None
-    def eos_id(self) -> Optional[int]: return None
-    def pad_id(self) -> Optional[int]: return None
-    def get_vocab_size(self) -> int: return len(self.get_vocab())
-
-    def __call__(self, text: str) -> List[int]:
-        """Alias for encode()."""
-        return self.encode(text)
-
-# --- Concrete tokenizer (inlined) ---
-# lan_tokenizer_sft.py
-"""
-LAN Tokenizer with SFT support (CoT format with <T> and <sep> tokens).
-
-This extends the base LAN tokenizer with SFT-specific functionality:
-- <T> token for marking thinking/CoT content
-- <sep> token for separating prompt from response
-"""
-from typing import List, Dict, Optional, Tuple
-import io
-import chess, chess.pgn
-from tokenizers import Tokenizer
-from tokenizers.models import WordLevel
-from tokenizers.pre_tokenizers import WhitespaceSplit
-_RESULT = {"1-0", "0-1", "1/2-1/2", "*"}
-FILES = "abcdefgh"
-RANKS = "12345678"
-SQUARES = [f+r for f in FILES for r in RANKS]
-PROMOS = "QRBN"
-DIGITS = set("0123456789")
-
-# SFT special tokens for CoT format
-T_TOKEN = "<T>"
-T_END_TOKEN = "</T>"
-SEP_TOKEN = "<sep>"
-
-# Environment interaction / reward special tokens
-CALL_ENV_TOKEN = "<call_env>"
-VERIFY_TOKEN = "<verify>"
-REWARD_POS_TOKEN = "<+1>"
-REWARD_NEG_TOKEN = "<-1>"
-REWARD_ZERO_TOKEN = "<0>"
-ENV_TOKENS = [CALL_ENV_TOKEN]
-REWARD_TOKENS = [VERIFY_TOKEN, REWARD_POS_TOKEN, REWARD_NEG_TOKEN, REWARD_ZERO_TOKEN]
-
-def _vocab_with_sft(
-    include_move_numbers: bool,
-    keep_result: bool,
-    bos: str,
-    eos: str,
-    unk: str,
-    include_env_tokens: bool = False,
-    include_reward_tokens: bool = False,
-) -> Dict[str, int]:
-    """Create vocabulary including SFT special tokens."""
-    base = [bos, eos, unk]
-    ops = ["x", "=", "+", "#", "O-O", "O-O-O", ".", "..."]
-    toks = base + list("KQRBNP") + SQUARES + list(PROMOS) + ops
-    if include_move_numbers:
-        toks += list("0123456789")
-    if keep_result:
-        toks += sorted(_RESULT)
-
-    # Add SFT special tokens for CoT format
-    sft_tokens = [T_TOKEN, T_END_TOKEN, SEP_TOKEN]
-    toks += sft_tokens
-
-    # Add environment / reward tokens when requested
-    if include_env_tokens:
-        toks += ENV_TOKENS
-    if include_reward_tokens:
-        toks += REWARD_TOKENS
-
-    return {t: i for i, t in enumerate(dict.fromkeys(toks))}
-
-
-class LanTokenizerSFT(BaseTokenizer):
-    """
-    LAN Tokenizer with SFT capabilities.
-
-    This tokenizer extends the base LAN tokenizer with:
-    - <T> token for marking thinking/CoT boundaries
-    - <sep> token for separating candidate trajectories
-
-    CoT Format: {prompt} <T> <sep> {traj1} <sep> {traj2} <sep> ... <sep> {trajN} <sep> <T> {answer}
-
-    Where:
-    - {prompt}: The game history/board state (PGN moves)
-    - {trajN}: Candidate reasoning trajectories
-    - {answer}: The final best move
-    """
-
-    # Special tokens for CoT format
-    T = T_TOKEN
-    T_END = T_END_TOKEN
-    SEP = SEP_TOKEN
-
-    # Environment / reward tokens (class-level constants for easy access)
-    CALL_ENV = CALL_ENV_TOKEN   # "<call_env>"
-    VERIFY = VERIFY_TOKEN       # "<verify>"
-    REWARD_POS = REWARD_POS_TOKEN  # "<+1>"
-    REWARD_NEG = REWARD_NEG_TOKEN  # "<-1>"
-    REWARD_ZERO = REWARD_ZERO_TOKEN  # "<0>"
-    ENV_TOKENS = ENV_TOKENS     # full list
-
-    def __init__(self, config: Optional[dict] = None):
-        """
-        Args:
-            config: Configuration dict with tokenizer settings.
-                include_env_tokens (bool): add <call_env>, <verify>, <+1>, <-1>, <0>
-                    to the vocabulary.  Default: False.
-        """
-        config = config or {}
-
-        include_move_numbers = config.get("include_move_numbers", False)
-        include_black_tripledots = config.get("include_black_tripledots", False)
-        bos = config.get("bos", "<bos>")
-        eos = config.get("eos", "<eos>")
-        unk = config.get("unk", "<unk>")
-        keep_result = config.get("keep_result", False)
-        include_env_tokens = config.get("include_env_tokens", False)
-        include_reward_tokens = config.get("include_reward_tokens", False)
-
-        self._bos = bos
-        self._eos = eos
-        self._unk = unk
-        self._keep_res = keep_result
-        self._include_nums = include_move_numbers
-        self._include_black_ellipses = include_black_tripledots
-        self._include_env_tokens = include_env_tokens
-        self._include_reward_tokens = include_reward_tokens
-
-        # Create vocabulary with SFT tokens
-        tok2id = _vocab_with_sft(
-            include_move_numbers, keep_result, bos, eos, unk,
-            include_env_tokens=include_env_tokens,
-            include_reward_tokens=include_reward_tokens,
-        )
-        self._tok2id = tok2id
-
-        # Initialize tokenizer
-        self.tk = Tokenizer(WordLevel(vocab=tok2id, unk_token=self._unk))
-        self.tk.pre_tokenizer = WhitespaceSplit()
-    
-    def _pgn_to_tokens(self, text: str) -> Optional[List[str]]:
-        """Convert PGN text to tokens."""
-        import os, contextlib
-        with open(os.devnull, "w") as devnull, contextlib.redirect_stderr(devnull):
-            g = chess.pgn.read_game(io.StringIO(text))
-        if g is None:
-            return None
-        
-        b, out, n = g.board(), [], 1
-        for mv in g.mainline_moves():
-            if b.turn == chess.WHITE and self._include_nums:
-                out += list(str(n)) + (
-                    ["..."] if self._include_black_ellipses and b.fullmove_number < n else ["."]
-                )
-            
-            if b.is_castling(mv):
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                b.pop()
-                out.append("O-O" if chess.square_file(mv.to_square) == 6 else "O-O-O")
-                if suf:
-                    out.append(suf)
-                b.push(mv)
-            else:
-                piece = b.piece_at(mv.from_square).symbol().upper()
-                frm = chess.square_name(mv.from_square)
-                to = chess.square_name(mv.to_square)
-                is_cap = b.is_capture(mv)
-                promo = mv.promotion
-                
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                
-                # Emit LAN tokens
-                out.append(piece)
-                out.append(frm)
-                if is_cap:
-                    out.append("x")
-                out.append(to)
-                if promo:
-                    out += ["=", chess.piece_symbol(promo).upper()]
-                if suf:
-                    out.append(suf)
-            
-            if b.turn == chess.WHITE:
-                n += 1
-        
-        res = g.headers.get("Result")
-        if self._keep_res and res in _RESULT:
-            out.append(res)
-        
-        return out
-    
-    def _lan_move_to_tokens(self, move: str) -> List[str]:
-        """
-        Convert a single LAN move to tokens.
-        
-        LAN format: [Piece][from_square][x]?[to_square][=Promo]?[+#]?
-        
-        Examples:
-            "Ng1f3" -> ["N", "g1", "f3"]
-            "Nd4xe6" -> ["N", "d4", "x", "e6"]
-            "Pe2e4" -> ["P", "e2", "e4"]
-            "Pe4xd5" -> ["P", "e4", "x", "d5"]
-            "O-O" -> ["O-O"]
-            "O-O-O" -> ["O-O-O"]
-            "Pe7e8=Q" -> ["P", "e7", "e8", "=", "Q"]
-            "Ng1f3+" -> ["N", "g1", "f3", "+"]
-        """
-        # Handle castling
-        if move in {"O-O", "O-O-O"}:
-            return [move]
-        if move.rstrip("+#") in {"O-O", "O-O-O"}:
-            base = move.rstrip("+#")
-            suffix = move[len(base):]
-            return [base] + ([suffix] if suffix else [])
-        
-        out = []
-        i = 0
-        n = len(move)
-        
-        # Get piece letter (required in LAN format)
-        if i < n and move[i] in "KQRBNP":
-            out.append(move[i])
-            i += 1
-        else:
-            # No piece letter - might be malformed, return as-is
-            return [move]
-        
-        # Get from square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle capture
-        if i < n and move[i] == "x":
-            out.append("x")
-            i += 1
-        
-        # Get to square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle promotion
-        if i < n and move[i] == "=":
-            out.append("=")
-            i += 1
-            if i < n and move[i] in PROMOS:
-                out.append(move[i])
-                i += 1
-        
-        # Handle check/checkmate
-        if i < n and move[i] in "+#":
-            out.append(move[i])
-            i += 1
-        
-        return out
-    
-    def _active_env_tokens(self) -> set:
-        """Return the set of env tokens that are active for this instance."""
-        return set(ENV_TOKENS) if self._include_env_tokens else set()
-
-    def _cot_to_tokens(self, text: str) -> List[str]:
-        """
-        Convert CoT formatted text to tokens.
-        Handles special tokens and LAN moves.
-        """
-        env_toks = self._active_env_tokens()
-        out = []
-        for token in text.split():
-            if token in {self.T, self.T_END, self.SEP} or token in env_toks:
-                # Keep special tokens as-is
-                out.append(token)
-            elif token in _RESULT:
-                # Game result
-                out.append(token)
-            elif token and token[0].isdigit() and "." in token:
-                # Move number like "1." or "15..."
-                # Split into digits and dots
-                num_part = token.rstrip(".")
-                dot_part = token[len(num_part):]
-                out.extend(list(num_part))
-                if dot_part:
-                    out.append("..." if len(dot_part) > 1 else ".")
-            elif token and all(c.isdigit() for c in token):
-                # Pure number - tokenize each digit
-                out.extend(list(token))
-            else:
-                # LAN move - tokenize it
-                out.extend(self._lan_move_to_tokens(token))
-        return out
-    
-    def encode(self, text: str) -> List[int]:
-        """
-        Encode text to token IDs.
-        
-        Args:
-            text: Text to encode (can be PGN or CoT formatted)
-        
-        Returns:
-            List of token IDs
-        """
-        # Check if this is CoT-formatted text (contains special tokens)
-        sft_special = (
-            [self.T, self.T_END, self.SEP]
-            + (ENV_TOKENS if self._include_env_tokens else [])
-        )
-        is_cot_format = any(token in text for token in sft_special)
-        
-        if is_cot_format:
-            t_idx = text.index(self.T)
-            prompt_part = text[:t_idx].strip()
-            rest_part = text[t_idx:]  # starts with <T>
-
-            pgn_tokens = self._pgn_to_tokens(prompt_part) if prompt_part else None
-            if pgn_tokens is None:
-                pgn_tokens = self._cot_to_tokens(prompt_part) if prompt_part else []
-            rest_tokens = self._cot_to_tokens(rest_part)
-            tokens = [self._bos] + pgn_tokens + rest_tokens + [self._eos]
-        else:
-            pgn_tokens = self._pgn_to_tokens(text)
-            if pgn_tokens is not None and len(pgn_tokens) > 0:
-                tokens = [self._bos] + pgn_tokens + [self._eos]
-            else:
-                # Not valid PGN — treat each word as a LAN move
-                lan_tokens = []
-                for word in text.split():
-                    lan_tokens.extend(self._lan_move_to_tokens(word))
-                tokens = [self._bos] + lan_tokens + [self._eos]
-        
-        return self.tk.encode(" ".join(tokens)).ids
-    
-    def decode(self, ids: List[int]) -> str:
-        """
-        Decode token IDs to text.
-        
-        Args:
-            ids: List of token IDs
-        
-        Returns:
-            Decoded text
-        """
-        toks = [t for t in self.tk.decode(ids).split() if t not in {self._bos, self._eos}]
-        
-        # Otherwise, use LAN decoding logic
-        out: List[str] = []
-        i, n = 0, len(toks)
-        
-        while i < n:
-            t = toks[i]
-            
-            if t in {self.T, self.T_END, self.SEP} or t in _RESULT or t in self._active_env_tokens():
-                out.append(t)
-                i += 1
-                continue
-            
-            if t and all(ch in DIGITS for ch in t):
-                j = i
-                num = []
-                while j < n and all(ch in DIGITS for ch in toks[j]):
-                    num.append(toks[j])
-                    j += 1
-                dots = ""
-                if j < n and toks[j] in {".", "..."}:
-                    dots = toks[j]
-                    j += 1
-                out.append("".join(num) + dots)
-                i = j
-                continue
-            
-            if t in {"O-O", "O-O-O"}:
-                j = i + 1
-                suf = toks[j] if j < n and toks[j] in {"+", "#"} else ""
-                if suf:
-                    j += 1
-                out.append(t + suf)
-                i = j
-                continue
-            
-            if t in set("KQRBNP"):
-                piece = t
-                j = i + 1
-                frm = toks[j] if j < n else ""
-                j += 1
-                cap = ""
-                if j < n and toks[j] == "x":
-                    cap = "x"
-                    j += 1
-                to = toks[j] if j < n else ""
-                j += 1
-                promo = ""
-                if j + 1 <= n - 1 and toks[j] == "=" and toks[j + 1] in set(PROMOS):
-                    promo = "=" + toks[j + 1]
-                    j += 2
-                suf = ""
-                if j < n and toks[j] in {"+", "#"}:
-                    suf = toks[j]
-                    j += 1
-                lan = f"{piece}{frm}{cap}{to}{promo}{suf}"
-                out.append(lan)
-                i = j
-                continue
-            
-            out.append(t)
-            i += 1
-        
-        return " ".join(out)
-    
-    def get_vocab(self) -> Dict[str, int]:
-        """Get token-to-id vocabulary mapping."""
-        return self._tok2id
-    
-    def bos_id(self) -> Optional[int]:
-        """Get BOS token ID."""
-        return self._tok2id[self._bos]
-    
-    def eos_id(self) -> Optional[int]:
-        """Get EOS token ID."""
-        return self._tok2id[self._eos]
-    
-    def pad_id(self) -> Optional[int]:
-        """Get PAD token ID (uses BOS as pad by default)."""
-        return self._tok2id.get("<pad>", self.bos_id())
-    
-    def get_vocab_size(self) -> int:
-        """Get vocabulary size."""
-        return len(self._tok2id)
-    
-    def t_id(self) -> int:
-        """Get <T> token ID."""
-        return self._tok2id[self.T]
-    
-    def sep_id(self) -> int:
-        """Get <sep> token ID."""
-        return self._tok2id[self.SEP]
-    
-    def t_end_id(self) -> int:
-        """Get </T> token ID."""
-        return self._tok2id[self.T_END]
-    
-    # ------------------------------------------------------------------
-    # Environment / reward token accessors
-    # ------------------------------------------------------------------
-
-    def _require_env_tokens(self) -> None:
-        if not self._include_env_tokens:
-            raise ValueError(
-                "Environment tokens are not enabled. "
-                "Pass include_env_tokens=True in the config."
-            )
-
-    def call_env_id(self) -> int:
-        """Get <call_env> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[CALL_ENV_TOKEN]
-
-    def verify_id(self) -> int:
-        """Get <verify> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[VERIFY_TOKEN]
-
-    def reward_pos_id(self) -> int:
-        """Get <+1> (positive reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_POS_TOKEN]
-
-    def reward_neg_id(self) -> int:
-        """Get <-1> (negative reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_NEG_TOKEN]
-
-    def reward_zero_id(self) -> int:
-        """Get <0> (zero reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_ZERO_TOKEN]
-
-    def reward_id(self, value) -> int:
-        """
-        Get reward token ID by numeric value.
-
-        Args:
-            value: 1, -1, or 0  (or the strings "+1", "-1", "0")
-
-        Returns:
-            Token ID for the corresponding reward token.
-        """
-        self._require_env_tokens()
-        mapping = {1: REWARD_POS_TOKEN, -1: REWARD_NEG_TOKEN, 0: REWARD_ZERO_TOKEN,
-                   "+1": REWARD_POS_TOKEN, "-1": REWARD_NEG_TOKEN, "0": REWARD_ZERO_TOKEN}
-        if value not in mapping:
-            raise ValueError(f"reward value must be one of 1, -1, 0 (or '+1', '-1', '0'), got {value!r}")
-        return self._tok2id[mapping[value]]
-
-    def env_token_ids(self) -> Dict[str, int]:
-        """Get mapping of all env/reward special tokens to their IDs."""
-        self._require_env_tokens()
-        return {tok: self._tok2id[tok] for tok in ENV_TOKENS}
-    
-    def extract_parts(self, text: str) -> Tuple[Optional[str], Optional[List[str]], str]:
-        """
-        Extract prompt, trajectories and answer from BoN CoT formatted text.
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            prompt: The prompt/context (or None if not present)
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        if self.T not in text:
-            return None, None, text
-        
-        try:
-            # Split by <T> to get prompt, thinking section, and answer
-            t_parts = text.split(self.T)
-            if len(t_parts) < 3:
-                return None, None, text
-            
-            # t_parts[0] is prompt (before first <T>)
-            # t_parts[1] is the thinking section with trajectories
-            # t_parts[2] is the answer
-            prompt = t_parts[0].strip() if t_parts[0].strip() else None
-            thinking_section = t_parts[1].strip()
-            answer = t_parts[2].strip()
-            
-            # Split thinking section by <sep> to get trajectories
-            trajectories = [t.strip() for t in thinking_section.split(self.SEP) if t.strip()]
-            
-            return prompt, trajectories, answer
-        except (ValueError, IndexError):
-            return None, None, text
-    
-    def extract_thinking_and_answer(self, text: str) -> Tuple[Optional[List[str]], str]:
-        """
-        Extract trajectories and answer from BoN CoT formatted text (ignores prompt).
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        _, trajectories, answer = self.extract_parts(text)
-        return trajectories, answer
-    
-    def get_sft_special_tokens(self) -> List[str]:
-        """Get list of SFT special tokens (including env/reward tokens if enabled)."""
-        toks = [self.T, self.T_END, self.SEP]
-        if self._include_env_tokens:
-            toks += ENV_TOKENS
-        return toks
-
-    def get_sft_token_ids(self) -> Dict[str, int]:
-        """Get mapping of SFT special tokens to their IDs."""
-        result = {
-            self.T: self._tok2id[self.T],
-            self.T_END: self._tok2id[self.T_END],
-            self.SEP: self._tok2id[self.SEP],
-        }
-        if self._include_env_tokens:
-            for tok in ENV_TOKENS:
-                result[tok] = self._tok2id[tok]
-        return result
-    
-    def parse_cot_line(self, line: str) -> Tuple[Optional[List[str]], Optional[str]]:
-        """
-        Parse a CoT data line in format: <T> <sep> ... <sep> <T> {answer}
-        
-        Args:
-            line: A line from the CoT data file
-        
-        Returns:
-            trajectories: List of trajectory strings
-            answer: The final answer/move
-        """
-        line = line.strip()
-        if not line or not line.startswith(self.T):
-            return None, None
-        
-        return self.extract_thinking_and_answer(line)
-
-# ============================================================
-# HuggingFace-compatible wrapper (auto-generated)
-# ============================================================
-import json as _json
-from pathlib import Path as _Path
-from transformers import PreTrainedTokenizer
-import torch
-from transformers.tokenization_utils_base import BatchEncoding
-
-from huggingface_hub import hf_hub_download
-
-class HFTokenizerWrapper(PreTrainedTokenizer):
-    def __init__(self, model_max_length=2048, **kwargs):
-        # These are usually provided by from_pretrained
-        repo_id = kwargs.get("name_or_path") or kwargs.get("_name_or_path")
-        revision = kwargs.get("revision", None)
-
-        if not repo_id or "/" not in str(repo_id):
-            # Fallback: user may pass repo_id explicitly
-            repo_id = kwargs.get("repo_id", None)
-        if not repo_id:
-            raise ValueError("Cannot infer repo_id; pass repo_id=... or ensure name_or_path is set.")
-
-        import os
-        if os.path.isdir(repo_id):
-            vocab_path = os.path.join(repo_id, "vocab.json")
-            cfg_path   = os.path.join(repo_id, "tokenizer_config.json")
-        else:
-            vocab_path = hf_hub_download(repo_id=repo_id, filename="vocab.json", revision=revision)
-            cfg_path   = hf_hub_download(repo_id=repo_id, filename="tokenizer_config.json", revision=revision)
-
-        with open(vocab_path, "r", encoding="utf-8") as _f:
-            saved_vocab = _json.load(_f)
-        with open(cfg_path, "r", encoding="utf-8") as _f:
-            _tok_cfg = _json.load(_f)
-
-        lan_config = _tok_cfg.get("lan_config", {})
-        lan_class_name = _tok_cfg.get("lan_tokenizer_class", "LanTokenizerSFT")
-
-        _cls = globals()[lan_class_name]
-        custom_tokenizer = _cls(config=lan_config)
-
-        # Override vocab with the saved vocab
-        custom_tokenizer._tok2id = saved_vocab
-        from tokenizers import Tokenizer as _TkTokenizer
-        from tokenizers.models import WordLevel as _WordLevel
-        from tokenizers.pre_tokenizers import WhitespaceSplit as _WhitespaceSplit
-        custom_tokenizer.tk = _TkTokenizer(_WordLevel(vocab=saved_vocab, unk_token=custom_tokenizer._unk))
-        custom_tokenizer.tk.pre_tokenizer = _WhitespaceSplit()
-
-        self.custom_tokenizer = custom_tokenizer
-        self._vocab = dict(saved_vocab)
-        self._id_to_token = {i: t for t, i in self._vocab.items()}
-
-        bos_token = _tok_cfg.get("bos_token")
-        eos_token = _tok_cfg.get("eos_token")
-        pad_token = _tok_cfg.get("pad_token")
-        unk_token = _tok_cfg.get("unk_token")
-        env_token = _tok_cfg.get("env_token")
-        if "env_id" in _tok_cfg:
-            env_token = self._id_to_token[_tok_cfg.get("env_id")]
-        else:
-            env_token = _tok_cfg.get("env_token")
-        self.env_token = env_token
-
-        for _key in ("bos_token","eos_token","pad_token","unk_token","env_token",
-                     "model_max_length","name_or_path","lan_config",
-                     "lan_tokenizer_class","tokenizer_class","auto_map","use_fast",
-                     "revision","repo_id"):
-            kwargs.pop(_key, None)
-
-        super().__init__(
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            unk_token=unk_token,
-            model_max_length=model_max_length,
-            **kwargs,
-        )
-
-    # ---- PreTrainedTokenizer interface ----
-
-    @property
-    def vocab_size(self):
-        return len(self._vocab)
-
-    def get_vocab(self):
-        return dict(self._vocab)
-
-    def _tokenize(self, text):
-        return []  # we override encode/decode directly
-
-    def _convert_token_to_id(self, token):
-        return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
-
-    def _convert_id_to_token(self, index):
-        return self._id_to_token.get(index, self.unk_token or "")
-
-    def convert_tokens_to_string(self, tokens):
-        ids = [self._convert_token_to_id(t) for t in tokens]
-        return self.custom_tokenizer.decode(ids)
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        if token_ids_1 is None:
-            return token_ids_0
-        return token_ids_0 + token_ids_1
-
-    def encode(self, text, add_special_tokens=True, **kwargs):
-        ids = self.custom_tokenizer.encode(text)
-        if add_special_tokens:
-            return ids[:-1]  # strip trailing EOS; vLLM adds its own
-        if (len(ids) >= 2
-                and self.bos_token_id is not None
-                and self.eos_token_id is not None
-                and ids[0] == self.bos_token_id
-                and ids[-1] == self.eos_token_id):
-            return ids[1:-1]
-        return ids
-
-    def decode(self, token_ids, skip_special_tokens=True, **kwargs):
-        import numpy as np
-        if isinstance(token_ids, torch.Tensor):
-            token_ids = token_ids.detach().cpu().tolist()
-        elif isinstance(token_ids, np.ndarray):
-            token_ids = token_ids.tolist()
-        return self.custom_tokenizer.decode(token_ids)
-
-    def save_vocabulary(self, save_directory, filename_prefix=None):
-        save_directory = _Path(save_directory)
-        save_directory.mkdir(parents=True, exist_ok=True)
-        vocab_file = save_directory / (
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json"
-        )
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            _json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (str(vocab_file),)
-
-    def __call__(
-        self,
-        text,
-        text_pair=None,
-        add_special_tokens=True,
-        truncation=False,
-        max_length=None,
-        padding=False,
-        return_tensors=None,
-        **kwargs,
-    ):
-        if text_pair is not None:
-            raise ValueError("text_pair not supported for this tokenizer.")
-
-        # Normalize to batch
-        is_batched = isinstance(text, (list, tuple))
-        texts = list(text) if is_batched else [text]
-
-        input_ids = [self.encode(t, add_special_tokens=add_special_tokens) for t in texts]
-
-        # Truncation
-        if truncation and max_length is not None:
-            if self.truncation_side == "left":
-                input_ids = [ids[-max_length:] for ids in input_ids]
-            else:
-                input_ids = [ids[:max_length] for ids in input_ids]
-
-        # Attention masks (pre-padding)
-        attention_mask = [[1] * len(ids) for ids in input_ids]
-
-        # Padding
-        if padding:
-            if padding == "max_length":
-                if max_length is None:
-                    raise ValueError("padding='max_length' requires max_length.")
-                pad_to = max_length
-            else:
-                pad_to = max(len(ids) for ids in input_ids) if input_ids else 0
-
-            pad_id = self.pad_token_id
-            if pad_id is None:
-                pad_id = self.bos_token_id if self.bos_token_id is not None else 0
-
-            for i, ids in enumerate(input_ids):
-                pad_len = pad_to - len(ids)
-                if pad_len > 0:
-                    input_ids[i] = ids + [pad_id] * pad_len
-                    attention_mask[i] = attention_mask[i] + [0] * pad_len
-
-        data = {"input_ids": input_ids, "attention_mask": attention_mask}
-
-        # Unbatch if single example and no tensor return
-        if not is_batched and return_tensors is None:
-            data = {"input_ids": data["input_ids"][0], "attention_mask": data["attention_mask"][0]}
-
-        # Tensors
-        if return_tensors == "pt":
-            data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()}
-
-        return BatchEncoding(data, tensor_type=None)
-
-
-__all__ = ["HFTokenizerWrapper"]

C6p5e18_410m_alpha0.200_beta0.045/final/tokenizer_config.json

@@ -1,26 +0,0 @@
-{
-  "tokenizer_class": "HFTokenizerWrapper",
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenizer.HFTokenizerWrapper",
-      null
-    ]
-  },
-  "model_max_length": 2048,
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null,
-  "use_fast": false,
-  "lan_config": {
-    "name": "LanTokenizerSFT",
-    "include_move_numbers": false,
-    "include_black_tripledots": false,
-    "bos": "<bos>",
-    "eos": "<eos>",
-    "unk": "<unk>",
-    "keep_result": false
-  },
-  "lan_tokenizer_class": "LanTokenizerSFT"
-}

C6p5e18_410m_alpha0.200_beta0.045/final/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.200_beta0.045/final/vocab.json

@@ -1,86 +0,0 @@
-{
-  "<bos>": 0,
-  "<eos>": 1,
-  "<unk>": 2,
-  "K": 3,
-  "Q": 4,
-  "R": 5,
-  "B": 6,
-  "N": 7,
-  "P": 8,
-  "a1": 9,
-  "a2": 10,
-  "a3": 11,
-  "a4": 12,
-  "a5": 13,
-  "a6": 14,
-  "a7": 15,
-  "a8": 16,
-  "b1": 17,
-  "b2": 18,
-  "b3": 19,
-  "b4": 20,
-  "b5": 21,
-  "b6": 22,
-  "b7": 23,
-  "b8": 24,
-  "c1": 25,
-  "c2": 26,
-  "c3": 27,
-  "c4": 28,
-  "c5": 29,
-  "c6": 30,
-  "c7": 31,
-  "c8": 32,
-  "d1": 33,
-  "d2": 34,
-  "d3": 35,
-  "d4": 36,
-  "d5": 37,
-  "d6": 38,
-  "d7": 39,
-  "d8": 40,
-  "e1": 41,
-  "e2": 42,
-  "e3": 43,
-  "e4": 44,
-  "e5": 45,
-  "e6": 46,
-  "e7": 47,
-  "e8": 48,
-  "f1": 49,
-  "f2": 50,
-  "f3": 51,
-  "f4": 52,
-  "f5": 53,
-  "f6": 54,
-  "f7": 55,
-  "f8": 56,
-  "g1": 57,
-  "g2": 58,
-  "g3": 59,
-  "g4": 60,
-  "g5": 61,
-  "g6": 62,
-  "g7": 63,
-  "g8": 64,
-  "h1": 65,
-  "h2": 66,
-  "h3": 67,
-  "h4": 68,
-  "h5": 69,
-  "h6": 70,
-  "h7": 71,
-  "h8": 72,
-  "x": 73,
-  "=": 74,
-  "+": 75,
-  "#": 76,
-  "O-O": 77,
-  "O-O-O": 78,
-  ".": 79,
-  "...": 80,
-  "<T>": 81,
-  "</T>": 82,
-  "<sep>": 83
-}

C6p5e18_410m_alpha0.200_beta0.045/latest

@@ -1 +0,0 @@
-step_110

C6p5e18_410m_alpha0.200_beta0.045/latest_ckpt/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a864bb33decacb0fd02126c0c47b7576a3b5129830ef6b089487f009f7511efb
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/latest_ckpt/optimizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aac3284ed0372ec7cf556121eed9f601f73f2064d672d56ac1800f531ffe64b7
-size 1645034059

C6p5e18_410m_alpha0.200_beta0.045/latest_ckpt/random_states_0.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:85f3b361e9f4b1b254befe3769b50c204af81c5009a350ec4ce1e8e0d577d63d
-size 15081

C6p5e18_410m_alpha0.200_beta0.045/latest_ckpt/scheduler.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0183971c5d0a20e92ba8c8db76425ea86478f65d9a0e5fe494f3c85f26cc9c20
-size 1465

C6p5e18_410m_alpha0.200_beta0.045/latest_ckpt/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.200_beta0.045/step_100/config.json

@@ -1,64 +0,0 @@
-{
-  "architectures": [
-    "Qwen3ForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "dtype": "bfloat16",
-  "eos_token_id": 1,
-  "head_dim": 128,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_types": [
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention"
-  ],
-  "max_position_embeddings": 2048,
-  "max_window_layers": 28,
-  "model_type": "qwen3",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 4,
-  "pad_token_id": 0,
-  "rms_norm_eps": 1e-06,
-  "rope_parameters": {
-    "factor": 2.0,
-    "original_max_position_embeddings": 1024,
-    "type": "yarn"
-  },
-  "sliding_window": null,
-  "tie_word_embeddings": true,
-  "transformers_version": "5.3.0",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 84
-}

C6p5e18_410m_alpha0.200_beta0.045/step_100/generation_config.json

@@ -1,9 +0,0 @@
-{
-  "bos_token_id": 0,
-  "do_sample": true,
-  "eos_token_id": 1,
-  "max_new_tokens": 1024,
-  "temperature": 1.0,
-  "top_k": 50,
-  "transformers_version": "5.3.0"
-}

C6p5e18_410m_alpha0.200_beta0.045/step_100/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:328ea46a7de56a02f2ee7823a9ecb91e30d64823bcc7a7c886f3b6aabd1e6022
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/step_100/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null
-}

C6p5e18_410m_alpha0.200_beta0.045/step_100/tokenizer.py

@@ -1,818 +0,0 @@
-"""
-Auto-generated self-contained HF tokenizer.
-Do NOT edit manually -- regenerate via training.hf_tokenizer_utils.save_hf_tokenizer().
-"""
-from __future__ import annotations
-
-# --- BaseTokenizer (inlined) ---
-# base_tokenizer.py
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
-
-class BaseTokenizer(ABC):
-    """Minimal interface for tokenizers used in pretraining."""
-
-    # ---- required ----
-    @abstractmethod
-    def encode(self, text: str) -> List[int]:
-        """Convert text/PGN to token IDs."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def decode(self, ids: List[int]) -> str:
-        """Convert token IDs back to text/PGN."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def get_vocab(self) -> Dict[str, int]:
-        """Return token -> id mapping (if available)."""
-        raise NotImplementedError
-
-    def bos_id(self) -> Optional[int]: return None
-    def eos_id(self) -> Optional[int]: return None
-    def pad_id(self) -> Optional[int]: return None
-    def get_vocab_size(self) -> int: return len(self.get_vocab())
-
-    def __call__(self, text: str) -> List[int]:
-        """Alias for encode()."""
-        return self.encode(text)
-
-# --- Concrete tokenizer (inlined) ---
-# lan_tokenizer_sft.py
-"""
-LAN Tokenizer with SFT support (CoT format with <T> and <sep> tokens).
-
-This extends the base LAN tokenizer with SFT-specific functionality:
-- <T> token for marking thinking/CoT content
-- <sep> token for separating prompt from response
-"""
-from typing import List, Dict, Optional, Tuple
-import io
-import chess, chess.pgn
-from tokenizers import Tokenizer
-from tokenizers.models import WordLevel
-from tokenizers.pre_tokenizers import WhitespaceSplit
-_RESULT = {"1-0", "0-1", "1/2-1/2", "*"}
-FILES = "abcdefgh"
-RANKS = "12345678"
-SQUARES = [f+r for f in FILES for r in RANKS]
-PROMOS = "QRBN"
-DIGITS = set("0123456789")
-
-# SFT special tokens for CoT format
-T_TOKEN = "<T>"
-T_END_TOKEN = "</T>"
-SEP_TOKEN = "<sep>"
-
-# Environment interaction / reward special tokens
-CALL_ENV_TOKEN = "<call_env>"
-VERIFY_TOKEN = "<verify>"
-REWARD_POS_TOKEN = "<+1>"
-REWARD_NEG_TOKEN = "<-1>"
-REWARD_ZERO_TOKEN = "<0>"
-ENV_TOKENS = [CALL_ENV_TOKEN]
-REWARD_TOKENS = [VERIFY_TOKEN, REWARD_POS_TOKEN, REWARD_NEG_TOKEN, REWARD_ZERO_TOKEN]
-
-def _vocab_with_sft(
-    include_move_numbers: bool,
-    keep_result: bool,
-    bos: str,
-    eos: str,
-    unk: str,
-    include_env_tokens: bool = False,
-    include_reward_tokens: bool = False,
-) -> Dict[str, int]:
-    """Create vocabulary including SFT special tokens."""
-    base = [bos, eos, unk]
-    ops = ["x", "=", "+", "#", "O-O", "O-O-O", ".", "..."]
-    toks = base + list("KQRBNP") + SQUARES + list(PROMOS) + ops
-    if include_move_numbers:
-        toks += list("0123456789")
-    if keep_result:
-        toks += sorted(_RESULT)
-
-    # Add SFT special tokens for CoT format
-    sft_tokens = [T_TOKEN, T_END_TOKEN, SEP_TOKEN]
-    toks += sft_tokens
-
-    # Add environment / reward tokens when requested
-    if include_env_tokens:
-        toks += ENV_TOKENS
-    if include_reward_tokens:
-        toks += REWARD_TOKENS
-
-    return {t: i for i, t in enumerate(dict.fromkeys(toks))}
-
-
-class LanTokenizerSFT(BaseTokenizer):
-    """
-    LAN Tokenizer with SFT capabilities.
-
-    This tokenizer extends the base LAN tokenizer with:
-    - <T> token for marking thinking/CoT boundaries
-    - <sep> token for separating candidate trajectories
-
-    CoT Format: {prompt} <T> <sep> {traj1} <sep> {traj2} <sep> ... <sep> {trajN} <sep> <T> {answer}
-
-    Where:
-    - {prompt}: The game history/board state (PGN moves)
-    - {trajN}: Candidate reasoning trajectories
-    - {answer}: The final best move
-    """
-
-    # Special tokens for CoT format
-    T = T_TOKEN
-    T_END = T_END_TOKEN
-    SEP = SEP_TOKEN
-
-    # Environment / reward tokens (class-level constants for easy access)
-    CALL_ENV = CALL_ENV_TOKEN   # "<call_env>"
-    VERIFY = VERIFY_TOKEN       # "<verify>"
-    REWARD_POS = REWARD_POS_TOKEN  # "<+1>"
-    REWARD_NEG = REWARD_NEG_TOKEN  # "<-1>"
-    REWARD_ZERO = REWARD_ZERO_TOKEN  # "<0>"
-    ENV_TOKENS = ENV_TOKENS     # full list
-
-    def __init__(self, config: Optional[dict] = None):
-        """
-        Args:
-            config: Configuration dict with tokenizer settings.
-                include_env_tokens (bool): add <call_env>, <verify>, <+1>, <-1>, <0>
-                    to the vocabulary.  Default: False.
-        """
-        config = config or {}
-
-        include_move_numbers = config.get("include_move_numbers", False)
-        include_black_tripledots = config.get("include_black_tripledots", False)
-        bos = config.get("bos", "<bos>")
-        eos = config.get("eos", "<eos>")
-        unk = config.get("unk", "<unk>")
-        keep_result = config.get("keep_result", False)
-        include_env_tokens = config.get("include_env_tokens", False)
-        include_reward_tokens = config.get("include_reward_tokens", False)
-
-        self._bos = bos
-        self._eos = eos
-        self._unk = unk
-        self._keep_res = keep_result
-        self._include_nums = include_move_numbers
-        self._include_black_ellipses = include_black_tripledots
-        self._include_env_tokens = include_env_tokens
-        self._include_reward_tokens = include_reward_tokens
-
-        # Create vocabulary with SFT tokens
-        tok2id = _vocab_with_sft(
-            include_move_numbers, keep_result, bos, eos, unk,
-            include_env_tokens=include_env_tokens,
-            include_reward_tokens=include_reward_tokens,
-        )
-        self._tok2id = tok2id
-
-        # Initialize tokenizer
-        self.tk = Tokenizer(WordLevel(vocab=tok2id, unk_token=self._unk))
-        self.tk.pre_tokenizer = WhitespaceSplit()
-    
-    def _pgn_to_tokens(self, text: str) -> Optional[List[str]]:
-        """Convert PGN text to tokens."""
-        import os, contextlib
-        with open(os.devnull, "w") as devnull, contextlib.redirect_stderr(devnull):
-            g = chess.pgn.read_game(io.StringIO(text))
-        if g is None:
-            return None
-        
-        b, out, n = g.board(), [], 1
-        for mv in g.mainline_moves():
-            if b.turn == chess.WHITE and self._include_nums:
-                out += list(str(n)) + (
-                    ["..."] if self._include_black_ellipses and b.fullmove_number < n else ["."]
-                )
-            
-            if b.is_castling(mv):
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                b.pop()
-                out.append("O-O" if chess.square_file(mv.to_square) == 6 else "O-O-O")
-                if suf:
-                    out.append(suf)
-                b.push(mv)
-            else:
-                piece = b.piece_at(mv.from_square).symbol().upper()
-                frm = chess.square_name(mv.from_square)
-                to = chess.square_name(mv.to_square)
-                is_cap = b.is_capture(mv)
-                promo = mv.promotion
-                
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                
-                # Emit LAN tokens
-                out.append(piece)
-                out.append(frm)
-                if is_cap:
-                    out.append("x")
-                out.append(to)
-                if promo:
-                    out += ["=", chess.piece_symbol(promo).upper()]
-                if suf:
-                    out.append(suf)
-            
-            if b.turn == chess.WHITE:
-                n += 1
-        
-        res = g.headers.get("Result")
-        if self._keep_res and res in _RESULT:
-            out.append(res)
-        
-        return out
-    
-    def _lan_move_to_tokens(self, move: str) -> List[str]:
-        """
-        Convert a single LAN move to tokens.
-        
-        LAN format: [Piece][from_square][x]?[to_square][=Promo]?[+#]?
-        
-        Examples:
-            "Ng1f3" -> ["N", "g1", "f3"]
-            "Nd4xe6" -> ["N", "d4", "x", "e6"]
-            "Pe2e4" -> ["P", "e2", "e4"]
-            "Pe4xd5" -> ["P", "e4", "x", "d5"]
-            "O-O" -> ["O-O"]
-            "O-O-O" -> ["O-O-O"]
-            "Pe7e8=Q" -> ["P", "e7", "e8", "=", "Q"]
-            "Ng1f3+" -> ["N", "g1", "f3", "+"]
-        """
-        # Handle castling
-        if move in {"O-O", "O-O-O"}:
-            return [move]
-        if move.rstrip("+#") in {"O-O", "O-O-O"}:
-            base = move.rstrip("+#")
-            suffix = move[len(base):]
-            return [base] + ([suffix] if suffix else [])
-        
-        out = []
-        i = 0
-        n = len(move)
-        
-        # Get piece letter (required in LAN format)
-        if i < n and move[i] in "KQRBNP":
-            out.append(move[i])
-            i += 1
-        else:
-            # No piece letter - might be malformed, return as-is
-            return [move]
-        
-        # Get from square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle capture
-        if i < n and move[i] == "x":
-            out.append("x")
-            i += 1
-        
-        # Get to square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle promotion
-        if i < n and move[i] == "=":
-            out.append("=")
-            i += 1
-            if i < n and move[i] in PROMOS:
-                out.append(move[i])
-                i += 1
-        
-        # Handle check/checkmate
-        if i < n and move[i] in "+#":
-            out.append(move[i])
-            i += 1
-        
-        return out
-    
-    def _active_env_tokens(self) -> set:
-        """Return the set of env tokens that are active for this instance."""
-        return set(ENV_TOKENS) if self._include_env_tokens else set()
-
-    def _cot_to_tokens(self, text: str) -> List[str]:
-        """
-        Convert CoT formatted text to tokens.
-        Handles special tokens and LAN moves.
-        """
-        env_toks = self._active_env_tokens()
-        out = []
-        for token in text.split():
-            if token in {self.T, self.T_END, self.SEP} or token in env_toks:
-                # Keep special tokens as-is
-                out.append(token)
-            elif token in _RESULT:
-                # Game result
-                out.append(token)
-            elif token and token[0].isdigit() and "." in token:
-                # Move number like "1." or "15..."
-                # Split into digits and dots
-                num_part = token.rstrip(".")
-                dot_part = token[len(num_part):]
-                out.extend(list(num_part))
-                if dot_part:
-                    out.append("..." if len(dot_part) > 1 else ".")
-            elif token and all(c.isdigit() for c in token):
-                # Pure number - tokenize each digit
-                out.extend(list(token))
-            else:
-                # LAN move - tokenize it
-                out.extend(self._lan_move_to_tokens(token))
-        return out
-    
-    def encode(self, text: str) -> List[int]:
-        """
-        Encode text to token IDs.
-        
-        Args:
-            text: Text to encode (can be PGN or CoT formatted)
-        
-        Returns:
-            List of token IDs
-        """
-        # Check if this is CoT-formatted text (contains special tokens)
-        sft_special = (
-            [self.T, self.T_END, self.SEP]
-            + (ENV_TOKENS if self._include_env_tokens else [])
-        )
-        is_cot_format = any(token in text for token in sft_special)
-        
-        if is_cot_format:
-            t_idx = text.index(self.T)
-            prompt_part = text[:t_idx].strip()
-            rest_part = text[t_idx:]  # starts with <T>
-
-            pgn_tokens = self._pgn_to_tokens(prompt_part) if prompt_part else None
-            if pgn_tokens is None:
-                pgn_tokens = self._cot_to_tokens(prompt_part) if prompt_part else []
-            rest_tokens = self._cot_to_tokens(rest_part)
-            tokens = [self._bos] + pgn_tokens + rest_tokens + [self._eos]
-        else:
-            pgn_tokens = self._pgn_to_tokens(text)
-            if pgn_tokens is not None and len(pgn_tokens) > 0:
-                tokens = [self._bos] + pgn_tokens + [self._eos]
-            else:
-                # Not valid PGN — treat each word as a LAN move
-                lan_tokens = []
-                for word in text.split():
-                    lan_tokens.extend(self._lan_move_to_tokens(word))
-                tokens = [self._bos] + lan_tokens + [self._eos]
-        
-        return self.tk.encode(" ".join(tokens)).ids
-    
-    def decode(self, ids: List[int]) -> str:
-        """
-        Decode token IDs to text.
-        
-        Args:
-            ids: List of token IDs
-        
-        Returns:
-            Decoded text
-        """
-        toks = [t for t in self.tk.decode(ids).split() if t not in {self._bos, self._eos}]
-        
-        # Otherwise, use LAN decoding logic
-        out: List[str] = []
-        i, n = 0, len(toks)
-        
-        while i < n:
-            t = toks[i]
-            
-            if t in {self.T, self.T_END, self.SEP} or t in _RESULT or t in self._active_env_tokens():
-                out.append(t)
-                i += 1
-                continue
-            
-            if t and all(ch in DIGITS for ch in t):
-                j = i
-                num = []
-                while j < n and all(ch in DIGITS for ch in toks[j]):
-                    num.append(toks[j])
-                    j += 1
-                dots = ""
-                if j < n and toks[j] in {".", "..."}:
-                    dots = toks[j]
-                    j += 1
-                out.append("".join(num) + dots)
-                i = j
-                continue
-            
-            if t in {"O-O", "O-O-O"}:
-                j = i + 1
-                suf = toks[j] if j < n and toks[j] in {"+", "#"} else ""
-                if suf:
-                    j += 1
-                out.append(t + suf)
-                i = j
-                continue
-            
-            if t in set("KQRBNP"):
-                piece = t
-                j = i + 1
-                frm = toks[j] if j < n else ""
-                j += 1
-                cap = ""
-                if j < n and toks[j] == "x":
-                    cap = "x"
-                    j += 1
-                to = toks[j] if j < n else ""
-                j += 1
-                promo = ""
-                if j + 1 <= n - 1 and toks[j] == "=" and toks[j + 1] in set(PROMOS):
-                    promo = "=" + toks[j + 1]
-                    j += 2
-                suf = ""
-                if j < n and toks[j] in {"+", "#"}:
-                    suf = toks[j]
-                    j += 1
-                lan = f"{piece}{frm}{cap}{to}{promo}{suf}"
-                out.append(lan)
-                i = j
-                continue
-            
-            out.append(t)
-            i += 1
-        
-        return " ".join(out)
-    
-    def get_vocab(self) -> Dict[str, int]:
-        """Get token-to-id vocabulary mapping."""
-        return self._tok2id
-    
-    def bos_id(self) -> Optional[int]:
-        """Get BOS token ID."""
-        return self._tok2id[self._bos]
-    
-    def eos_id(self) -> Optional[int]:
-        """Get EOS token ID."""
-        return self._tok2id[self._eos]
-    
-    def pad_id(self) -> Optional[int]:
-        """Get PAD token ID (uses BOS as pad by default)."""
-        return self._tok2id.get("<pad>", self.bos_id())
-    
-    def get_vocab_size(self) -> int:
-        """Get vocabulary size."""
-        return len(self._tok2id)
-    
-    def t_id(self) -> int:
-        """Get <T> token ID."""
-        return self._tok2id[self.T]
-    
-    def sep_id(self) -> int:
-        """Get <sep> token ID."""
-        return self._tok2id[self.SEP]
-    
-    def t_end_id(self) -> int:
-        """Get </T> token ID."""
-        return self._tok2id[self.T_END]
-    
-    # ------------------------------------------------------------------
-    # Environment / reward token accessors
-    # ------------------------------------------------------------------
-
-    def _require_env_tokens(self) -> None:
-        if not self._include_env_tokens:
-            raise ValueError(
-                "Environment tokens are not enabled. "
-                "Pass include_env_tokens=True in the config."
-            )
-
-    def call_env_id(self) -> int:
-        """Get <call_env> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[CALL_ENV_TOKEN]
-
-    def verify_id(self) -> int:
-        """Get <verify> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[VERIFY_TOKEN]
-
-    def reward_pos_id(self) -> int:
-        """Get <+1> (positive reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_POS_TOKEN]
-
-    def reward_neg_id(self) -> int:
-        """Get <-1> (negative reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_NEG_TOKEN]
-
-    def reward_zero_id(self) -> int:
-        """Get <0> (zero reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_ZERO_TOKEN]
-
-    def reward_id(self, value) -> int:
-        """
-        Get reward token ID by numeric value.
-
-        Args:
-            value: 1, -1, or 0  (or the strings "+1", "-1", "0")
-
-        Returns:
-            Token ID for the corresponding reward token.
-        """
-        self._require_env_tokens()
-        mapping = {1: REWARD_POS_TOKEN, -1: REWARD_NEG_TOKEN, 0: REWARD_ZERO_TOKEN,
-                   "+1": REWARD_POS_TOKEN, "-1": REWARD_NEG_TOKEN, "0": REWARD_ZERO_TOKEN}
-        if value not in mapping:
-            raise ValueError(f"reward value must be one of 1, -1, 0 (or '+1', '-1', '0'), got {value!r}")
-        return self._tok2id[mapping[value]]
-
-    def env_token_ids(self) -> Dict[str, int]:
-        """Get mapping of all env/reward special tokens to their IDs."""
-        self._require_env_tokens()
-        return {tok: self._tok2id[tok] for tok in ENV_TOKENS}
-    
-    def extract_parts(self, text: str) -> Tuple[Optional[str], Optional[List[str]], str]:
-        """
-        Extract prompt, trajectories and answer from BoN CoT formatted text.
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            prompt: The prompt/context (or None if not present)
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        if self.T not in text:
-            return None, None, text
-        
-        try:
-            # Split by <T> to get prompt, thinking section, and answer
-            t_parts = text.split(self.T)
-            if len(t_parts) < 3:
-                return None, None, text
-            
-            # t_parts[0] is prompt (before first <T>)
-            # t_parts[1] is the thinking section with trajectories
-            # t_parts[2] is the answer
-            prompt = t_parts[0].strip() if t_parts[0].strip() else None
-            thinking_section = t_parts[1].strip()
-            answer = t_parts[2].strip()
-            
-            # Split thinking section by <sep> to get trajectories
-            trajectories = [t.strip() for t in thinking_section.split(self.SEP) if t.strip()]
-            
-            return prompt, trajectories, answer
-        except (ValueError, IndexError):
-            return None, None, text
-    
-    def extract_thinking_and_answer(self, text: str) -> Tuple[Optional[List[str]], str]:
-        """
-        Extract trajectories and answer from BoN CoT formatted text (ignores prompt).
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        _, trajectories, answer = self.extract_parts(text)
-        return trajectories, answer
-    
-    def get_sft_special_tokens(self) -> List[str]:
-        """Get list of SFT special tokens (including env/reward tokens if enabled)."""
-        toks = [self.T, self.T_END, self.SEP]
-        if self._include_env_tokens:
-            toks += ENV_TOKENS
-        return toks
-
-    def get_sft_token_ids(self) -> Dict[str, int]:
-        """Get mapping of SFT special tokens to their IDs."""
-        result = {
-            self.T: self._tok2id[self.T],
-            self.T_END: self._tok2id[self.T_END],
-            self.SEP: self._tok2id[self.SEP],
-        }
-        if self._include_env_tokens:
-            for tok in ENV_TOKENS:
-                result[tok] = self._tok2id[tok]
-        return result
-    
-    def parse_cot_line(self, line: str) -> Tuple[Optional[List[str]], Optional[str]]:
-        """
-        Parse a CoT data line in format: <T> <sep> ... <sep> <T> {answer}
-        
-        Args:
-            line: A line from the CoT data file
-        
-        Returns:
-            trajectories: List of trajectory strings
-            answer: The final answer/move
-        """
-        line = line.strip()
-        if not line or not line.startswith(self.T):
-            return None, None
-        
-        return self.extract_thinking_and_answer(line)
-
-# ============================================================
-# HuggingFace-compatible wrapper (auto-generated)
-# ============================================================
-import json as _json
-from pathlib import Path as _Path
-from transformers import PreTrainedTokenizer
-import torch
-from transformers.tokenization_utils_base import BatchEncoding
-
-from huggingface_hub import hf_hub_download
-
-class HFTokenizerWrapper(PreTrainedTokenizer):
-    def __init__(self, model_max_length=2048, **kwargs):
-        # These are usually provided by from_pretrained
-        repo_id = kwargs.get("name_or_path") or kwargs.get("_name_or_path")
-        revision = kwargs.get("revision", None)
-
-        if not repo_id or "/" not in str(repo_id):
-            # Fallback: user may pass repo_id explicitly
-            repo_id = kwargs.get("repo_id", None)
-        if not repo_id:
-            raise ValueError("Cannot infer repo_id; pass repo_id=... or ensure name_or_path is set.")
-
-        import os
-        if os.path.isdir(repo_id):
-            vocab_path = os.path.join(repo_id, "vocab.json")
-            cfg_path   = os.path.join(repo_id, "tokenizer_config.json")
-        else:
-            vocab_path = hf_hub_download(repo_id=repo_id, filename="vocab.json", revision=revision)
-            cfg_path   = hf_hub_download(repo_id=repo_id, filename="tokenizer_config.json", revision=revision)
-
-        with open(vocab_path, "r", encoding="utf-8") as _f:
-            saved_vocab = _json.load(_f)
-        with open(cfg_path, "r", encoding="utf-8") as _f:
-            _tok_cfg = _json.load(_f)
-
-        lan_config = _tok_cfg.get("lan_config", {})
-        lan_class_name = _tok_cfg.get("lan_tokenizer_class", "LanTokenizerSFT")
-
-        _cls = globals()[lan_class_name]
-        custom_tokenizer = _cls(config=lan_config)
-
-        # Override vocab with the saved vocab
-        custom_tokenizer._tok2id = saved_vocab
-        from tokenizers import Tokenizer as _TkTokenizer
-        from tokenizers.models import WordLevel as _WordLevel
-        from tokenizers.pre_tokenizers import WhitespaceSplit as _WhitespaceSplit
-        custom_tokenizer.tk = _TkTokenizer(_WordLevel(vocab=saved_vocab, unk_token=custom_tokenizer._unk))
-        custom_tokenizer.tk.pre_tokenizer = _WhitespaceSplit()
-
-        self.custom_tokenizer = custom_tokenizer
-        self._vocab = dict(saved_vocab)
-        self._id_to_token = {i: t for t, i in self._vocab.items()}
-
-        bos_token = _tok_cfg.get("bos_token")
-        eos_token = _tok_cfg.get("eos_token")
-        pad_token = _tok_cfg.get("pad_token")
-        unk_token = _tok_cfg.get("unk_token")
-        env_token = _tok_cfg.get("env_token")
-        if "env_id" in _tok_cfg:
-            env_token = self._id_to_token[_tok_cfg.get("env_id")]
-        else:
-            env_token = _tok_cfg.get("env_token")
-        self.env_token = env_token
-
-        for _key in ("bos_token","eos_token","pad_token","unk_token","env_token",
-                     "model_max_length","name_or_path","lan_config",
-                     "lan_tokenizer_class","tokenizer_class","auto_map","use_fast",
-                     "revision","repo_id"):
-            kwargs.pop(_key, None)
-
-        super().__init__(
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            unk_token=unk_token,
-            model_max_length=model_max_length,
-            **kwargs,
-        )
-
-    # ---- PreTrainedTokenizer interface ----
-
-    @property
-    def vocab_size(self):
-        return len(self._vocab)
-
-    def get_vocab(self):
-        return dict(self._vocab)
-
-    def _tokenize(self, text):
-        return []  # we override encode/decode directly
-
-    def _convert_token_to_id(self, token):
-        return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
-
-    def _convert_id_to_token(self, index):
-        return self._id_to_token.get(index, self.unk_token or "")
-
-    def convert_tokens_to_string(self, tokens):
-        ids = [self._convert_token_to_id(t) for t in tokens]
-        return self.custom_tokenizer.decode(ids)
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        if token_ids_1 is None:
-            return token_ids_0
-        return token_ids_0 + token_ids_1
-
-    def encode(self, text, add_special_tokens=True, **kwargs):
-        ids = self.custom_tokenizer.encode(text)
-        if add_special_tokens:
-            return ids[:-1]  # strip trailing EOS; vLLM adds its own
-        if (len(ids) >= 2
-                and self.bos_token_id is not None
-                and self.eos_token_id is not None
-                and ids[0] == self.bos_token_id
-                and ids[-1] == self.eos_token_id):
-            return ids[1:-1]
-        return ids
-
-    def decode(self, token_ids, skip_special_tokens=True, **kwargs):
-        import numpy as np
-        if isinstance(token_ids, torch.Tensor):
-            token_ids = token_ids.detach().cpu().tolist()
-        elif isinstance(token_ids, np.ndarray):
-            token_ids = token_ids.tolist()
-        return self.custom_tokenizer.decode(token_ids)
-
-    def save_vocabulary(self, save_directory, filename_prefix=None):
-        save_directory = _Path(save_directory)
-        save_directory.mkdir(parents=True, exist_ok=True)
-        vocab_file = save_directory / (
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json"
-        )
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            _json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (str(vocab_file),)
-
-    def __call__(
-        self,
-        text,
-        text_pair=None,
-        add_special_tokens=True,
-        truncation=False,
-        max_length=None,
-        padding=False,
-        return_tensors=None,
-        **kwargs,
-    ):
-        if text_pair is not None:
-            raise ValueError("text_pair not supported for this tokenizer.")
-
-        # Normalize to batch
-        is_batched = isinstance(text, (list, tuple))
-        texts = list(text) if is_batched else [text]
-
-        input_ids = [self.encode(t, add_special_tokens=add_special_tokens) for t in texts]
-
-        # Truncation
-        if truncation and max_length is not None:
-            if self.truncation_side == "left":
-                input_ids = [ids[-max_length:] for ids in input_ids]
-            else:
-                input_ids = [ids[:max_length] for ids in input_ids]
-
-        # Attention masks (pre-padding)
-        attention_mask = [[1] * len(ids) for ids in input_ids]
-
-        # Padding
-        if padding:
-            if padding == "max_length":
-                if max_length is None:
-                    raise ValueError("padding='max_length' requires max_length.")
-                pad_to = max_length
-            else:
-                pad_to = max(len(ids) for ids in input_ids) if input_ids else 0
-
-            pad_id = self.pad_token_id
-            if pad_id is None:
-                pad_id = self.bos_token_id if self.bos_token_id is not None else 0
-
-            for i, ids in enumerate(input_ids):
-                pad_len = pad_to - len(ids)
-                if pad_len > 0:
-                    input_ids[i] = ids + [pad_id] * pad_len
-                    attention_mask[i] = attention_mask[i] + [0] * pad_len
-
-        data = {"input_ids": input_ids, "attention_mask": attention_mask}
-
-        # Unbatch if single example and no tensor return
-        if not is_batched and return_tensors is None:
-            data = {"input_ids": data["input_ids"][0], "attention_mask": data["attention_mask"][0]}
-
-        # Tensors
-        if return_tensors == "pt":
-            data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()}
-
-        return BatchEncoding(data, tensor_type=None)
-
-
-__all__ = ["HFTokenizerWrapper"]

C6p5e18_410m_alpha0.200_beta0.045/step_100/tokenizer_config.json

@@ -1,26 +0,0 @@
-{
-  "tokenizer_class": "HFTokenizerWrapper",
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenizer.HFTokenizerWrapper",
-      null
-    ]
-  },
-  "model_max_length": 2048,
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null,
-  "use_fast": false,
-  "lan_config": {
-    "name": "LanTokenizerSFT",
-    "include_move_numbers": false,
-    "include_black_tripledots": false,
-    "bos": "<bos>",
-    "eos": "<eos>",
-    "unk": "<unk>",
-    "keep_result": false
-  },
-  "lan_tokenizer_class": "LanTokenizerSFT"
-}

C6p5e18_410m_alpha0.200_beta0.045/step_100/training_state.json

@@ -1 +0,0 @@
-{"step": 100, "epoch": 3}

C6p5e18_410m_alpha0.200_beta0.045/step_100/vocab.json

@@ -1,86 +0,0 @@
-{
-  "<bos>": 0,
-  "<eos>": 1,
-  "<unk>": 2,
-  "K": 3,
-  "Q": 4,
-  "R": 5,
-  "B": 6,
-  "N": 7,
-  "P": 8,
-  "a1": 9,
-  "a2": 10,
-  "a3": 11,
-  "a4": 12,
-  "a5": 13,
-  "a6": 14,
-  "a7": 15,
-  "a8": 16,
-  "b1": 17,
-  "b2": 18,
-  "b3": 19,
-  "b4": 20,
-  "b5": 21,
-  "b6": 22,
-  "b7": 23,
-  "b8": 24,
-  "c1": 25,
-  "c2": 26,
-  "c3": 27,
-  "c4": 28,
-  "c5": 29,
-  "c6": 30,
-  "c7": 31,
-  "c8": 32,
-  "d1": 33,
-  "d2": 34,
-  "d3": 35,
-  "d4": 36,
-  "d5": 37,
-  "d6": 38,
-  "d7": 39,
-  "d8": 40,
-  "e1": 41,
-  "e2": 42,
-  "e3": 43,
-  "e4": 44,
-  "e5": 45,
-  "e6": 46,
-  "e7": 47,
-  "e8": 48,
-  "f1": 49,
-  "f2": 50,
-  "f3": 51,
-  "f4": 52,
-  "f5": 53,
-  "f6": 54,
-  "f7": 55,
-  "f8": 56,
-  "g1": 57,
-  "g2": 58,
-  "g3": 59,
-  "g4": 60,
-  "g5": 61,
-  "g6": 62,
-  "g7": 63,
-  "g8": 64,
-  "h1": 65,
-  "h2": 66,
-  "h3": 67,
-  "h4": 68,
-  "h5": 69,
-  "h6": 70,
-  "h7": 71,
-  "h8": 72,
-  "x": 73,
-  "=": 74,
-  "+": 75,
-  "#": 76,
-  "O-O": 77,
-  "O-O-O": 78,
-  ".": 79,
-  "...": 80,
-  "<T>": 81,
-  "</T>": 82,
-  "<sep>": 83
-}

C6p5e18_410m_alpha0.200_beta0.045/step_110/config.json

@@ -1,64 +0,0 @@
-{
-  "architectures": [
-    "Qwen3ForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "dtype": "bfloat16",
-  "eos_token_id": 1,
-  "head_dim": 128,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_types": [
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention",
-    "full_attention"
-  ],
-  "max_position_embeddings": 2048,
-  "max_window_layers": 28,
-  "model_type": "qwen3",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 4,
-  "pad_token_id": 0,
-  "rms_norm_eps": 1e-06,
-  "rope_parameters": {
-    "factor": 2.0,
-    "original_max_position_embeddings": 1024,
-    "type": "yarn"
-  },
-  "sliding_window": null,
-  "tie_word_embeddings": true,
-  "transformers_version": "5.3.0",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 84
-}

C6p5e18_410m_alpha0.200_beta0.045/step_110/generation_config.json

@@ -1,9 +0,0 @@
-{
-  "bos_token_id": 0,
-  "do_sample": true,
-  "eos_token_id": 1,
-  "max_new_tokens": 1024,
-  "temperature": 1.0,
-  "top_k": 50,
-  "transformers_version": "5.3.0"
-}

C6p5e18_410m_alpha0.200_beta0.045/step_110/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a864bb33decacb0fd02126c0c47b7576a3b5129830ef6b089487f009f7511efb
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/step_110/optimizer_states/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a864bb33decacb0fd02126c0c47b7576a3b5129830ef6b089487f009f7511efb
-size 822421664

C6p5e18_410m_alpha0.200_beta0.045/step_110/optimizer_states/optimizer.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aac3284ed0372ec7cf556121eed9f601f73f2064d672d56ac1800f531ffe64b7
-size 1645034059

C6p5e18_410m_alpha0.200_beta0.045/step_110/optimizer_states/random_states_0.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:85f3b361e9f4b1b254befe3769b50c204af81c5009a350ec4ce1e8e0d577d63d
-size 15081

C6p5e18_410m_alpha0.200_beta0.045/step_110/optimizer_states/scheduler.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0183971c5d0a20e92ba8c8db76425ea86478f65d9a0e5fe494f3c85f26cc9c20
-size 1465

C6p5e18_410m_alpha0.200_beta0.045/step_110/optimizer_states/training_state.json

@@ -1 +0,0 @@
-{"step": 110, "epoch": 3}

C6p5e18_410m_alpha0.200_beta0.045/step_110/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null
-}

C6p5e18_410m_alpha0.200_beta0.045/step_110/tokenizer.py

@@ -1,818 +0,0 @@
-"""
-Auto-generated self-contained HF tokenizer.
-Do NOT edit manually -- regenerate via training.hf_tokenizer_utils.save_hf_tokenizer().
-"""
-from __future__ import annotations
-
-# --- BaseTokenizer (inlined) ---
-# base_tokenizer.py
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
-
-class BaseTokenizer(ABC):
-    """Minimal interface for tokenizers used in pretraining."""
-
-    # ---- required ----
-    @abstractmethod
-    def encode(self, text: str) -> List[int]:
-        """Convert text/PGN to token IDs."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def decode(self, ids: List[int]) -> str:
-        """Convert token IDs back to text/PGN."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def get_vocab(self) -> Dict[str, int]:
-        """Return token -> id mapping (if available)."""
-        raise NotImplementedError
-
-    def bos_id(self) -> Optional[int]: return None
-    def eos_id(self) -> Optional[int]: return None
-    def pad_id(self) -> Optional[int]: return None
-    def get_vocab_size(self) -> int: return len(self.get_vocab())
-
-    def __call__(self, text: str) -> List[int]:
-        """Alias for encode()."""
-        return self.encode(text)
-
-# --- Concrete tokenizer (inlined) ---
-# lan_tokenizer_sft.py
-"""
-LAN Tokenizer with SFT support (CoT format with <T> and <sep> tokens).
-
-This extends the base LAN tokenizer with SFT-specific functionality:
-- <T> token for marking thinking/CoT content
-- <sep> token for separating prompt from response
-"""
-from typing import List, Dict, Optional, Tuple
-import io
-import chess, chess.pgn
-from tokenizers import Tokenizer
-from tokenizers.models import WordLevel
-from tokenizers.pre_tokenizers import WhitespaceSplit
-_RESULT = {"1-0", "0-1", "1/2-1/2", "*"}
-FILES = "abcdefgh"
-RANKS = "12345678"
-SQUARES = [f+r for f in FILES for r in RANKS]
-PROMOS = "QRBN"
-DIGITS = set("0123456789")
-
-# SFT special tokens for CoT format
-T_TOKEN = "<T>"
-T_END_TOKEN = "</T>"
-SEP_TOKEN = "<sep>"
-
-# Environment interaction / reward special tokens
-CALL_ENV_TOKEN = "<call_env>"
-VERIFY_TOKEN = "<verify>"
-REWARD_POS_TOKEN = "<+1>"
-REWARD_NEG_TOKEN = "<-1>"
-REWARD_ZERO_TOKEN = "<0>"
-ENV_TOKENS = [CALL_ENV_TOKEN]
-REWARD_TOKENS = [VERIFY_TOKEN, REWARD_POS_TOKEN, REWARD_NEG_TOKEN, REWARD_ZERO_TOKEN]
-
-def _vocab_with_sft(
-    include_move_numbers: bool,
-    keep_result: bool,
-    bos: str,
-    eos: str,
-    unk: str,
-    include_env_tokens: bool = False,
-    include_reward_tokens: bool = False,
-) -> Dict[str, int]:
-    """Create vocabulary including SFT special tokens."""
-    base = [bos, eos, unk]
-    ops = ["x", "=", "+", "#", "O-O", "O-O-O", ".", "..."]
-    toks = base + list("KQRBNP") + SQUARES + list(PROMOS) + ops
-    if include_move_numbers:
-        toks += list("0123456789")
-    if keep_result:
-        toks += sorted(_RESULT)
-
-    # Add SFT special tokens for CoT format
-    sft_tokens = [T_TOKEN, T_END_TOKEN, SEP_TOKEN]
-    toks += sft_tokens
-
-    # Add environment / reward tokens when requested
-    if include_env_tokens:
-        toks += ENV_TOKENS
-    if include_reward_tokens:
-        toks += REWARD_TOKENS
-
-    return {t: i for i, t in enumerate(dict.fromkeys(toks))}
-
-
-class LanTokenizerSFT(BaseTokenizer):
-    """
-    LAN Tokenizer with SFT capabilities.
-
-    This tokenizer extends the base LAN tokenizer with:
-    - <T> token for marking thinking/CoT boundaries
-    - <sep> token for separating candidate trajectories
-
-    CoT Format: {prompt} <T> <sep> {traj1} <sep> {traj2} <sep> ... <sep> {trajN} <sep> <T> {answer}
-
-    Where:
-    - {prompt}: The game history/board state (PGN moves)
-    - {trajN}: Candidate reasoning trajectories
-    - {answer}: The final best move
-    """
-
-    # Special tokens for CoT format
-    T = T_TOKEN
-    T_END = T_END_TOKEN
-    SEP = SEP_TOKEN
-
-    # Environment / reward tokens (class-level constants for easy access)
-    CALL_ENV = CALL_ENV_TOKEN   # "<call_env>"
-    VERIFY = VERIFY_TOKEN       # "<verify>"
-    REWARD_POS = REWARD_POS_TOKEN  # "<+1>"
-    REWARD_NEG = REWARD_NEG_TOKEN  # "<-1>"
-    REWARD_ZERO = REWARD_ZERO_TOKEN  # "<0>"
-    ENV_TOKENS = ENV_TOKENS     # full list
-
-    def __init__(self, config: Optional[dict] = None):
-        """
-        Args:
-            config: Configuration dict with tokenizer settings.
-                include_env_tokens (bool): add <call_env>, <verify>, <+1>, <-1>, <0>
-                    to the vocabulary.  Default: False.
-        """
-        config = config or {}
-
-        include_move_numbers = config.get("include_move_numbers", False)
-        include_black_tripledots = config.get("include_black_tripledots", False)
-        bos = config.get("bos", "<bos>")
-        eos = config.get("eos", "<eos>")
-        unk = config.get("unk", "<unk>")
-        keep_result = config.get("keep_result", False)
-        include_env_tokens = config.get("include_env_tokens", False)
-        include_reward_tokens = config.get("include_reward_tokens", False)
-
-        self._bos = bos
-        self._eos = eos
-        self._unk = unk
-        self._keep_res = keep_result
-        self._include_nums = include_move_numbers
-        self._include_black_ellipses = include_black_tripledots
-        self._include_env_tokens = include_env_tokens
-        self._include_reward_tokens = include_reward_tokens
-
-        # Create vocabulary with SFT tokens
-        tok2id = _vocab_with_sft(
-            include_move_numbers, keep_result, bos, eos, unk,
-            include_env_tokens=include_env_tokens,
-            include_reward_tokens=include_reward_tokens,
-        )
-        self._tok2id = tok2id
-
-        # Initialize tokenizer
-        self.tk = Tokenizer(WordLevel(vocab=tok2id, unk_token=self._unk))
-        self.tk.pre_tokenizer = WhitespaceSplit()
-    
-    def _pgn_to_tokens(self, text: str) -> Optional[List[str]]:
-        """Convert PGN text to tokens."""
-        import os, contextlib
-        with open(os.devnull, "w") as devnull, contextlib.redirect_stderr(devnull):
-            g = chess.pgn.read_game(io.StringIO(text))
-        if g is None:
-            return None
-        
-        b, out, n = g.board(), [], 1
-        for mv in g.mainline_moves():
-            if b.turn == chess.WHITE and self._include_nums:
-                out += list(str(n)) + (
-                    ["..."] if self._include_black_ellipses and b.fullmove_number < n else ["."]
-                )
-            
-            if b.is_castling(mv):
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                b.pop()
-                out.append("O-O" if chess.square_file(mv.to_square) == 6 else "O-O-O")
-                if suf:
-                    out.append(suf)
-                b.push(mv)
-            else:
-                piece = b.piece_at(mv.from_square).symbol().upper()
-                frm = chess.square_name(mv.from_square)
-                to = chess.square_name(mv.to_square)
-                is_cap = b.is_capture(mv)
-                promo = mv.promotion
-                
-                b.push(mv)
-                suf = "#" if b.is_checkmate() else ("+" if b.is_check() else "")
-                
-                # Emit LAN tokens
-                out.append(piece)
-                out.append(frm)
-                if is_cap:
-                    out.append("x")
-                out.append(to)
-                if promo:
-                    out += ["=", chess.piece_symbol(promo).upper()]
-                if suf:
-                    out.append(suf)
-            
-            if b.turn == chess.WHITE:
-                n += 1
-        
-        res = g.headers.get("Result")
-        if self._keep_res and res in _RESULT:
-            out.append(res)
-        
-        return out
-    
-    def _lan_move_to_tokens(self, move: str) -> List[str]:
-        """
-        Convert a single LAN move to tokens.
-        
-        LAN format: [Piece][from_square][x]?[to_square][=Promo]?[+#]?
-        
-        Examples:
-            "Ng1f3" -> ["N", "g1", "f3"]
-            "Nd4xe6" -> ["N", "d4", "x", "e6"]
-            "Pe2e4" -> ["P", "e2", "e4"]
-            "Pe4xd5" -> ["P", "e4", "x", "d5"]
-            "O-O" -> ["O-O"]
-            "O-O-O" -> ["O-O-O"]
-            "Pe7e8=Q" -> ["P", "e7", "e8", "=", "Q"]
-            "Ng1f3+" -> ["N", "g1", "f3", "+"]
-        """
-        # Handle castling
-        if move in {"O-O", "O-O-O"}:
-            return [move]
-        if move.rstrip("+#") in {"O-O", "O-O-O"}:
-            base = move.rstrip("+#")
-            suffix = move[len(base):]
-            return [base] + ([suffix] if suffix else [])
-        
-        out = []
-        i = 0
-        n = len(move)
-        
-        # Get piece letter (required in LAN format)
-        if i < n and move[i] in "KQRBNP":
-            out.append(move[i])
-            i += 1
-        else:
-            # No piece letter - might be malformed, return as-is
-            return [move]
-        
-        # Get from square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle capture
-        if i < n and move[i] == "x":
-            out.append("x")
-            i += 1
-        
-        # Get to square (required in LAN format)
-        if i + 1 < n and move[i] in FILES and move[i + 1] in RANKS:
-            out.append(move[i:i+2])
-            i += 2
-        
-        # Handle promotion
-        if i < n and move[i] == "=":
-            out.append("=")
-            i += 1
-            if i < n and move[i] in PROMOS:
-                out.append(move[i])
-                i += 1
-        
-        # Handle check/checkmate
-        if i < n and move[i] in "+#":
-            out.append(move[i])
-            i += 1
-        
-        return out
-    
-    def _active_env_tokens(self) -> set:
-        """Return the set of env tokens that are active for this instance."""
-        return set(ENV_TOKENS) if self._include_env_tokens else set()
-
-    def _cot_to_tokens(self, text: str) -> List[str]:
-        """
-        Convert CoT formatted text to tokens.
-        Handles special tokens and LAN moves.
-        """
-        env_toks = self._active_env_tokens()
-        out = []
-        for token in text.split():
-            if token in {self.T, self.T_END, self.SEP} or token in env_toks:
-                # Keep special tokens as-is
-                out.append(token)
-            elif token in _RESULT:
-                # Game result
-                out.append(token)
-            elif token and token[0].isdigit() and "." in token:
-                # Move number like "1." or "15..."
-                # Split into digits and dots
-                num_part = token.rstrip(".")
-                dot_part = token[len(num_part):]
-                out.extend(list(num_part))
-                if dot_part:
-                    out.append("..." if len(dot_part) > 1 else ".")
-            elif token and all(c.isdigit() for c in token):
-                # Pure number - tokenize each digit
-                out.extend(list(token))
-            else:
-                # LAN move - tokenize it
-                out.extend(self._lan_move_to_tokens(token))
-        return out
-    
-    def encode(self, text: str) -> List[int]:
-        """
-        Encode text to token IDs.
-        
-        Args:
-            text: Text to encode (can be PGN or CoT formatted)
-        
-        Returns:
-            List of token IDs
-        """
-        # Check if this is CoT-formatted text (contains special tokens)
-        sft_special = (
-            [self.T, self.T_END, self.SEP]
-            + (ENV_TOKENS if self._include_env_tokens else [])
-        )
-        is_cot_format = any(token in text for token in sft_special)
-        
-        if is_cot_format:
-            t_idx = text.index(self.T)
-            prompt_part = text[:t_idx].strip()
-            rest_part = text[t_idx:]  # starts with <T>
-
-            pgn_tokens = self._pgn_to_tokens(prompt_part) if prompt_part else None
-            if pgn_tokens is None:
-                pgn_tokens = self._cot_to_tokens(prompt_part) if prompt_part else []
-            rest_tokens = self._cot_to_tokens(rest_part)
-            tokens = [self._bos] + pgn_tokens + rest_tokens + [self._eos]
-        else:
-            pgn_tokens = self._pgn_to_tokens(text)
-            if pgn_tokens is not None and len(pgn_tokens) > 0:
-                tokens = [self._bos] + pgn_tokens + [self._eos]
-            else:
-                # Not valid PGN — treat each word as a LAN move
-                lan_tokens = []
-                for word in text.split():
-                    lan_tokens.extend(self._lan_move_to_tokens(word))
-                tokens = [self._bos] + lan_tokens + [self._eos]
-        
-        return self.tk.encode(" ".join(tokens)).ids
-    
-    def decode(self, ids: List[int]) -> str:
-        """
-        Decode token IDs to text.
-        
-        Args:
-            ids: List of token IDs
-        
-        Returns:
-            Decoded text
-        """
-        toks = [t for t in self.tk.decode(ids).split() if t not in {self._bos, self._eos}]
-        
-        # Otherwise, use LAN decoding logic
-        out: List[str] = []
-        i, n = 0, len(toks)
-        
-        while i < n:
-            t = toks[i]
-            
-            if t in {self.T, self.T_END, self.SEP} or t in _RESULT or t in self._active_env_tokens():
-                out.append(t)
-                i += 1
-                continue
-            
-            if t and all(ch in DIGITS for ch in t):
-                j = i
-                num = []
-                while j < n and all(ch in DIGITS for ch in toks[j]):
-                    num.append(toks[j])
-                    j += 1
-                dots = ""
-                if j < n and toks[j] in {".", "..."}:
-                    dots = toks[j]
-                    j += 1
-                out.append("".join(num) + dots)
-                i = j
-                continue
-            
-            if t in {"O-O", "O-O-O"}:
-                j = i + 1
-                suf = toks[j] if j < n and toks[j] in {"+", "#"} else ""
-                if suf:
-                    j += 1
-                out.append(t + suf)
-                i = j
-                continue
-            
-            if t in set("KQRBNP"):
-                piece = t
-                j = i + 1
-                frm = toks[j] if j < n else ""
-                j += 1
-                cap = ""
-                if j < n and toks[j] == "x":
-                    cap = "x"
-                    j += 1
-                to = toks[j] if j < n else ""
-                j += 1
-                promo = ""
-                if j + 1 <= n - 1 and toks[j] == "=" and toks[j + 1] in set(PROMOS):
-                    promo = "=" + toks[j + 1]
-                    j += 2
-                suf = ""
-                if j < n and toks[j] in {"+", "#"}:
-                    suf = toks[j]
-                    j += 1
-                lan = f"{piece}{frm}{cap}{to}{promo}{suf}"
-                out.append(lan)
-                i = j
-                continue
-            
-            out.append(t)
-            i += 1
-        
-        return " ".join(out)
-    
-    def get_vocab(self) -> Dict[str, int]:
-        """Get token-to-id vocabulary mapping."""
-        return self._tok2id
-    
-    def bos_id(self) -> Optional[int]:
-        """Get BOS token ID."""
-        return self._tok2id[self._bos]
-    
-    def eos_id(self) -> Optional[int]:
-        """Get EOS token ID."""
-        return self._tok2id[self._eos]
-    
-    def pad_id(self) -> Optional[int]:
-        """Get PAD token ID (uses BOS as pad by default)."""
-        return self._tok2id.get("<pad>", self.bos_id())
-    
-    def get_vocab_size(self) -> int:
-        """Get vocabulary size."""
-        return len(self._tok2id)
-    
-    def t_id(self) -> int:
-        """Get <T> token ID."""
-        return self._tok2id[self.T]
-    
-    def sep_id(self) -> int:
-        """Get <sep> token ID."""
-        return self._tok2id[self.SEP]
-    
-    def t_end_id(self) -> int:
-        """Get </T> token ID."""
-        return self._tok2id[self.T_END]
-    
-    # ------------------------------------------------------------------
-    # Environment / reward token accessors
-    # ------------------------------------------------------------------
-
-    def _require_env_tokens(self) -> None:
-        if not self._include_env_tokens:
-            raise ValueError(
-                "Environment tokens are not enabled. "
-                "Pass include_env_tokens=True in the config."
-            )
-
-    def call_env_id(self) -> int:
-        """Get <call_env> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[CALL_ENV_TOKEN]
-
-    def verify_id(self) -> int:
-        """Get <verify> token ID."""
-        self._require_env_tokens()
-        return self._tok2id[VERIFY_TOKEN]
-
-    def reward_pos_id(self) -> int:
-        """Get <+1> (positive reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_POS_TOKEN]
-
-    def reward_neg_id(self) -> int:
-        """Get <-1> (negative reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_NEG_TOKEN]
-
-    def reward_zero_id(self) -> int:
-        """Get <0> (zero reward) token ID."""
-        self._require_env_tokens()
-        return self._tok2id[REWARD_ZERO_TOKEN]
-
-    def reward_id(self, value) -> int:
-        """
-        Get reward token ID by numeric value.
-
-        Args:
-            value: 1, -1, or 0  (or the strings "+1", "-1", "0")
-
-        Returns:
-            Token ID for the corresponding reward token.
-        """
-        self._require_env_tokens()
-        mapping = {1: REWARD_POS_TOKEN, -1: REWARD_NEG_TOKEN, 0: REWARD_ZERO_TOKEN,
-                   "+1": REWARD_POS_TOKEN, "-1": REWARD_NEG_TOKEN, "0": REWARD_ZERO_TOKEN}
-        if value not in mapping:
-            raise ValueError(f"reward value must be one of 1, -1, 0 (or '+1', '-1', '0'), got {value!r}")
-        return self._tok2id[mapping[value]]
-
-    def env_token_ids(self) -> Dict[str, int]:
-        """Get mapping of all env/reward special tokens to their IDs."""
-        self._require_env_tokens()
-        return {tok: self._tok2id[tok] for tok in ENV_TOKENS}
-    
-    def extract_parts(self, text: str) -> Tuple[Optional[str], Optional[List[str]], str]:
-        """
-        Extract prompt, trajectories and answer from BoN CoT formatted text.
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            prompt: The prompt/context (or None if not present)
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        if self.T not in text:
-            return None, None, text
-        
-        try:
-            # Split by <T> to get prompt, thinking section, and answer
-            t_parts = text.split(self.T)
-            if len(t_parts) < 3:
-                return None, None, text
-            
-            # t_parts[0] is prompt (before first <T>)
-            # t_parts[1] is the thinking section with trajectories
-            # t_parts[2] is the answer
-            prompt = t_parts[0].strip() if t_parts[0].strip() else None
-            thinking_section = t_parts[1].strip()
-            answer = t_parts[2].strip()
-            
-            # Split thinking section by <sep> to get trajectories
-            trajectories = [t.strip() for t in thinking_section.split(self.SEP) if t.strip()]
-            
-            return prompt, trajectories, answer
-        except (ValueError, IndexError):
-            return None, None, text
-    
-    def extract_thinking_and_answer(self, text: str) -> Tuple[Optional[List[str]], str]:
-        """
-        Extract trajectories and answer from BoN CoT formatted text (ignores prompt).
-        
-        Args:
-            text: Text in format: {prompt} <T> <sep> {traj1} <sep> ... <sep> <T> {answer}
-        
-        Returns:
-            trajectories: List of trajectory strings (or None if not present)
-            answer: The final answer
-        """
-        _, trajectories, answer = self.extract_parts(text)
-        return trajectories, answer
-    
-    def get_sft_special_tokens(self) -> List[str]:
-        """Get list of SFT special tokens (including env/reward tokens if enabled)."""
-        toks = [self.T, self.T_END, self.SEP]
-        if self._include_env_tokens:
-            toks += ENV_TOKENS
-        return toks
-
-    def get_sft_token_ids(self) -> Dict[str, int]:
-        """Get mapping of SFT special tokens to their IDs."""
-        result = {
-            self.T: self._tok2id[self.T],
-            self.T_END: self._tok2id[self.T_END],
-            self.SEP: self._tok2id[self.SEP],
-        }
-        if self._include_env_tokens:
-            for tok in ENV_TOKENS:
-                result[tok] = self._tok2id[tok]
-        return result
-    
-    def parse_cot_line(self, line: str) -> Tuple[Optional[List[str]], Optional[str]]:
-        """
-        Parse a CoT data line in format: <T> <sep> ... <sep> <T> {answer}
-        
-        Args:
-            line: A line from the CoT data file
-        
-        Returns:
-            trajectories: List of trajectory strings
-            answer: The final answer/move
-        """
-        line = line.strip()
-        if not line or not line.startswith(self.T):
-            return None, None
-        
-        return self.extract_thinking_and_answer(line)
-
-# ============================================================
-# HuggingFace-compatible wrapper (auto-generated)
-# ============================================================
-import json as _json
-from pathlib import Path as _Path
-from transformers import PreTrainedTokenizer
-import torch
-from transformers.tokenization_utils_base import BatchEncoding
-
-from huggingface_hub import hf_hub_download
-
-class HFTokenizerWrapper(PreTrainedTokenizer):
-    def __init__(self, model_max_length=2048, **kwargs):
-        # These are usually provided by from_pretrained
-        repo_id = kwargs.get("name_or_path") or kwargs.get("_name_or_path")
-        revision = kwargs.get("revision", None)
-
-        if not repo_id or "/" not in str(repo_id):
-            # Fallback: user may pass repo_id explicitly
-            repo_id = kwargs.get("repo_id", None)
-        if not repo_id:
-            raise ValueError("Cannot infer repo_id; pass repo_id=... or ensure name_or_path is set.")
-
-        import os
-        if os.path.isdir(repo_id):
-            vocab_path = os.path.join(repo_id, "vocab.json")
-            cfg_path   = os.path.join(repo_id, "tokenizer_config.json")
-        else:
-            vocab_path = hf_hub_download(repo_id=repo_id, filename="vocab.json", revision=revision)
-            cfg_path   = hf_hub_download(repo_id=repo_id, filename="tokenizer_config.json", revision=revision)
-
-        with open(vocab_path, "r", encoding="utf-8") as _f:
-            saved_vocab = _json.load(_f)
-        with open(cfg_path, "r", encoding="utf-8") as _f:
-            _tok_cfg = _json.load(_f)
-
-        lan_config = _tok_cfg.get("lan_config", {})
-        lan_class_name = _tok_cfg.get("lan_tokenizer_class", "LanTokenizerSFT")
-
-        _cls = globals()[lan_class_name]
-        custom_tokenizer = _cls(config=lan_config)
-
-        # Override vocab with the saved vocab
-        custom_tokenizer._tok2id = saved_vocab
-        from tokenizers import Tokenizer as _TkTokenizer
-        from tokenizers.models import WordLevel as _WordLevel
-        from tokenizers.pre_tokenizers import WhitespaceSplit as _WhitespaceSplit
-        custom_tokenizer.tk = _TkTokenizer(_WordLevel(vocab=saved_vocab, unk_token=custom_tokenizer._unk))
-        custom_tokenizer.tk.pre_tokenizer = _WhitespaceSplit()
-
-        self.custom_tokenizer = custom_tokenizer
-        self._vocab = dict(saved_vocab)
-        self._id_to_token = {i: t for t, i in self._vocab.items()}
-
-        bos_token = _tok_cfg.get("bos_token")
-        eos_token = _tok_cfg.get("eos_token")
-        pad_token = _tok_cfg.get("pad_token")
-        unk_token = _tok_cfg.get("unk_token")
-        env_token = _tok_cfg.get("env_token")
-        if "env_id" in _tok_cfg:
-            env_token = self._id_to_token[_tok_cfg.get("env_id")]
-        else:
-            env_token = _tok_cfg.get("env_token")
-        self.env_token = env_token
-
-        for _key in ("bos_token","eos_token","pad_token","unk_token","env_token",
-                     "model_max_length","name_or_path","lan_config",
-                     "lan_tokenizer_class","tokenizer_class","auto_map","use_fast",
-                     "revision","repo_id"):
-            kwargs.pop(_key, None)
-
-        super().__init__(
-            bos_token=bos_token,
-            eos_token=eos_token,
-            pad_token=pad_token,
-            unk_token=unk_token,
-            model_max_length=model_max_length,
-            **kwargs,
-        )
-
-    # ---- PreTrainedTokenizer interface ----
-
-    @property
-    def vocab_size(self):
-        return len(self._vocab)
-
-    def get_vocab(self):
-        return dict(self._vocab)
-
-    def _tokenize(self, text):
-        return []  # we override encode/decode directly
-
-    def _convert_token_to_id(self, token):
-        return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
-
-    def _convert_id_to_token(self, index):
-        return self._id_to_token.get(index, self.unk_token or "")
-
-    def convert_tokens_to_string(self, tokens):
-        ids = [self._convert_token_to_id(t) for t in tokens]
-        return self.custom_tokenizer.decode(ids)
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        if token_ids_1 is None:
-            return token_ids_0
-        return token_ids_0 + token_ids_1
-
-    def encode(self, text, add_special_tokens=True, **kwargs):
-        ids = self.custom_tokenizer.encode(text)
-        if add_special_tokens:
-            return ids[:-1]  # strip trailing EOS; vLLM adds its own
-        if (len(ids) >= 2
-                and self.bos_token_id is not None
-                and self.eos_token_id is not None
-                and ids[0] == self.bos_token_id
-                and ids[-1] == self.eos_token_id):
-            return ids[1:-1]
-        return ids
-
-    def decode(self, token_ids, skip_special_tokens=True, **kwargs):
-        import numpy as np
-        if isinstance(token_ids, torch.Tensor):
-            token_ids = token_ids.detach().cpu().tolist()
-        elif isinstance(token_ids, np.ndarray):
-            token_ids = token_ids.tolist()
-        return self.custom_tokenizer.decode(token_ids)
-
-    def save_vocabulary(self, save_directory, filename_prefix=None):
-        save_directory = _Path(save_directory)
-        save_directory.mkdir(parents=True, exist_ok=True)
-        vocab_file = save_directory / (
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json"
-        )
-        with open(vocab_file, "w", encoding="utf-8") as f:
-            _json.dump(self._vocab, f, ensure_ascii=False, indent=2)
-        return (str(vocab_file),)
-
-    def __call__(
-        self,
-        text,
-        text_pair=None,
-        add_special_tokens=True,
-        truncation=False,
-        max_length=None,
-        padding=False,
-        return_tensors=None,
-        **kwargs,
-    ):
-        if text_pair is not None:
-            raise ValueError("text_pair not supported for this tokenizer.")
-
-        # Normalize to batch
-        is_batched = isinstance(text, (list, tuple))
-        texts = list(text) if is_batched else [text]
-
-        input_ids = [self.encode(t, add_special_tokens=add_special_tokens) for t in texts]
-
-        # Truncation
-        if truncation and max_length is not None:
-            if self.truncation_side == "left":
-                input_ids = [ids[-max_length:] for ids in input_ids]
-            else:
-                input_ids = [ids[:max_length] for ids in input_ids]
-
-        # Attention masks (pre-padding)
-        attention_mask = [[1] * len(ids) for ids in input_ids]
-
-        # Padding
-        if padding:
-            if padding == "max_length":
-                if max_length is None:
-                    raise ValueError("padding='max_length' requires max_length.")
-                pad_to = max_length
-            else:
-                pad_to = max(len(ids) for ids in input_ids) if input_ids else 0
-
-            pad_id = self.pad_token_id
-            if pad_id is None:
-                pad_id = self.bos_token_id if self.bos_token_id is not None else 0
-
-            for i, ids in enumerate(input_ids):
-                pad_len = pad_to - len(ids)
-                if pad_len > 0:
-                    input_ids[i] = ids + [pad_id] * pad_len
-                    attention_mask[i] = attention_mask[i] + [0] * pad_len
-
-        data = {"input_ids": input_ids, "attention_mask": attention_mask}
-
-        # Unbatch if single example and no tensor return
-        if not is_batched and return_tensors is None:
-            data = {"input_ids": data["input_ids"][0], "attention_mask": data["attention_mask"][0]}
-
-        # Tensors
-        if return_tensors == "pt":
-            data = {k: torch.tensor(v, dtype=torch.long) for k, v in data.items()}
-
-        return BatchEncoding(data, tensor_type=None)
-
-
-__all__ = ["HFTokenizerWrapper"]

C6p5e18_410m_alpha0.200_beta0.045/step_110/tokenizer_config.json

@@ -1,26 +0,0 @@
-{
-  "tokenizer_class": "HFTokenizerWrapper",
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenizer.HFTokenizerWrapper",
-      null
-    ]
-  },
-  "model_max_length": 2048,
-  "bos_token": "<bos>",
-  "eos_token": "<eos>",
-  "pad_token": "<bos>",
-  "unk_token": "<unk>",
-  "env_token": null,
-  "use_fast": false,
-  "lan_config": {
-    "name": "LanTokenizerSFT",
-    "include_move_numbers": false,
-    "include_black_tripledots": false,
-    "bos": "<bos>",
-    "eos": "<eos>",
-    "unk": "<unk>",
-    "keep_result": false
-  },
-  "lan_tokenizer_class": "LanTokenizerSFT"
-}