# Example remote-training host file for `cascade-trainer --remote-hosts`. # # This file is TRAINER-LOCAL — keep it out of git and OFF the public chain.toml. # It lists the SSH GPU pods (Lium, Targon, or any host you can SSH into) that the # trainer dispatches a round's king and challenger to, in parallel. The first # host trains the king, the second the challenger (more hosts → a round-robin # pool for multi-challenger rounds). # # Each pod must already have: cascade installed (`pip install -e '.[train,hippius]'`), # torch + a GPU, and registry/S3 access. The pod does NOT need the bittensor # wallet — signing/publishing stays on the orchestrator. Seed the pod's env once # when you rent it (HIPPIUS_HUB_TOKEN, HIPPIUS_S3_ACCESS_KEY, # HIPPIUS_S3_SECRET_KEY) OR list them under forward_env to have the orchestrator # pass them over SSH. # # WANDB_API_KEY is special: when [wandb] enabled=true, the trainer AUTO-forwards # it to every pod (the training — and its wandb run — happen on the pod), so you # do NOT need to list it under forward_env. It's only forwarded when present in # the orchestrator's own env; without it, pod-side wandb silently no-ops and the # runs show no per-step training logs. # # For BYTE-EXACT re-derivation, rent the SAME GPU SKU on both pods and pin it in # chain.toml [training] expected_gpu (the reference trainer runs deterministically; # the validator rejects any round whose two checkpoints weren't trained on it). # # STAGE TAGS (optional, default "any"): stage = "heat" | "final" | "any" restricts # which round stage a pod serves. Heats can run on a CHEAPER SKU class (e.g. # A6000) than the final — heat checkpoints are trainer-internal (screened and # discarded, never validated), and the fixed token budget keeps every heat # entrant on identical compute regardless of SKU. Two rules: # * keep each stage's pods a single SKU (heat entrants stay numerically # comparable; the final's king/challenger must match for the validator gate), # * only the FINAL pods need the expected_gpu SKU from chain.toml. # Untagged ("any") hosts serve both stages — the pre-stage behaviour. [[host]] name = "king-box" host = "203.0.113.10" # pod public IP / hostname port = 22 user = "root" key_path = "~/.ssh/lium_ed25519" remote_python = "/root/cascade/.venv/bin/python" workdir = "/root/cascade" # where cascade is installed on the pod cuda_device = "0" # pins CUDA_VISIBLE_DEVICES on the pod # chain_toml = "/root/cascade/chain.toml" # only if non-default on the pod # forward_env = ["HIPPIUS_HUB_TOKEN", "HIPPIUS_S3_ACCESS_KEY", "HIPPIUS_S3_SECRET_KEY"] # ssh_options = ["ServerAliveInterval=30", "ServerAliveCountMax=120"] [[host]] name = "challenger-box" host = "203.0.113.11" port = 22 user = "root" key_path = "~/.ssh/targon_ed25519" remote_python = "/root/cascade/.venv/bin/python" workdir = "/root/cascade" cuda_device = "0"