scripts/remote_hosts.example.toml

# Example remote-training host file for `cascade-trainer --remote-hosts`.
#
# This file is TRAINER-LOCAL — keep it out of git and OFF the public chain.toml.
# It lists the SSH GPU pods (Lium, Targon, or any host you can SSH into) that the
# trainer dispatches a round's king and challenger to, in parallel. The first
# host trains the king, the second the challenger (more hosts → a round-robin
# pool for multi-challenger rounds).
#
# Each pod must already have: cascade installed (`pip install -e '.[train,hippius]'`),
# torch + a GPU, and registry/S3 access. The pod does NOT need the bittensor
# wallet — signing/publishing stays on the orchestrator. Seed the pod's env once
# when you rent it (HIPPIUS_HUB_TOKEN, HIPPIUS_S3_ACCESS_KEY,
# HIPPIUS_S3_SECRET_KEY) OR list them under forward_env to have the orchestrator
# pass them over SSH.
#
# WANDB_API_KEY is special: when [wandb] enabled=true, the trainer AUTO-forwards
# it to every pod (the training — and its wandb run — happen on the pod), so you
# do NOT need to list it under forward_env. It's only forwarded when present in
# the orchestrator's own env; without it, pod-side wandb silently no-ops and the
# runs show no per-step training logs.
#
# For BYTE-EXACT re-derivation, rent the SAME GPU SKU on both pods and pin it in
# chain.toml [training] expected_gpu (the reference trainer runs deterministically;
# the validator rejects any round whose two checkpoints weren't trained on it).
#
# STAGE TAGS (optional, default "any"): stage = "heat" | "final" | "any" restricts
# which round stage a pod serves. Heats can run on a CHEAPER SKU class (e.g.
# A6000) than the final — heat checkpoints are trainer-internal (screened and
# discarded, never validated), and the fixed token budget keeps every heat
# entrant on identical compute regardless of SKU. Two rules:
#   * keep each stage's pods a single SKU (heat entrants stay numerically
#     comparable; the final's king/challenger must match for the validator gate),
#   * only the FINAL pods need the expected_gpu SKU from chain.toml.
# Untagged ("any") hosts serve both stages — the pre-stage behaviour.

[[host]]
name          = "king-box"
host          = "203.0.113.10"        # pod public IP / hostname
port          = 22
user          = "root"
key_path      = "~/.ssh/lium_ed25519"
remote_python = "/root/cascade/.venv/bin/python"
workdir       = "/root/cascade"     # where cascade is installed on the pod
cuda_device   = "0"                    # pins CUDA_VISIBLE_DEVICES on the pod
# chain_toml  = "/root/cascade/chain.toml"   # only if non-default on the pod
# forward_env = ["HIPPIUS_HUB_TOKEN", "HIPPIUS_S3_ACCESS_KEY", "HIPPIUS_S3_SECRET_KEY"]
# ssh_options = ["ServerAliveInterval=30", "ServerAliveCountMax=120"]

[[host]]
name          = "challenger-box"
host          = "203.0.113.11"
port          = 22
user          = "root"
key_path      = "~/.ssh/targon_ed25519"
remote_python = "/root/cascade/.venv/bin/python"
workdir       = "/root/cascade"
cuda_device   = "0"