File size: 3,018 Bytes
bb02ad0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | # Example remote-training host file for `cascade-trainer --remote-hosts`.
#
# This file is TRAINER-LOCAL — keep it out of git and OFF the public chain.toml.
# It lists the SSH GPU pods (Lium, Targon, or any host you can SSH into) that the
# trainer dispatches a round's king and challenger to, in parallel. The first
# host trains the king, the second the challenger (more hosts → a round-robin
# pool for multi-challenger rounds).
#
# Each pod must already have: cascade installed (`pip install -e '.[train,hippius]'`),
# torch + a GPU, and registry/S3 access. The pod does NOT need the bittensor
# wallet — signing/publishing stays on the orchestrator. Seed the pod's env once
# when you rent it (HIPPIUS_HUB_TOKEN, HIPPIUS_S3_ACCESS_KEY,
# HIPPIUS_S3_SECRET_KEY) OR list them under forward_env to have the orchestrator
# pass them over SSH.
#
# WANDB_API_KEY is special: when [wandb] enabled=true, the trainer AUTO-forwards
# it to every pod (the training — and its wandb run — happen on the pod), so you
# do NOT need to list it under forward_env. It's only forwarded when present in
# the orchestrator's own env; without it, pod-side wandb silently no-ops and the
# runs show no per-step training logs.
#
# For BYTE-EXACT re-derivation, rent the SAME GPU SKU on both pods and pin it in
# chain.toml [training] expected_gpu (the reference trainer runs deterministically;
# the validator rejects any round whose two checkpoints weren't trained on it).
#
# STAGE TAGS (optional, default "any"): stage = "heat" | "final" | "any" restricts
# which round stage a pod serves. Heats can run on a CHEAPER SKU class (e.g.
# A6000) than the final — heat checkpoints are trainer-internal (screened and
# discarded, never validated), and the fixed token budget keeps every heat
# entrant on identical compute regardless of SKU. Two rules:
# * keep each stage's pods a single SKU (heat entrants stay numerically
# comparable; the final's king/challenger must match for the validator gate),
# * only the FINAL pods need the expected_gpu SKU from chain.toml.
# Untagged ("any") hosts serve both stages — the pre-stage behaviour.
[[host]]
name = "king-box"
host = "203.0.113.10" # pod public IP / hostname
port = 22
user = "root"
key_path = "~/.ssh/lium_ed25519"
remote_python = "/root/cascade/.venv/bin/python"
workdir = "/root/cascade" # where cascade is installed on the pod
cuda_device = "0" # pins CUDA_VISIBLE_DEVICES on the pod
# chain_toml = "/root/cascade/chain.toml" # only if non-default on the pod
# forward_env = ["HIPPIUS_HUB_TOKEN", "HIPPIUS_S3_ACCESS_KEY", "HIPPIUS_S3_SECRET_KEY"]
# ssh_options = ["ServerAliveInterval=30", "ServerAliveCountMax=120"]
[[host]]
name = "challenger-box"
host = "203.0.113.11"
port = 22
user = "root"
key_path = "~/.ssh/targon_ed25519"
remote_python = "/root/cascade/.venv/bin/python"
workdir = "/root/cascade"
cuda_device = "0"
|