scripts/provision.example.toml

# Example config for `cascade-provisioner` — the per-round GPU pod renter.
#
#   cascade-provisioner --config provision.toml --chain-toml chain.toml \
#       --work-root /root/cascade/_train_work [--dry-run] [--once]
#
# This file is OPERATOR-LOCAL: keep it out of git (it names your SSH key and
# hosts file) and OFF the public chain.toml. Round shape (epoch cadence, final
# training hours, the image-digest pin) is read from chain.toml — this file
# only holds the RENTAL policy: what to rent, where, and how much to spend.
#
# The provisioner holds NO wallet — only provider API keys (env:
# SHADEFORM_API_KEY / LIUM_API_KEY, see deploy/cascade-provisioner.service)
# and the orchestrator's SSH keypair. Signing/publishing stays on the trainer.

[provisioner]
# Where the trainer's --remote-hosts points. The provisioner writes this file
# atomically per round (heat + final entries), re-renders it final-only when
# the heat settles, and EMPTIES it when it can't rent — an empty file is the
# trainer's explicit "train locally this round" signal.
hosts_path = "/root/cascade/hosts.toml"

# Crash-safe rental ledger (default: <work-root>/provisioner_state.json).
# Restarts resume teardown from here; delete it only if you have verified by
# hand that no cascade-* pod is still running on any provider.
state_path = "/root/cascade/_train_work/provisioner_state.json"

# Digest-pinned worker image (see docs/DEPLOY_PODS.md §1). MUST be @sha256 —
# a moving tag breaks the expected_gpu / train_image_digest contract.
image = "ghcr.io/tensorlink-ai/cascade-worker@sha256:0000000000000000000000000000000000000000000000000000000000000000"

# ── OR: bootstrap mode (no worker image published yet) ───────────────────────
# With bootstrap_script set, pods rent BARE (image becomes the provider's
# template to boot, digest pin not required) and the script — run ON the
# orchestrator with POD_IP/POD_PORT/POD_USER/POD_KEY/POD_STAGE/POD_WORKDIR in
# env — provisions them over SSH (rsync source + `uv sync --frozen`; see
# deploy/pod_bootstrap.sh). The health gate then verifies what it built.
# bootstrap_script    = "/root/cascade/deploy/pod_bootstrap.sh"
# bootstrap_timeout_s = 1800
# pod_user            = "root"
# Shadeform in bootstrap mode needs VM-mode launches: the account's registered
# SSH key id (GET /v1/sshkeys) — pods boot as bare VMs, user "shadeform".
# shadeform_ssh_key_id = "00000000-0000-0000-0000-000000000000"

# Operator's static [[host]] entries (a long-lived hand-rented pod, e.g. the
# final box) — validated at startup with the trainer's own loader, appended
# verbatim to EVERY hosts.toml publish, and NEVER dropped by provisioner
# activity (teardown re-renders, no-capacity rounds, budget refusals).
# static_hosts = "/root/cascade/static_hosts.toml"

# ── elastic validator eval pod (optional; see [provisioner.eval] below) ──────
# Where the VALIDATOR's --eval-hosts points — a SEPARATE file from hosts_path
# (the two consumers have different lifecycles; the provisioner refuses equal
# paths). Published when a round's manifest appears, cleared on teardown; the
# validator re-reads it lazily at each offloaded eval, so clearing mid-round
# is safe (its next eval just runs locally). Unset = eval stage off.
# eval_hosts_path = "/root/cascade/eval_hosts.toml"
# The eval pod's primary teardown signal: this deployment's validator
# publishes round receipts at receipts/<validator_hotkey>/round-<id>.json —
# when the served round's receipt appears the pod is torn down. A newer
# manifest (round superseded) and the TTL below also end it. Unset = only
# those two backstops apply.
# receipt_prefix = "receipts/5ValidatorHotkey.../"

# Orchestrator SSH keypair: the PUBLIC key is injected into each pod
# ($SSH_PUBKEY); the PRIVATE key path is written into hosts.toml for the
# trainer's dispatches (and used by the provisioner's own health checks).
ssh_pubkey   = "~/.ssh/cascade_ed25519.pub"     # inline "ssh-…" also accepted
ssh_key_path = "~/.ssh/cascade_ed25519"

# Rent inside the last N blocks of the epoch (~12s each; 25 ≈ 5 min). Must sit
# within the trainer's reveal margin — the field is only countable once timed
# reveals have landed — and be < epoch_blocks. Pods finish booting AFTER the
# round starts; the trainer's --hosts-wait-seconds covers that gap.
trigger_margin_blocks = 25

# HARD per-round spend ceiling in USD, projected at WORST case (every pod
# billed for the full TTL). If the projection exceeds this, the round rents
# NOTHING and the trainer trains locally — a runaway field can never surprise
# you on the bill.
max_spend_per_round = 25.0

# Pod time-to-live in epochs — the no-cooperation teardown backstop. Even if
# the heat marker, the manifest, and the ledger are ALL lost, every pod dies
# this many epochs after rent.
ttl_epochs = 1

# Rules of escalation. A stage whose rung delivers NO healthy pod (the launch
# call failed, or every pod and its one replacement was a dud) re-enters the
# SKU ladder at the next (candidate × provider) rung — capacity re-probed at
# escalation time, each rung re-checked against max_spend_per_round — for at
# most escalate_deadline_s of wall clock per attempt. Renting runs in a
# worker thread (the poll loop keeps tearing down / heartbeating); the
# deadline bounds the single worker so the JIT final, the retries, and the
# next round's trigger are never starved (0 = a failed rung is final).
# A PARTIAL fleet below min_viable_fleet of its slot demand gets ONE
# same-candidate top-up batch — never a different SKU (0 = never top up).
escalate_deadline_s = 1800
min_viable_fleet    = 0.5

# Within-round retry: a stage that rented NOTHING (no capacity anywhere, over
# budget, or every pod a dud past escalation) re-attempts the full
# pick→budget→rent pipeline on this cadence, for as long as the stage can
# still matter — the heat while one serial screening wave fits in its
# remaining window (the fleet re-sizes to that window), the final while its
# full training hours + boot margin remain. 0 = one attempt per round.
rent_retry_cooldown_s = 900

# When to rent the FINAL fleet. "margin": with the heat at the epoch boundary
# (the default). "heat_complete": just-in-time at the trainer's
# heat_complete.json marker — the fleet is sized off the marker's ACTUAL
# finalist list and the pinned-SKU pods stop idle-billing through the heat.
# Requires a trainer new enough to re-read hosts before the duel (it waits
# --hosts-wait-seconds for final-tagged entries; boot is 10-15 min). If the
# primary final rung probes scarce at the margin, the final rents early via
# the ladder anyway — the thin-market exception.
final_rent_on = "margin"

# Health-gate knobs (see cascade/provision/health.py for the full seven).
min_disk_gb = 20.0

# Pod-side layout (matches deploy/Dockerfile).
remote_python = "/root/cascade/.venv/bin/python"
workdir       = "/root/cascade"
# chain_toml  = "/root/cascade/chain.testnet.toml"   # only if non-default on the pod
poll_seconds  = 30

# ── heat fleet: cheap SKU, sized off the revealed field ──────────────────────
# Heat checkpoints are trainer-internal (screened and discarded, never
# validated), so heats run on whatever is cheap — here 8×A6000 clusters. The
# fleet is SLOT-sized: ceil(field × heat_hours × slot_overhead / window) GPU
# slots, then ceil(slots / gpus_per_pod) pods, capped at max_pods. sku is the
# EXACT `nvidia-smi --query-gpu=name` string — the health gate byte-compares
# every GPU of every pod against it.
# max_pods caps PODS ACROSS ALL SHAPES; pod count is always slot-derived, so a
# high cap lets small shapes scale out without over-renting big ones. max_pods
# = 0 marks the stage UNMANAGED (served entirely by static_hosts entries).
# market_sku is the marketplace's alias when it differs from nvidia-smi's
# device string ("A6000" on lium vs "NVIDIA RTX A6000" on the pod).
[provisioner.heat]
sku           = "NVIDIA GeForce RTX 4090"
market_sku    = "RTX4090"
gpus_per_pod  = 4            # 4x cluster: one hosts.toml entry per GPU
max_pods      = 8
providers     = ["lium", "shadeform"]   # priority order; a stage never splits providers
max_price_hr  = 2.60         # USD per POD-hour; dearer offers are skipped
slot_overhead = 1.3          # pads for image pull / corpus build / one flaky retry

# Ordered SKU/shape fallbacks: the round takes the FIRST candidate × provider
# with capacity for the whole heat fleet (pod count re-derived per shape). A
# stage never mixes candidates, so within-round fairness holds by construction;
# heat scores only rank within a round, so cross-round SKU variance is free.
[[provisioner.heat.candidate]]
sku          = "NVIDIA RTX A6000"
market_sku   = "A6000"
gpus_per_pod = 4
max_price_hr = 2.40

[[provisioner.heat.candidate]]           # bigger cluster if that's what's stocked
sku          = "NVIDIA GeForce RTX 4090"
market_sku   = "RTX4090"
gpus_per_pod = 8
max_price_hr = 5.00

[[provisioner.heat.candidate]]           # last resort: 2x is the floor — 1x
sku          = "NVIDIA GeForce RTX 4090"  # singles pay a full bootstrap per
market_sku   = "RTX4090"                  # lane and rarely beat the trainer's
gpus_per_pod = 2                          # local fallback
max_price_hr = 1.30

# ── final fleet: the pinned SKU, king + finalists on ONE pod ─────────────────
# Default shape is a single pod with gpus_per_pod >= 1 + [round] finalists:
# every final run reports the same physical box's GPU, so the validator's
# expected_gpu pairing holds by construction. One 2×L40S pod = king + 1
# finalist. (gpus_per_pod = 1 falls back to one pod per run.)
[provisioner.final]
sku          = "NVIDIA L40S"
market_sku   = "L40S"
gpus_per_pod = 2
max_pods     = 2             # 0 = unmanaged: a static hand-rented pod serves finals
providers    = ["lium", "shadeform"]
max_price_hr = 3.0

# ── eval pod: the validator's per-round offload box (OPTIONAL) ───────────────
# Manifest-triggered, unlike heat/final: rented when a round's manifest
# publishes (that is when the validator's GIFT-Eval gate + cascade bench need
# GPU — exactly as the trainer fleet is torn down), torn down when the round's
# receipt appears under receipt_prefix, a newer manifest supersedes it, or the
# TTL fires. One pod (max_pods = 1), one GPU is plenty. Absent table (or
# max_pods = 0, or eval_hosts_path unset) = feature off: validators eval
# locally on their --device, the pre-elastic behaviour. [[provisioner.eval.
# candidate]] SKU fallbacks work exactly like the heat's.
# [provisioner.eval]
# sku          = "NVIDIA L40S"
# market_sku   = "L40S"
# gpus_per_pod = 1
# max_pods     = 1
# providers    = ["lium", "shadeform"]
# max_price_hr = 1.20

# ── per-provider pod paths (bootstrap mode) ──────────────────────────────────
# Providers boot different base users: lium pods are root, shadeform VMs land
# as the "shadeform" user. Absent providers use the global defaults above.
# [provisioner.pods.shadeform]
# user          = "shadeform"
# workdir       = "/home/shadeform/cascade"
# remote_python = "/home/shadeform/cascade/.venv/bin/python"