fly-trader distribution: code, fly #62, selector #59, connectome, seed (branch distribution @ 5cb1652)

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.venv
+.env
+data
+logs
+dist
+web
+media
+tests
+.pytest_cache
+.ruff_cache
+__pycache__
+*.pyc
+.agents
+.claude

.env.example

@@ -0,0 +1,50 @@
+# fly-trader distribution -- copy to .env and fill in the two API keys. Everything else is set for a 0.5 SOL wallet.
+# Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines.
+
+# --- keys (never logged) ---
+# Helius: https://dashboard.helius.dev (RPC for balances and transactions)
+HELIUS_API_KEY=
+# Jupiter: https://portal.jup.ag (swaps and token stats)
+JUPITER_API_KEY=
+# Leave empty. `docker compose exec fly fly-trader wallet new` writes it here.
+BOT_PRIVATE_KEY=
+
+# --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) ---
+DATABASE_URL=postgresql:///fly_trader
+
+# --- money ---
+# Live money. Until the wallet is funded the engine trades paper and says so on the console.
+LIVE_ENABLED=1
+# What you fund the wallet with; also the paper warm-up bankroll.
+CAPITAL_SOL=0.5
+# Hard cap per position.
+MAX_POSITION_SOL=0.05
+# Per position, of the bankroll minus the gas reserve.
+MAX_POSITION_FRACTION=0.10
+# Per position, of the pool's SOL reserve (bounds price impact).
+MAX_POOL_SHARE=0.02
+# Smaller sizes are skipped.
+MIN_POSITION_SOL=0.02
+# Never spent (fees and token-account rent).
+GAS_RESERVE_SOL=0.05
+KELLY_FRACTION=0.25
+
+# --- rails ---
+# 20 % below the wallet's peak: stop entering ...
+KILL_SWITCH_DRAWDOWN=0.20
+# ... and sell everything open. Clear it on the console's "Safety & wallet" page or with `fly-trader reset-circuit --kill`.
+KILL_SWITCH_LIQUIDATE=1
+
+# --- paper warm-up, then live ---
+# No minimum racing time; after 20 closed paper trades the fly trades the wallet; no race against the paper selector.
+HANDOVER_DAYS=0
+HANDOVER_MIN_TRADES=20
+HANDOVER_BEAT_SELECTOR=0
+# A restart keeps the paper book, so the warm-up count survives it.
+RESET_ON_START=0
+
+# --- runtime ---
+SOLANA_CLUSTER=mainnet-beta
+DEVICE=cpu
+DATA_DIR=data
+LOG_DIR=logs

.gitignore

@@ -0,0 +1,18 @@
+.env
+.venv/
+data/
+logs/
+dist/
+models/wallet_skill/*.parquet   # ~300 MB: on Hugging Face, fetched by docker/entrypoint.sh
+__pycache__/
+*.pyc
+.pytest_cache/
+*.egg-info/
+.streamlit/secrets.toml
+.DS_Store
+.agents/
+.claude/
+
+# large media assets - deployed straight to Gandi, not tracked in GitHub source repo
+web/assets/*.mp4
+media/*.mp4

Dockerfile

@@ -0,0 +1,21 @@
+# fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU).
+FROM python:3.13-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    UV_PROJECT_ENVIRONMENT=/app/.venv \
+    PATH="/app/.venv/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app
+COPY . .
+# CPU-only torch first (the default Linux wheel pulls CUDA, ~3 GB); the project's own pin is then already satisfied.
+RUN uv venv /app/.venv \
+    && uv pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.12" \
+    && uv pip install . \
+    && chmod +x /app/docker/entrypoint.sh
+
+EXPOSE 8501
+ENTRYPOINT ["/app/docker/entrypoint.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 fly-trader contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,150 @@
+---
+license: mit
+pipeline_tag: reinforcement-learning
+library_name: pytorch
+tags:
+  - solana
+  - memecoin
+  - pump.fun
+  - trading
+  - connectome
+  - drosophila
+  - flywire
+---
+
+# fly-trader — distribution
+
+**A fruit-fly connectome that trades Solana memecoins with your money.**
+
+> ## ⚠️ Read this first
+>
+> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun, holds them
+> for about two hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few
+> months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is
+> financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no
+> warranty of any kind. See [LICENSE](LICENSE).
+
+This is the ready-to-run branch: the trading engine, its console, a trained fly, the selector that taught it, and the
+connectome they run on. The training pipeline lives on [`master`](https://github.com/bryceweiner/fly-trader).
+
+## What it does
+
+- Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute
+  candles (a token needs that much history before its features mean anything).
+- The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses —
+  predicts each token's net return over the next 120 minutes. It was taught by a gradient-boosted selector; it then keeps
+  learning from what its own trades earn, through plasticity at its mushroom body's KC→MBON synapses.
+- It buys when the predicted return clears its own buy line, sizes the position from measured certainty (a quarter
+  of the growth-optimal bet, capped), holds 120 minutes, sells.
+- **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that,
+  and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console.
+- **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three
+  failed transactions in a row trip a circuit breaker; a stale market feed halts entries; if live fills trail the paper
+  mirror by more than 2 points per trade over 50 trades, entries pause and it tells you.
+
+## Requirements
+
+- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database.
+- A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key.
+- 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5.
+- Any CPU. No GPU is used.
+
+## Start trading
+
+1. Download this branch and open a terminal in it:
+   ```bash
+   git clone -b distribution https://github.com/bryceweiner/fly-trader.git && cd fly-trader
+   ```
+2. Create your `.env` and paste the two API keys into it:
+   ```bash
+   cp .env.example .env
+   ```
+3. Start it:
+   ```bash
+   docker compose up -d --build
+   ```
+   The first start builds the image (several minutes), creates the database, installs the models and starts the market
+   feed and the trading engine.
+4. Create the bot wallet — it prints the public key and writes the secret into your `.env`:
+   ```bash
+   docker compose exec fly fly-trader wallet new
+   ```
+   **Back up `.env` now.** It is the only copy of that key.
+5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key:
+   ```bash
+   docker compose restart fly
+   ```
+6. Open the console: <http://localhost:8501>. Watch the **Overview** page.
+
+That is all. From here it runs on its own. The first ~24 hours are a warm-up (it needs a day of candles before it
+scores anything); then it trades paper until 20 trades have closed; then it trades the wallet.
+
+## Day to day
+
+| Want to | Do |
+|---|---|
+| See what it is doing | <http://localhost:8501> — **Overview** (books, open positions, wealth), **Trades**, **Safety & wallet** (rails, wallet), **Processes** |
+| Pause new entries | Safety & wallet → **Pause entries**, or `docker compose exec fly fly-trader pause-entries` |
+| Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) |
+| Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself |
+| See the logs | `docker compose logs -f fly` |
+| Start again from nothing | `docker compose down -v` (deletes the database) and delete `data/` |
+
+The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails.
+
+## What is in the box
+
+| Path | What |
+|---|---|
+| `models/policies/fly_*.pt` | the fly (`brain_snapshots` #62): taught by selector #59, two strategies (`ev`, `capitulation`), 120-minute hold, its own buy lines |
+| `models/selectors/selector_*.joblib` | the selector that taught it (#59), kept so the console can show the paper selector alongside |
+| `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale |
+| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with) and the name of the newest per-day table. The table itself (~300 MB) is on Hugging Face; the container fetches it once at first start |
+| `seed/seed.sql` | the database rows that admit the fly to trade: the model records, the passed replay verdict, autostart |
+| `.env.example` | every knob, set for 0.5 SOL |
+| `docker-compose.yml`, `Dockerfile`, `docker/entrypoint.sh` | one Postgres, one engine + console |
+
+The fly's replay verdict shipped here: over the 28 evaluation days it made +1.52 % per trade after real costs (profit
+factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast.
+
+## Tuning
+
+Edit `.env`, then `docker compose restart fly`. The ones that matter:
+
+| Variable | Shipped | Meaning |
+|---|---|---|
+| `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll |
+| `MAX_POSITION_SOL` / `MAX_POSITION_FRACTION` | 0.05 / 0.10 | per-position caps (the smaller wins) |
+| `GAS_RESERVE_SOL` | 0.05 | never spent |
+| `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything |
+| `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live |
+| `LIVE_ENABLED` | 1 | set 0 to run paper only |
+
+Scaling the wallet up scales the position caps with it (`MAX_POSITION_FRACTION`), but `MAX_POSITION_SOL` is a hard
+ceiling: raise it deliberately or not at all.
+
+## Refreshing the model
+
+The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a
+passed replay:
+
+```bash
+.venv/bin/python tools/make_seed.py --out dist
+cp -R dist/models/* models/ && cp -R dist/seed/* seed/
+```
+
+Commit `models/` and `seed/` on this branch. The seed is idempotent: an install that already has these rows is not
+changed by it, so pull the branch, `docker compose up -d --build`, and the new model is in place.
+
+## Also on Hugging Face
+
+- Model and this code: [`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader)
+- The training database (the full corpus, for retraining on `master`): [`bryceweiner/fly-trader-seed`](https://huggingface.co/datasets/bryceweiner/fly-trader-seed)
+
+## Credits
+
+FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel
+et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain)
+(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag).
+
+[MIT](LICENSE).

config/mbon_valence.json

@@ -0,0 +1,27 @@
+{
+ "_citation": "Approach = MBON-11 (gamma1pedc>alpha/beta, GABA), 12, 13, 14, 15, 16, 18 (cholinergic) and avoidance = MBON-01..07 (all glutamatergic): Aso et al. 2014 eLife 3:e04580 (activation screen; 'all MBONs eliciting aversion were glutamatergic and all the MBONs eliciting attraction were either GABAergic or cholinergic'); NT per type from Aso et al. 2014 eLife 3:e04577 Table 1. MBON17 (alpha'3m) was tested jointly with MBON16 (alpha'3ap) in that screen. MBON19 and MBON23 are NOT in these lists: MBON19's NT is N.D. in Aso 2014 and MBON23 is a later hemibrain type (Li et al. 2020). MBON21, MBON29 (cholinergic, aversive) and MBON33 (attractive) per Rubin & Aso 2024 eLife 90523 (activation results read from partial text; verify). '-like' annotations are left unassigned (no valence evidence). PAM = reward, PPL1 = punishment: Aso et al. 2014 eLife 3:e04577 compartment map; Li et al. 2020 ('PPL1 DANs generally convey punishment, PAM DANs generally convey reward'). Learning = depression of opposite-valence KC->MBON synapses: Hige et al. 2015 Neuron, Owald et al. 2015 Neuron.",
+ "approach": [
+  "MBON11",
+  "MBON12",
+  "MBON13",
+  "MBON14",
+  "MBON15",
+  "MBON16",
+  "MBON17",
+  "MBON18",
+  "MBON33"
+ ],
+ "avoid": [
+  "MBON01",
+  "MBON02",
+  "MBON03",
+  "MBON04",
+  "MBON05",
+  "MBON06",
+  "MBON07",
+  "MBON21",
+  "MBON29"
+ ],
+ "dan_reward_prefix": "PAM",
+ "dan_punishment_prefix": "PPL1"
+}

docker-compose.yml

@@ -0,0 +1,39 @@
+# fly-trader, fire-and-forget: one Postgres, one trading container. The console is published on 127.0.0.1 only --
+# it can start and stop the workers and clear the kill switch, so it must never face the internet.
+services:
+  db:
+    image: postgres:17
+    environment:
+      POSTGRES_USER: fly
+      POSTGRES_PASSWORD: fly
+      POSTGRES_DB: fly_trader
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    restart: unless-stopped
+
+  fly:
+    build: .
+    env_file: .env
+    environment:
+      DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader
+      DATA_DIR: /app/data
+      LOG_DIR: /app/logs
+      DEVICE: cpu
+    volumes:
+      - ./data:/app/data        # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds
+      - ./logs:/app/logs
+      - ./.env:/app/.env        # `fly-trader wallet new` writes the bot key here
+    ports:
+      - "127.0.0.1:8501:8501"
+    depends_on:
+      db:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  pgdata:

docker/entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# First boot: schema, the shipped models into the data volume, the seed rows that admit them. Every boot: the console,
+# which autostarts the two trading workers (market feed + trading engine). Nothing here overwrites a running install.
+set -e
+cd /app
+
+until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do
+  echo "fly-trader: waiting for postgres"; sleep 2
+done
+
+fly-trader init-db
+
+for sub in policies selectors connectome; do
+  mkdir -p "data/brain/$sub"
+  for f in models/$sub/*; do
+    [ -e "$f" ] || continue
+    [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/"
+  done
+done
+
+# The wallet-skill inputs: the fitted config (in git) and the newest table (~300 MB: kept on Hugging Face, fetched once).
+mkdir -p data/corpus/wallet_skill
+for f in models/wallet_skill/*; do
+  [ -e "$f" ] || continue
+  [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/
+done
+T=$(cat models/wallet_skill/TABLE 2>/dev/null || true)
+if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then
+  echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once)"
+  curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/bryceweiner/fly-trader/resolve/main/models/wallet_skill/$T" \
+    && mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T" \
+    || echo "fly-trader: WARNING could not fetch the wallet-skill table; the paper selector will not trade and the fly's skill inputs read as zero until it is present"
+fi
+
+psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql
+
+echo "fly-trader: console at http://localhost:8501"
+exec fly-trader ui --port "${PORT:-8501}"

fly_trader/__init__.py

@@ -0,0 +1,3 @@
+"""fly-trader: a FlyWire-connectome memecoin trader on Solana."""
+
+__version__ = "0.1.0"

fly_trader/__main__.py

@@ -0,0 +1,229 @@
+"""fly-trader command line. Every command is a thin wrapper over library functions that the
+Streamlit console also calls. Always run as ``.venv/bin/python -m fly_trader <command>``."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def _cmd_init_db(args):
+    from .db import schema
+    from .db.connection import database_name
+    created = schema.ensure_database()
+    version = schema.apply_schema()
+    tables = schema.table_names()
+    print(f"database={database_name()} created={created} schema_version={version} tables={len(tables)}")
+    for t in tables:
+        print(f"  {t}")
+
+
+def _cmd_wallet(args):
+    from .chain import keys
+    if args.action == "new":
+        print(keys.create_wallet())
+    else:
+        print(keys.show_wallet())
+
+
+def _cmd_discover(args):
+    from .ingest import discovery
+    if args.once:
+        discovery.run_once()
+    else:
+        discovery.run_forever()
+
+
+def _cmd_build_connectome(args):
+    from .brain import connectome_build
+    connectome_build.main(annotations=args.annotations)
+
+
+def _cmd_run(args):
+    from .agent import runner
+    runner.main()
+
+
+def _cmd_swap_smoke(args):
+    from .execution import broker_live
+    broker_live.swap_smoke(args.sol)
+
+
+def _cmd_status(args):
+    from .db import queries
+    queries.print_status()
+
+
+def _cmd_reset_circuit(args):
+    from .agent import rails
+    rails.reset_circuit(kill=args.kill)
+
+
+def _cmd_pause(args):
+    from .agent import rails
+    rails.set_entries_paused(True)
+
+
+def _cmd_resume(args):
+    from .agent import rails
+    rails.set_entries_paused(False)
+
+
+def _cmd_reset_training(args):
+    from .ops.reset import reset_training_state
+    import json
+    print(json.dumps(reset_training_state(reason="cli", archive=not args.no_archive), indent=1))
+
+
+def _cmd_ui(args):
+    from .ops.supervisor import run_console
+    run_console(port=args.port)
+
+
+def _cmd_verify_fills(args):
+    from .execution import ledger
+    ledger.verify_fills()
+
+
+def _cmd_close_atas(args):
+    from .execution import broker_live
+    broker_live.close_empty_atas()
+
+
+def _cmd_replay_pull(args):
+    from .ingest import replay_pull
+    replay_pull.main()
+
+
+def _cmd_pumpstream(args):
+    from .ingest import pumpstream
+    pumpstream.main()
+
+
+def _cmd_build_corpus_features(args):
+    from .train import corpus_features
+    corpus_features.main()
+
+
+def _cmd_refetch_pool_ids(args):
+    from .ingest import replay_pull
+    print(f"{replay_pull.refetch_missing_pool_id()} archive hours marked for re-download (run the replay worker or fly-trader replay-pull)")
+
+
+def _cmd_fit_wallet_skill(args):
+    from .logging_setup import setup
+    from .train import wallet_skill
+    setup("train")
+    out = wallet_skill.fit(days=args.days)
+    print(json.dumps({k: out.get(k) for k in ("h", "L", "selection", "evaluation", "baseline", "beats_baseline", "candidates", "stopped")}, indent=1, default=str))
+
+
+def _cmd_train_selector(args):
+    from .logging_setup import setup
+    from .train import selector
+    setup("train")
+    selector.main(days=args.days)
+
+
+def _cmd_train_fly_selector(args):
+    from .logging_setup import setup
+    from .train import fly_selector
+    setup("train")
+    fly_selector.main(days=args.days, epochs=args.epochs)
+
+
+def _cmd_fly_replay(args):
+    from .logging_setup import setup
+    from .train import fly_replay
+    setup("train")
+    out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START)
+    print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str))
+
+
+def _cmd_selector_eval(args):
+    import runpy
+    runpy.run_module("fly_trader.train.selector_eval", run_name="__main__")
+
+
+def _cmd_build_mature(args):
+    from .train import mature
+    mature.main()
+
+
+def _cmd_build_corpus_meta(args):
+    from .train import corpus_meta
+    corpus_meta.main()
+
+
+def _cmd_assemble_replay(args):
+    from .train import replay_assemble
+    replay_assemble.main()
+
+
+def _cmd_backtest_corpus(args):
+    from .train import corpus_backtest
+    argv = ["--fees", args.fees]
+    if args.max_tokens: argv += ["--max-tokens", str(args.max_tokens)]
+    if args.days: argv += ["--days", str(args.days)]
+    if args.only: argv += ["--only", args.only]
+    argv += ["--universe", args.universe]
+    corpus_backtest.main(argv)
+
+
+def _cmd_worker(args):
+    from .ops import procs
+    procs.worker_cli(args.action, args.name)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="fly-trader")
+    sub = p.add_subparsers(dest="command", required=True)
+    sub.add_parser("init-db").set_defaults(fn=_cmd_init_db)
+    w = sub.add_parser("wallet"); w.add_argument("action", choices=["new", "show"]); w.set_defaults(fn=_cmd_wallet)
+    d = sub.add_parser("discover"); d.add_argument("--once", action="store_true"); d.set_defaults(fn=_cmd_discover)
+    sub.add_parser("replay-pull").set_defaults(fn=_cmd_replay_pull)
+    sub.add_parser("refetch-pool-ids", help="mark archive hours without pool_id for re-download").set_defaults(fn=_cmd_refetch_pool_ids)
+    fw = sub.add_parser("fit-wallet-skill", help="fit the wallet-skill tables' hold and lookback by the selector's objective (one-off, many hours, resumable)")
+    fw.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)"); fw.set_defaults(fn=_cmd_fit_wallet_skill)
+    sub.add_parser("pumpstream").set_defaults(fn=_cmd_pumpstream)
+    sub.add_parser("build-corpus-features").set_defaults(fn=_cmd_build_corpus_features)
+    sub.add_parser("assemble-replay").set_defaults(fn=_cmd_assemble_replay)
+    sub.add_parser("build-corpus-meta").set_defaults(fn=_cmd_build_corpus_meta)
+    sub.add_parser("build-mature").set_defaults(fn=_cmd_build_mature)
+    sub.add_parser("selector-eval").set_defaults(fn=_cmd_selector_eval)
+    ts_ = sub.add_parser("train-selector"); ts_.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)")
+    ts_.set_defaults(fn=_cmd_train_selector)
+    tf_ = sub.add_parser("train-fly-selector"); tf_.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)")
+    tf_.add_argument("--epochs", type=int, default=None, help="cap on full passes over the distillation rows (default: until the fit stops improving)")
+    tf_.set_defaults(fn=_cmd_train_fly_selector)
+    fr = sub.add_parser("fly-replay", help="bootstrap the fly once, let it learn over the corpus, judge it (the gate before it trades)")
+    fr.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)")
+    fr.add_argument("--start-day", type=int, default=None,
+                    help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, "
+                         "the earliest day that leaves the teacher's walk-forward enough days in each half")
+    fr.set_defaults(fn=_cmd_fly_replay)
+    bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None)
+    bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus)
+    b = sub.add_parser("build-connectome"); b.add_argument("--annotations", default=None); b.set_defaults(fn=_cmd_build_connectome)
+    sub.add_parser("run").set_defaults(fn=_cmd_run)
+    s = sub.add_parser("swap-smoke"); s.add_argument("--sol", type=float, default=0.01); s.set_defaults(fn=_cmd_swap_smoke)
+    sub.add_parser("status").set_defaults(fn=_cmd_status)
+    r = sub.add_parser("reset-circuit"); r.add_argument("--kill", action="store_true"); r.set_defaults(fn=_cmd_reset_circuit)
+    sub.add_parser("pause-entries").set_defaults(fn=_cmd_pause)
+    sub.add_parser("resume-entries").set_defaults(fn=_cmd_resume)
+    u = sub.add_parser("ui"); u.add_argument("--port", type=int, default=8501); u.set_defaults(fn=_cmd_ui)
+    rt = sub.add_parser("reset-training"); rt.add_argument("--no-archive", action="store_true"); rt.set_defaults(fn=_cmd_reset_training)
+    sub.add_parser("verify-fills").set_defaults(fn=_cmd_verify_fills)
+    sub.add_parser("close-empty-atas").set_defaults(fn=_cmd_close_atas)
+    wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker)
+    return p
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    rc = args.fn(args)
+    return rc if isinstance(rc, int) else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fly_trader/agent/__init__.py

@@ -0,0 +1 @@
+

fly_trader/agent/fly_live.py

@@ -0,0 +1,159 @@
+"""Live execution for the fly once it holds the selector's seat: its decisions also trade the bot wallet (book 'live').
+
+The paper book ``paper_fly`` keeps running as the mirror. Each trade minute — only after the handover and only while
+signing is allowed (LIVE_ENABLED=1 and the live prerequisites, chain/cluster_guard.py) — ``LiveMirror.minute``:
+- drains the execution worker (execution/worker.py books live positions from balance-verified fills);
+- exits: live positions held the hold are sold (exit slippage, the forced ladder once ``OVERDUE_S`` late), retried every
+  minute; ``DEAD_BAG_S`` past the hold a position that could not be sold is written off (dead-bag: closed at 0, so it
+  leaves wealth) and its tokens are swept every ``SWEEP_S``;
+- entries: every paper entry of this minute is mirrored with the same score, line and certainty bands, sized from the
+  wallet (SOL balance + open live positions at cost as the bankroll, the SOL balance as cash; the gas reserve is never
+  spent — agent/sizing.py), one position per token, unless the kill switch, the circuit breaker or paused entries block;
+- wealth: wallet SOL + open positions net of exit cost → ``wealth_marks`` (book 'live'); the kill switch on its own peak,
+  and with ``KILL_SWITCH_LIQUIDATE`` every open position is sold at the forced slippage while it stays on;
+- gap: over the last ``GAP_TRADES`` closed live trades, the mean return per trade trailing the paper mirror's same
+  decisions by more than ``GAP_MAX`` pauses entries and raises an alert. Learning is unaffected: it learns from market
+  labels, not fills.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timedelta, timezone
+
+import numpy as np
+
+from .. import config
+from ..chain.balances import snapshot_balances
+from ..db.apilog import record_event
+from ..execution import ledger
+from ..execution.worker import ExecRequest
+from ..market.exit_cost import exit_cost_fraction
+from . import rails, sizing
+
+log = logging.getLogger(__name__)
+BOOK, MIRROR = "live", "paper_fly"
+DEAD_BAG_S, OVERDUE_S, SWEEP_S = 3 * 3600.0, 600.0, 3600.0
+GAP_TRADES, GAP_MAX = 50, 0.02
+
+
+class LiveMirror:
+    def __init__(self, horizon_s: float, broker=None, worker=None):
+        if broker is None:
+            from ..execution.broker_live import LiveBroker
+            broker = LiveBroker()
+        if worker is None:
+            from ..execution.worker import ExecutionWorker
+            worker = ExecutionWorker(broker)
+        self.broker, self.worker, self.H = broker, worker, float(horizon_s)
+        self.last_sweep = 0.0
+
+    def _decision(self, conn, ctx, run_id, beat_id, p, reason: str, forced: bool) -> int:
+        return int(conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, size_sol, forced, reason, detail) VALUES (%s,%s,%s,%s,%s,'fly_exit',%s,%s,%s,%s) RETURNING id",
+                                (beat_id, run_id, ctx.m1, p["mint"], p["pool"], float(p["cost_sol"]), forced, reason, json.dumps({"book": BOOK}))).fetchone()["id"])
+
+    def minute(self, ctx, *, run_id: str, beat_id: int, entries: list, line: float, table: list | None) -> dict:
+        conn, m1 = ctx.conn, ctx.m1
+        drained = self.worker.drain()
+        for r in drained:
+            if not r.ok:
+                log.warning("live %s %s failed: %s", r.request.side, r.request.mint, r.error)
+        inflight = self.worker.pending()
+        snap = snapshot_balances(self.broker.rpc, self.broker.pubkey); sol_free = snap.lamports / config.LAMPORTS_PER_SOL
+        n_exit = n_dead = 0
+        for p in ledger.open_positions(conn, BOOK):
+            held = (m1 - p["opened_at"]).total_seconds(); H = float(p.get("hold_s") or self.H)          # each position's own hold
+            if held < H or p["mint"] in inflight:
+                continue
+            if held >= H + DEAD_BAG_S:
+                did = self._decision(conn, ctx, run_id, beat_id, p, f"dead-bag: unsold {held / 3600:.1f} h after entry", True)
+                ledger.close_position(conn, position_id=int(p["id"]), exit_price=0.0, proceeds_sol=0.0, fees_sol=0.0, decision_id=did, forced_kind="dead_bag", ts=m1)
+                record_event("warning", "fly_live", f"dead-bag: {p['mint']} could not be sold; written off, its tokens are swept hourly", {"position": int(p["id"]), "cost_sol": float(p["cost_sol"])})
+                n_dead += 1
+                continue
+            overdue = held >= H + OVERDUE_S
+            did = self._decision(conn, ctx, run_id, beat_id, p, f"held {held / 60:.0f} min" + (" (retrying)" if overdue else ""), False)
+            n_exit += bool(self.worker.submit(ExecRequest(decision_id=did, mint=p["mint"], pool=p["pool"], side="sell", amount_in=int(p["qty"]),
+                                                          slippage_bps=config.SLIPPAGE_FORCED_BPS if overdue else config.SLIPPAGE_EXIT_BPS,
+                                                          max_slippage_bps=config.SLIPPAGE_FORCED_BPS, decimals=int(p.get("decimals") or 6), position_id=int(p["id"]))))
+        opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight)
+        c = rails.load_circuit(conn)
+        blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None
+        bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens); cash = sol_free; n_enter = 0; skipped = []
+        for e in entries:
+            if e["mint"] in held_mints:
+                skipped.append((e["mint"], "held")); continue
+            size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"])
+            if blocked or size <= 0:
+                skipped.append((e["mint"], blocked or why)); continue
+            if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL),
+                                              slippage_bps=config.SLIPPAGE_ENTRY_BPS, max_slippage_bps=config.MAX_SLIPPAGE_ENTRY_BPS, decimals=int(e["info"]["decimals"] or 6), size_sol=size)):
+                n_enter += 1; cash -= size; held_mints.add(e["mint"])
+                conn.execute("UPDATE decisions SET book_targets = array_append(COALESCE(book_targets, '{}'), 'live') WHERE id = %s", (e["decision_id"],))
+        ledger.mark_positions(conn, BOOK, ctx.prices, {}, m1)
+        gross_v = exit_cost = exposure = 0.0
+        for p in ledger.open_positions(conn, BOOK):
+            px = float(p.get("last_mark_price") or p["entry_price"] or 0.0)
+            gross = int(p["qty"]) / 10 ** int(p.get("decimals") or 6) * px
+            gross_v += gross; exposure += float(p["cost_sol"])
+            exit_cost += gross * exit_cost_fraction(gross, ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"]), ctx.mcap(p["mint"], px), p.get("program_label"), ctx.fees.get(p["mint"]))
+        wealth = sol_free + gross_v - exit_cost
+        pk = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (BOOK,)).fetchone(); peak = max(float(pk["pk"] or 0.0), wealth)
+        conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) "
+                     "ON CONFLICT (beat_id, book) DO NOTHING",
+                     (beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
+        killed = rails.check_drawdown(conn, wealth, peak)
+        n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0
+        gap = self.gap(conn)
+        if ctx.m1_epoch - self.last_sweep >= SWEEP_S:
+            self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1)
+        return {"stage": "trading live", "wallet_sol": sol_free, "wealth": wealth, "entered": n_enter, "exited": n_exit, "dead_bags": n_dead, "in_flight": len(inflight),
+                "blocked": blocked, "skipped": skipped[:10], "gap": gap, "failed_orders": sum(1 for r in drained if not r.ok), "liquidated": n_liq}
+
+    def liquidate(self, conn, ctx, run_id: str, beat_id: int) -> int:
+        """Sell every open live position at the forced slippage while the kill switch is on (``KILL_SWITCH_LIQUIDATE``):
+        a book that has lost ``KILL_SWITCH_DRAWDOWN`` of its peak stops holding, not only entering. Called every trade
+        minute the switch stays on, so a sell that fails is retried until the book is flat; positions already in flight
+        (this minute's scheduled exits included) are left to the worker."""
+        n = 0; inflight = self.worker.pending()
+        for p in ledger.open_positions(conn, BOOK):
+            if p["mint"] in inflight:
+                continue
+            did = self._decision(conn, ctx, run_id, beat_id, p, "kill switch: liquidating", True)
+            n += bool(self.worker.submit(ExecRequest(decision_id=did, mint=p["mint"], pool=p["pool"], side="sell", amount_in=int(p["qty"]),
+                                                     slippage_bps=config.SLIPPAGE_FORCED_BPS, max_slippage_bps=config.SLIPPAGE_FORCED_BPS,
+                                                     decimals=int(p.get("decimals") or 6), position_id=int(p["id"]))))
+        if n:
+            record_event("error", "fly_live", f"kill switch: liquidating {n} open live position(s)", {"positions": n})
+        return n
+
+    def gap(self, conn) -> dict:
+        """Live vs its paper mirror on the same decisions; pauses entries when live trails by more than ``GAP_MAX`` per trade."""
+        rows = conn.execute(f"""SELECT l.realized_sol / NULLIF(l.cost_sol, 0) AS rl, p.realized_sol / NULLIF(p.cost_sol, 0) AS rp
+                               FROM positions l JOIN positions p ON p.entry_decision_id = l.entry_decision_id AND p.book = '{MIRROR}' AND p.status = 'closed'
+                               WHERE l.book = '{BOOK}' AND l.status = 'closed' AND l.entry_decision_id IS NOT NULL ORDER BY l.closed_at DESC LIMIT %s""", (GAP_TRADES,)).fetchall()
+        rl = np.array([r["rl"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64)
+        rp = np.array([r["rp"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64)
+        out = {"trades": int(len(rl)), "live_mean": float(rl.mean()) if len(rl) else None, "mirror_mean": float(rp.mean()) if len(rp) else None}
+        if len(rl) >= GAP_TRADES and rl.mean() - rp.mean() < -GAP_MAX:
+            c = rails.load_circuit(conn)
+            if not c.entries_paused:
+                conn.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = 1")
+                conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('gap_pause', %s)", (json.dumps(out),))
+                record_event("error", "fly_live", f"entries paused: live trails its paper mirror by {(rp.mean() - rl.mean()) * 100:.1f} points per trade over {len(rl)} trades", out)
+            out["paused"] = True
+        return out
+
+    def sweep(self, conn, snap, inflight: set, m1: datetime) -> int:
+        """Sell whatever is left of the last week's dead-bag tokens (forced slippage; proceeds reach the wallet)."""
+        n = 0
+        for r in conn.execute("SELECT DISTINCT mint FROM positions WHERE book = %s AND forced_exit_kind = 'dead_bag' AND closed_at > %s",
+                              (BOOK, m1 - timedelta(days=7))).fetchall():
+            amt = snap.token(r["mint"])
+            if amt > 0 and r["mint"] not in inflight:
+                n += bool(self.worker.submit(ExecRequest(decision_id=None, mint=r["mint"], pool=None, side="sell", amount_in=amt, slippage_bps=config.SLIPPAGE_FORCED_BPS,
+                                                         max_slippage_bps=config.SLIPPAGE_FORCED_BPS, decimals=int(snap.decimals.get(r["mint"], 6)), forced_kind="dead_bag_sweep")))
+        return n
+
+    def stop(self) -> None:
+        self.worker.stop()

fly_trader/agent/fly_session.py

@@ -0,0 +1,510 @@
+"""The plastic fly: trading its own book (``paper_fly``) on the minute engine and learning from the market every minute.
+
+It starts once the fly design passed its replay (``train/fly_replay.verdict``, current definitions) and a deployable
+bootstrap exists (``train/fly_selector.latest_deployable``); the replay's chosen configuration (α, half-life) is the
+learning rate and forgetting of its mushroom body (``brain/plastic.py``). Each minute (agent/minute_engine.py):
+
+1. bars: minute start, open, close and exit cost of every traded mint with a pending tag feed the label resolver;
+2. labels: tags whose exit minute has now been seen are resolved exactly as ``train/decisions.build`` labels a row
+   (``resolve_label``) and teach the mushroom body; a label that cannot be resolved is dropped;
+3. calibration: at the first minute of each UTC day, the line and sizing of the plastic fly and of its frozen shadow
+   (the bootstrap, D = 0) from their scores of the minutes resolved in the last seven days (``train/fly_calibrate.py``);
+4. scoring: eligible minutes are scored by the plastic fly and its shadow, stored (``fly_scored``: pending rows keep the
+   feature vector, so tags survive restarts) and tagged; only minutes at or above the line teach it (brain/plastic.py);
+5. trading (fresh stream, trade minutes): the race rules (``agent/paper_trading.py``) at the plastic line and sizing.
+
+Hourly: learning statistics (``fly_updates``), a snapshot of the plastic state (kind 'fly_plastic'; hourly for 7 days,
+then one per day), the rollback checks (``train/fly_governance.py``; a rollback restores the newest good snapshot at
+least a day old, or D = 0, pauses the checks a day, and the third within 7 days freezes learning and requests a
+re-bootstrap), and the handover (``HANDOVER_DAYS`` of racing, at least ``HANDOVER_MIN_TRADES`` trades, realized P&L
+at least the selector's). The state file ``data/brain/plastic/state.pt`` is written atomically every minute, before
+tags are marked resolved: on restart, pending tags at or before its watermark count as applied.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import math
+import os
+import time
+import uuid
+from collections import deque
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from .. import config
+from ..brain import plastic
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..execution import ledger
+from ..execution.broker_paper import PaperBroker
+from ..ops.reset import fly_state_dir
+from ..train import fly_calibrate, fly_governance as gov, fly_selector
+from ..train.decisions import X_COLS
+from . import handover, paper_trading
+
+log = logging.getLogger(__name__)
+BOOK, KIND = "paper_fly", "fly"
+BAR_KEEP = 270               # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin)
+NEXT_OPEN_S = 120.0          # entry at the next traded minute's open when it comes within two minutes (decisions.build)
+JUMP = 50.0
+LABEL_LAG_S = 60.0
+ROLLBACK_PAUSE_S, ROLLBACK_LIMIT, ROLLBACK_WINDOW_S, ROLLBACK_MIN_AGE_S = 86400.0, 3, 7 * 86400.0, 86400.0
+SNAP_HOURLY_DAYS, SCORED_KEEP_DAYS = 7, 35
+COMMAND_KEY = "fly_command"
+
+
+def send_command(cmd: str) -> None:
+    """Queue an operator command for the running fly (applied at its next minute): 'pause', 'resume' or 'rollback'."""
+    assert cmd in ("pause", "resume", "rollback"), cmd
+    with transaction() as conn:
+        conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                     (COMMAND_KEY, json.dumps({"cmd": cmd, "at": datetime.now(timezone.utc).isoformat()})))
+
+
+def resolve_label(bars: list, t: float, horizon_s: float) -> float | None:
+    """Net return of a row scored at minute start ``t`` from the mint's bars [(minute start, open, close, exit cost)]
+    in time order — ``decisions.build``'s ``fwd_pess``: entry at the next traded minute's open when it comes within
+    ``NEXT_OPEN_S`` (else the close), exit at the last close at or before ``t + horizon_s``, the one-side exit cost at
+    both ends. None when the row's own bar is missing, a price is not positive, or the exit minute jumped 50× from the
+    one before (the archive drops such exits)."""
+    rows = [b for b in bars if b[0] >= t]
+    if not rows or rows[0][0] != t:
+        return None
+    j = 0
+    for i, b in enumerate(rows):
+        if b[0] <= t + horizon_s:
+            j = i
+        else:
+            break
+    entry = rows[1][1] if len(rows) > 1 and rows[1][0] - t <= NEXT_OPEN_S and rows[1][1] else rows[0][2]
+    exit_px = rows[j][2]
+    if not entry or entry <= 0 or not exit_px or exit_px <= 0:
+        return None
+    if j > 0 and rows[j - 1][2] and not (1 / JUMP <= exit_px / rows[j - 1][2] <= JUMP):
+        return None
+    return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1)
+
+
+def try_start(live: bool = False) -> tuple["FlyBook | None", str]:
+    """A FlyBook when the fly may trade, else (None, why)."""
+    from ..train import fly_replay
+    v = fly_replay.verdict()
+    if not v:
+        return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)"
+    if not v.get("passed"):
+        return None, f"the fly's replay did not pass: {v.get('reason')}"
+    with transaction() as conn:
+        boot = fly_selector.latest_deployable(conn)
+    if boot is None:
+        return None, "no deployable fly bootstrap (the training pipeline bootstraps one)"
+    try:
+        return FlyBook(boot, v, live=live), "ok"
+    except Exception as e:
+        log.exception("fly book failed to start")
+        return None, f"fly book failed to start: {type(e).__name__}: {e}"
+
+
+class FlyBook:
+    name = "fly"
+
+    def __init__(self, boot: dict, verdict: dict, live: bool = False):
+        self.live = live; self.done = False; self.mirror = None
+        self.verdict = verdict
+        self.state_path = fly_state_dir() / "state.pt"
+        self._load_bootstrap(boot)
+        self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(BOOK)
+        with transaction() as conn:
+            conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
+                         (self.run_id, KIND, json.dumps({"channels": self.cfg, "book": BOOK}, default=str), self.boot_id))
+            self._restore_tags(conn)
+        record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"],
+                                                            "pending": len(self.pending), "learning_frozen": self.learning_frozen})
+        log.info("fly session: bootstrap %d, channels %s, lines %s, %d pending tags", self.boot_id, self.cfg, self.lines["plastic"], len(self.pending))
+
+    # ---- model and state ----
+    def _load_bootstrap(self, boot: dict) -> None:
+        self.boot_id = int(boot["id"])
+        self.fly = fly_selector.load(boot["path"])
+        self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int)
+        self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names}
+        self.H = max(self.holds.values())
+        per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}}
+        self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names}
+        self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf
+        net = self.fly.net
+        self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy())
+        for j, k in enumerate(self.names):                      # each strategy's channel learns and forgets at its own replay-chosen rate
+            a, h = self.cfg[k]; cols = self.bank.learn[j]
+            self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf")
+        self.pending = plastic.PendingTags(net.n_kc, net.k_active)
+        self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {}
+        self.lines = {"plastic": dict(self.fly.lines), "frozen": dict(self.fly.lines)}
+        self.sizing = {"plastic": {k: list(v) for k, v in self.fly.sizings.items()}, "frozen": {k: list(v) for k, v in self.fly.sizings.items()}}
+        self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names}
+        self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None
+        self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {}
+        s = self._read_state()
+        if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg:
+            self.bank.load_state(s["bank"]); self.nu_set = True
+            self.lines, self.sizing = s["lines"], s["sizing"]
+            self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"]
+            self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"]
+        elif s:
+            self.race_started_at = s.get("race_started_at")        # a new bootstrap keeps racing the same book
+            with transaction() as conn:
+                conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s", (self.boot_id,))
+
+    def _read_state(self) -> dict | None:
+        if not self.state_path.exists():
+            return None
+        try:
+            return torch.load(self.state_path, map_location="cpu", weights_only=False)
+        except Exception:
+            log.exception("fly state unreadable; starting from the bootstrap")
+            return None
+
+    def _state(self) -> dict:
+        return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing,
+                "applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until,
+                "race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks,
+                "saved_at": time.time()}
+
+    def _write_state(self) -> None:
+        self.state_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self.state_path.with_suffix(".tmp")
+        torch.save(self._state(), tmp); os.replace(tmp, self.state_path)
+
+    def _restore_tags(self, conn) -> None:
+        rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s ORDER BY ts", (self.boot_id,)).fetchall()
+        rows = [r for r in rows if r["strategy"] in self.holds]
+        applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through]
+        if applied:
+            with conn.cursor() as cur:
+                cur.executemany("UPDATE fly_scored SET state = 'resolved', resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", applied)
+        todo = [r for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S > self.applied_through and r["x"] is not None]
+        groups: dict[tuple, list] = {}
+        for r in todo:
+            groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r)
+        for (t, name), rs in sorted(groups.items()):
+            j = self.names.index(name)
+            Y, u0, k = self.fly.parts_all(np.asarray([r["x"] for r in rs], dtype=np.float32))
+            w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])]))
+            self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, Y[:, j], u0, k, w, s=j)
+            for r in rs:
+                self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1
+
+    def maybe_reload(self) -> bool:
+        """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on."""
+        from ..train import fly_replay
+        with transaction() as conn:
+            boot = fly_selector.latest_deployable(conn)
+        v = fly_replay.verdict()
+        if boot is None or v is None or not v.get("passed") or int(boot["id"]) == self.boot_id:
+            return False
+        old = self.boot_id; self.verdict = v
+        self._write_state()
+        self._load_bootstrap(boot)
+        with transaction() as conn:
+            conn.execute("UPDATE runs SET brain_snapshot_id = %s WHERE run_id = %s", (self.boot_id, self.run_id))
+        self._write_state()
+        record_event("info", "fly", f"switched to fly bootstrap #{self.boot_id}", {"from": old, "to": self.boot_id, "lines": self.lines["plastic"]})
+        return True
+
+    # ---- engine interface ----
+    def on_bars(self, t_start: float, bars: dict) -> None:
+        for m in self.watch:
+            b = bars.get(m)
+            if b is not None:
+                q = self.bars.setdefault(m, deque(maxlen=BAR_KEEP))
+                if not q or q[-1][0] < b[0]:
+                    q.append(b)
+
+    def open_mints(self, conn) -> set[str]:
+        return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
+
+    def on_minute(self, ctx) -> dict:
+        conn, t_now = ctx.conn, ctx.t_start
+        self._commands(conn, ctx.m1_epoch)
+        learned = self._learn(conn, ctx.m1_epoch)
+        day = int(t_now // 86400)
+        if self.last_day is None:
+            self.last_day = day
+        elif day > self.last_day:
+            self.last_day = day; self._calibrate(conn, ctx.m1_epoch)
+        scored = self._score(conn, ctx)
+        drift = {k: float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) for j, k in enumerate(self.names)}
+        out = {"minute": ctx.m1.isoformat(), "eligible": len(ctx.mints), "picks": scored["picks"], "lines": self.lines["plastic"], "frozen_lines": self.lines["frozen"],
+               "line": self.lines["plastic"][self.names[0]], "frozen_line": self.lines["frozen"][self.names[0]],
+               "drift": max(drift.values()) if drift else 0.0, "drift_by_strategy": drift, "pending": len(self.pending), "learned": learned,
+               "learning_frozen": self.learning_frozen, "channels": self.cfg, "holds_min": {k: v / 60.0 for k, v in self.holds.items()}, "bootstrap": self.boot_id}
+        if ctx.trade and ctx.fresh:
+            if self.race_started_at is None:
+                self.race_started_at = ctx.m1_epoch
+            self.beat_no += 1
+            d = scored["decision"]
+            st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
+                                            scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"],
+                                            strategies=d["strategy"], tables=d["tables"])
+            out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading"
+            if self.live and handover.state(conn) is not None:
+                out["live"] = self._live(ctx, st)
+        else:
+            out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up"
+        hour = int(ctx.m1_epoch // 3600)
+        if self.last_hour is None:
+            self.last_hour = hour
+        elif hour > self.last_hour:
+            self.last_hour = hour; self._hourly(conn, ctx.m1_epoch)
+        self._write_state()
+        if learned.get("keys"):
+            with conn.cursor() as cur:
+                cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys"))
+        learned.pop("keys", None)
+        out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat()
+        conn.execute("INSERT INTO ui_settings (key, value) VALUES ('fly_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                     (json.dumps(out, default=str),))
+        return out
+
+    def _live(self, ctx, st: dict) -> dict:
+        """The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed."""
+        from ..chain.cluster_guard import signing_allowed
+        if not signing_allowed():
+            return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"}
+        if self.mirror is None:
+            from .fly_live import LiveMirror
+            self.mirror = LiveMirror(self.H)
+            record_event("info", "fly_live", "the fly trades the bot wallet", {"run_id": self.run_id})
+        return self.mirror.minute(ctx, run_id=self.run_id, beat_id=st["beat_id"], entries=st["entries"], line=self.lines["plastic"][self.names[0]],
+                                  table=self.sizing["plastic"][self.names[0]])
+
+    def finish(self) -> None:
+        if self.mirror is not None:
+            self.mirror.stop()
+        try:
+            self._write_state()
+        except Exception:
+            log.exception("fly state write failed at finish")
+        with transaction() as conn:
+            conn.execute("UPDATE runs SET ended_at = now(), status = 'finished' WHERE run_id = %s", (self.run_id,))
+        record_event("info", "fly", "fly session stopped", {"run_id": self.run_id, "minutes": self.beat_no})
+
+    # ---- learning ----
+    @staticmethod
+    def _empty_stats() -> dict:
+        return {"n": 0, "sum_delta": 0.0, "sum_abs": 0.0, "step": 0.0, "capped": 0, "unknown": 0}
+
+    def _learn(self, conn, now: float) -> dict:
+        tags = self.pending.pop_due(now, device=self.bank.dev)
+        if tags is None:
+            return {"n": 0}
+        labels = np.array([np.nan if (lb := resolve_label(list(self.bars.get(m, ())), t, self.holds[name])) is None else lb for t, m, name in tags.keys], dtype=np.float64)
+        ok = np.isfinite(labels)
+        for t, m, _ in tags.keys:
+            c = self.watch.get(m, 0) - 1
+            if c > 0:
+                self.watch[m] = c
+            else:
+                self.watch.pop(m, None); self.bars.pop(m, None)
+        stats = {"n": int(ok.sum()), "unknown": int((~ok).sum())}
+        if ok.any():
+            tv = tags.subset(ok)
+            if self.learning_frozen:
+                self.bank.decay_to(now)
+            else:
+                st = self.bank.update(tv, torch.tensor(labels[ok], dtype=torch.float32), now)
+                stats.update(mean_delta=st["mean_delta"][0], mean_abs_delta=st["mean_abs_delta"][0], step=st["step"][0], capped=bool(st["capped"][0]))
+                h = self.hour_stats; h["n"] += stats["n"]; h["sum_delta"] += st["mean_delta"][0] * stats["n"]; h["sum_abs"] += st["mean_abs_delta"][0] * stats["n"]
+                h["step"] += st["step"][0]; h["capped"] += int(st["capped"][0])
+        self.hour_stats["unknown"] += stats["unknown"]
+        self.applied_through = now
+        stats["keys"] = [(float(lb) if np.isfinite(lb) else None, "resolved" if np.isfinite(lb) else "unknown", datetime.fromtimestamp(t, timezone.utc), m, name)
+                         for (t, m, name), lb in zip(tags.keys, labels)]
+        return stats
+
+    def _score(self, conn, ctx) -> dict:
+        n = len(ctx.mints); empty = {"strategy": np.full(n, None, dtype=object), "score": np.full(n, -1.0), "hold_s": np.zeros(n), "threshold": np.full(n, np.inf),
+                                     "tables": [[] for _ in range(n)]}
+        if not n:
+            return {"decision": empty, "picks": 0}
+        X = ctx.X[:, self.idx]
+        Y, u0, k = self.fly.parts_all(X); trig = self.fly.triggers(ctx.X, X_COLS)
+        if not self.nu_set:
+            self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True
+        t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = []
+        V = np.full((n, len(self.names)), -np.inf)
+        for j, name in enumerate(self.names):
+            loc = np.flatnonzero(trig[:, j])
+            if not len(loc):
+                continue
+            li = torch.as_tensor(loc, device=Y.device)
+            with torch.no_grad():
+                sc, _ = self.bank.predict(Y[li, j], u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0]
+                fr = self.bank.frozen(Y[li, j], u0[li], s=np.full(len(loc), j))
+            v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v
+            line, fline = self.lines["plastic"][name], self.lines["frozen"][name]
+            rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id) for q, i in enumerate(loc)]
+            w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device))
+            self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, Y[li, j], u0[li], k[li], w, s=j)
+            for i in loc:
+                m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1
+                q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m)
+                if b is not None and (not q_ or q_[-1][0] < b[0]):
+                    q_.append(b)
+        if rows:
+            with conn.cursor() as cur:
+                cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id) "
+                                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", rows)
+        self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()}
+        d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf))
+        return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))}
+
+    # ---- daily and hourly ----
+    def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]:
+        """A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s)."""
+        return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND ts >= %s ORDER BY ts",
+                            (self.boot_id, name, datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall()
+
+    def _calibrate(self, conn, now: float) -> None:
+        day = datetime.fromtimestamp(now, timezone.utc).date()
+        for name in self.names:
+            rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label", name)
+            if not rows:
+                continue
+            ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
+            for arm, col in (("plastic", "score"), ("frozen", "frozen_score")):
+                cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab,
+                                              prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name])
+                if cal.changed:
+                    record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%", {"trades": cal.trades, "mean": cal.mean, "total": cal.total})
+                self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing
+                conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) "
+                             "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean",
+                             (day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, cal.total, cal.mean, fly_calibrate.WINDOW_DAYS))
+
+    def _drift(self, j: int) -> float:
+        return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0])
+
+    def _checks(self, conn, now: float) -> dict:
+        """The rollback checks per strategy (its own trades, its own channel's drift)."""
+        out = {}
+        for j, name in enumerate(self.names):
+            rows = self._resolved(conn, now - gov.SHADOW_DAYS * 86400.0, "ts, mint, score, line, frozen_score, frozen_line, label", name)
+            if rows:
+                ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64)
+                g = lambda c: np.array([r[c] for r in rows], dtype=np.float64)
+                H = self.holds[name]
+                rp = gov.picks_returns(ts, mint, H, g("score"), g("line"), lab); rf = gov.picks_returns(ts, mint, H, g("frozen_score"), g("frozen_line"), lab)
+                recent = ts + H + LABEL_LAG_S > now - gov.IC_HOURS * 3600.0
+                out[name] = gov.check((rp, rf), (g("score")[recent], lab[recent]), self._drift(j))
+            else:
+                out[name] = gov.check(None, None, self._drift(j))
+        return out
+
+    def _hourly(self, conn, now: float) -> None:
+        hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc)
+        checks = self._checks(conn, now); self.last_checks = checks
+        h = self.hour_stats; first = checks.get(self.names[0]) or {}
+        conn.execute("INSERT INTO fly_updates (hour, n, mean_delta, mean_abs_delta, step, capped, drift, ic, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (hour) DO NOTHING",
+                     (hour, h["n"], h["sum_delta"] / h["n"] if h["n"] else None, h["sum_abs"] / h["n"] if h["n"] else None, h["step"], h["capped"],
+                      max((c["drift"] for c in checks.values()), default=0.0), first.get("ic"), json.dumps({"unknown": h["unknown"], "checks": checks}, default=str)))
+        self.hour_stats = self._empty_stats()
+        good = not any(c["triggers"] for c in checks.values())
+        sid = self._snapshot(conn, now, good, checks)
+        for name, c in checks.items():
+            if c["triggers"] and now >= self.checks_paused_until.get(name, 0.0) and not self.learning_frozen:
+                self._rollback(conn, now, c, sid, name=name)
+        self._prune(conn, now)
+        self._handover_check(conn, now)
+
+    def _snapshot(self, conn, now: float, good: bool, checks: dict) -> int:
+        d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True)
+        path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt"
+        torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path)
+        note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "good": good, "checks": checks,
+                "drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}}
+        return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id",
+                                (self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"])
+
+    def _commands(self, conn, now: float) -> None:
+        """Operator commands from the console (``send_command``): pause / resume learning, roll back now."""
+        r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (COMMAND_KEY,)).fetchone()
+        if not r or not r["value"]:
+            return
+        v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
+        conn.execute("DELETE FROM ui_settings WHERE key = %s", (COMMAND_KEY,))
+        cmd = v.get("cmd")
+        if cmd == "pause":
+            self.learning_frozen = True; record_event("info", "fly", "learning paused by the operator")
+        elif cmd == "resume":
+            self.learning_frozen = False; self.rollbacks = {}; record_event("info", "fly", "learning resumed by the operator")
+        elif cmd == "rollback":
+            for j, name in enumerate(self.names):
+                self._rollback(conn, now, {"triggers": ["operator"], "drift": self._drift(j)}, None, count=False, name=name)
+
+    def _rollback(self, conn, now: float, checks: dict, from_sid: int | None, count: bool = True, name: str | None = None) -> None:
+        """Restore one strategy's channel (its KC→MBON columns) from the newest good snapshot at least a day old, or the bootstrap."""
+        name = name or self.names[0]; j = self.names.index(name); cols = self.bank.learn[j]
+        target = None
+        for r in conn.execute("SELECT id, path, note, ts FROM brain_snapshots WHERE kind = 'fly_plastic' AND ts <= %s ORDER BY id DESC",
+                              (datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall():
+            n = json.loads(r["note"] or "{}")
+            ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers")
+            if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists():
+                target = r; break
+        if target is not None:
+            snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy())
+            st = torch.load(target["path"], map_location="cpu", weights_only=False)["bank"]
+            snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float()
+            self.bank.D[0][:, cols] = snap.D[0][:, cols]
+        else:
+            self.bank.D[0][:, cols] = 0.0
+        self.checks_paused_until[name] = now + ROLLBACK_PAUSE_S
+        self.rollbacks[name] = [t for t in self.rollbacks.get(name, []) if t > now - ROLLBACK_WINDOW_S] + ([now] if count else [])
+        conn.execute("INSERT INTO fly_rollbacks (reason, checks, from_snapshot, to_snapshot) VALUES (%s,%s,%s,%s)",
+                     (f"{name}: " + ", ".join(checks["triggers"]), json.dumps(checks, default=str), from_sid, target["id"] if target is not None else None))
+        record_event("warning", "fly", f"{name} channel rolled back ({', '.join(checks['triggers'])})", {"to_snapshot": target["id"] if target is not None else "bootstrap (D = 0)", "checks": checks})
+        if count and len(self.rollbacks[name]) >= ROLLBACK_LIMIT:
+            self.learning_frozen = True
+            from ..train import pipeline
+            pipeline.request_run(fly=True, reason=f"{name}: {len(self.rollbacks[name])} rollbacks in 7 days")
+            record_event("error", "fly", f"learning frozen after {len(self.rollbacks[name])} {name} rollbacks in 7 days; re-bootstrap requested", {"rollbacks": self.rollbacks})
+
+    def _prune(self, conn, now: float) -> None:
+        rows = conn.execute("SELECT id, path, ts FROM brain_snapshots WHERE kind = 'fly_plastic' AND ts < %s ORDER BY ts DESC",
+                            (datetime.fromtimestamp(now - SNAP_HOURLY_DAYS * 86400.0, timezone.utc),)).fetchall()
+        kept_days: set = set(); drop = []
+        for r in rows:
+            d = r["ts"].date()
+            if d in kept_days:
+                drop.append(r)
+            else:
+                kept_days.add(d)
+        for r in drop:
+            conn.execute("DELETE FROM brain_snapshots WHERE id = %s", (r["id"],))
+            try:
+                Path(r["path"]).unlink(missing_ok=True)
+            except OSError:
+                pass
+        conn.execute("DELETE FROM fly_scored WHERE state <> 'pending' AND ts < %s", (datetime.fromtimestamp(now - SCORED_KEEP_DAYS * 86400.0, timezone.utc),))
+
+    def _handover_check(self, conn, now: float) -> None:
+        """The fly takes the selector's seat once it has raced ``config.HANDOVER_DAYS`` (0: no minimum, the window is the
+        whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``,
+        realized at least the paper selector's P&L over that window."""
+        days = config.HANDOVER_DAYS
+        if handover.state(conn) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0:
+            return
+        since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc)
+        pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s",
+                               (b, since)).fetchone() for b in (BOOK, "paper_selector")}
+        fly_pnl, sel_pnl, n = float(pnl[BOOK]["s"]), float(pnl["paper_selector"]["s"]), int(pnl[BOOK]["n"])
+        if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR):
+            detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n,
+                      "selector_trades": int(pnl["paper_selector"]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id}
+            handover.record(conn, detail)
+            record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades"
+                         + (f" in {days} days" if days > 0 else ""), detail)

fly_trader/agent/handover.py

@@ -0,0 +1,24 @@
+"""The handover: the plastic fly takes the selector's seat (agent/fly_session.py decides; ui_settings['handover']).
+
+After it the selector stops entering, exits to flat and is dropped while a fly trades; the weekly training schedule
+stops (train/pipeline.due runs only on request). If no fly can trade (replaced definitions, a failed re-bootstrap), the
+selector trades again until one can.
+"""
+from __future__ import annotations
+
+import json
+
+KEY = "handover"
+
+
+def state(conn) -> dict | None:
+    r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone()
+    if not r or r["value"] is None:
+        return None
+    v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"])
+    return v or None
+
+
+def record(conn, detail: dict) -> None:
+    conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                 (KEY, json.dumps(detail, default=str)))

fly_trader/agent/minute_engine.py

@@ -0,0 +1,342 @@
+"""The trading engine on the live stream: every UTC minute, features once per mint, then every book trades on them.
+
+Every UTC minute, once the stream has written it (``pumpstream_status.flushed_through``), the minute's rich candle per
+mint (``pump_minutes``: open/high/low/close, SOL volume by side, buy/sell counts, distinct traders, quote reserve; the
+same fields and filters ``train/mature.py`` builds from the archive) is fed to the live feature engine as two
+synthetic trades, exactly as in training. Graduation time and creation/creator facts come from ``corpus_meta``, the
+same source training uses. The full feature vector (``decisions.X_COLS``) of the rows in the models' universe
+(``selector.in_universe``) that pass the eligibility gate (pool ≥ ``MIN_RESQ_SOL``, 15-minute volume ≥
+``MIN_VOL_15M_SOL``, no scale break — a close 50× from the previous one or a reserve above 100,000 SOL — since the
+mint's state began) goes to every book; each picks its model's columns by name.
+
+Books — ``SelectorBook`` (agent/selector_session.py) and ``FlyBook`` (agent/fly_session.py) — implement ``name``,
+``on_bars(t_start, bars)`` (every processed minute, warm-up included: open, close and exit cost of every traded mint),
+``on_minute(ctx)``, ``open_mints(conn)``, ``maybe_reload()``, ``done`` and ``finish()``. Each book trades inside its own
+savepoint, so one book's failure never undoes the other's minute. Books join when they qualify (checked every
+``MODEL_CHECK_S``): the selector while it has a deployable model and has not handed its seat to a fly that is trading;
+the fly once its design passed the replay and a deployable bootstrap exists. No entries or exits while the stream is
+stale; missed minutes are replayed through the features without trading.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import math
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+import numpy as np
+
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..market.exit_cost import PUMP_SUPPLY
+from ..market.exit_cost import cost_at_size
+from ..market.features import FEATURES, FIDX, TokenMeta, TokenState
+from ..train.corpus_meta import FEATURE_COLS as META_COLS
+from ..train.decisions import MIN_RESQ_SOL, MIN_VOL_15M_SOL, X_COLS
+from ..train.flow import FlowWindow, MarketWindow
+from ..train.mature import MASKED
+from ..train.selector import in_universe
+
+log = logging.getLogger(__name__)
+WARMUP_MIN = 1440            # training warms each day on the previous day's candles
+IDLE_EVICT_S = 86400         # training has no history for a mint idle since before the previous day
+STREAM_STALE_S = 180         # no entries or exits when the stream's newest complete minute is older than this
+META_TTL_S = 600
+MODEL_CHECK_S = 600          # how often books reload newer models and new books are admitted
+
+
+class MintState:
+    __slots__ = ("st", "meta", "hist", "decimals", "pool", "program_label", "graduated_at", "meta_row", "prev_close", "broken", "flow")
+
+    def __init__(self, mint: str, decimals: int, pool: str | None, program_label: str | None, graduated_at: float | None, meta_row: dict | None):
+        self.st = TokenState(mint); self.meta = TokenMeta(mint=mint, program_label=program_label or "Pump.fun Amm", graduated_at=graduated_at)
+        self.hist: deque = deque(maxlen=200)       # (t_end, n_trades, n_traders) for the trailing hour
+        self.decimals, self.pool, self.program_label, self.graduated_at, self.meta_row = decimals, pool, program_label, graduated_at, meta_row
+        self.prev_close: float | None = None; self.broken = False
+        self.flow = FlowWindow()                    # wallet flow inputs (train/flow.py, as mature.build_day)
+
+
+@dataclass
+class Minute:
+    """What every book sees of one minute."""
+    conn: object
+    m0: datetime
+    m1: datetime
+    m1_epoch: float
+    agg: dict
+    X: np.ndarray            # [n, len(X_COLS)] eligible rows in the universe
+    infos: list
+    mints: list
+    bars: dict               # mint -> (minute start, open, close, exit cost) for every traded mint
+    prices: dict
+    resqs: dict
+    fees: dict
+    fresh: bool
+    trade: bool
+    engine: "MinuteEngine"
+
+    @property
+    def t_start(self) -> float:
+        return self.m1_epoch - 60.0
+
+    def last_resq(self, mint: str) -> float | None:
+        return self.engine._last_resq(self.conn, mint)
+
+    def mcap(self, mint: str, price: float | None) -> float | None:
+        return self.engine._mcap(mint, price)
+
+
+class MinuteEngine:
+    def __init__(self, books: list | None = None, live: bool = False):
+        self.books = list(books or []); self.live = live
+        self.states: dict[str, MintState] = {}
+        self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None
+        self.market = MarketWindow()                 # mkt_vol_1h over every traded mint (as mature._market_windows)
+
+    def has_book(self, name: str) -> bool:
+        return any(b.name == name for b in self.books)
+
+    # ---- lookups ----
+    def _meta(self, conn, mint: str) -> dict | None:
+        """``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at
+        graduation and the archive rebuild later replaces stream values, so the cache is refreshed."""
+        hit = self.meta_cache.get(mint)
+        if hit is None or time.time() - hit["at"] > META_TTL_S:
+            r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone()
+            hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()}
+        return hit["row"]
+
+    def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState:
+        s = self.states.get(mint)
+        if s is None:
+            t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone()
+            s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None)
+            self.states[mint] = s
+        row = self._meta(conn, mint)
+        s.meta_row = row
+        g = row["graduated_at"].timestamp() if row and row.get("graduated_at") else None
+        if g != s.graduated_at:                     # same source as training: corpus_meta.graduated_at
+            s.graduated_at = g; s.meta.graduated_at = g
+        s.meta.supply = float(row["supply"]) if row and row.get("supply") else PUMP_SUPPLY   # as training (train/mature.build_day)
+        return s
+
+    def _last_resq(self, conn, mint: str) -> float | None:
+        s = self.states.get(mint)
+        if s is not None and s.st.last_res_quote_sol:
+            return s.st.last_res_quote_sol
+        r = conn.execute("SELECT resq_sol FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone()
+        return float(r["resq_sol"]) if r else None
+
+    def _mcap(self, mint: str, price: float | None) -> float | None:
+        """Market cap in SOL (price × supply), which sets the pool fee tier; None without a price."""
+        s = self.states.get(mint)
+        return float(price) * (s.meta.supply if s is not None else PUMP_SUPPLY) if price else None
+
+    # ---- one minute ----
+    def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]:
+        """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive."""
+        rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
+                            "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall()
+        return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0,
+                            "nb": r["n_buys"] or 0, "ns": r["n_sells"] or 0, "n_traders": int(r["n_traders"] or 0), "resq": r["resq_sol"],
+                            "pool": r["pool_id"], "program_label": "Pump.fun Amm", "fee_rate": r["fee_rate"],
+                            "n_buyers": r["n_buyers"], "wash_sol": r["wash_sol"], "wash_buy_sol": r["wash_buy_sol"], "top_sell_sol": r["top_sell_sol"],
+                            "insider_sell_sol": r["insider_sell_sol"], "skill_buy": r["skill_buy"]} for r in rows if r["close"]}
+
+    def _features(self, conn, mint: str, a: dict, t_end: float) -> tuple[np.ndarray, dict]:
+        """The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs."""
+        s = self._state(conn, mint, a["pool"], a["program_label"])
+        st, price, resq = s.st, float(a["close"]), a["resq"]
+        if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5):
+            s.broken = True                          # train/decisions.py: a scale break makes the mint ineligible from that minute on
+        s.prev_close = price
+        if a["buy"] > 0:
+            st.append(t_end - 2e-3, price, a["buy"], True, None, resq)
+        if a["sell"] > 0 or a["buy"] <= 0:
+            st.append(t_end - 1e-3, price, a["sell"], False, None, resq)
+        s.hist.append((t_end, float(a["nb"] + a["ns"]), float(a["n_traders"])))
+        s.flow.append(t_end, a["buy"], a["sell"], a.get("n_buyers"), a.get("wash_sol"), a.get("wash_buy_sol"), a.get("top_sell_sol"), a.get("insider_sell_sol"), a.get("skill_buy"))
+        f, _mask = st.features(t_end, s.meta)
+        ht = np.array([h[0] for h in s.hist]); hn = np.array([h[1] for h in s.hist]); htr = np.array([h[2] for h in s.hist])
+        for k, w in (("1m", 60), ("5m", 300), ("15m", 900), ("1h", 3600)):
+            f[FIDX[f"logn_{k}"]] = math.log1p(float(hn[ht > t_end - w].sum()))
+        for name in MASKED:
+            f[FIDX[name]] = 0.0
+        if s.graduated_at is None:
+            f[FIDX["log_age_h"]] = 0.0
+        hod = ((t_end - 60.0) % 86400) / 3600.0                     # minute start, as in train/decisions.py
+        meta = s.meta_row or {}
+        extra = {"traders_15m": float(htr[ht > t_end - 900].sum()), "traders_1h": float(htr[ht > t_end - 3600].sum()), "n_trades_1m": float(a["nb"] + a["ns"]),
+                 "hod_s": math.sin(2 * math.pi * hod / 24), "hod_c": math.cos(2 * math.pi * hod / 24), "age_known": 1.0 if s.graduated_at is not None else 0.0,
+                 "meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)}
+        by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra,
+                   **{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}}
+        x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
+        info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]],
+                # "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades
+                "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq)), "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None,
+                "mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken,
+                "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known"))}
+        return x, info
+
+    @staticmethod
+    def eligible(info: dict) -> bool:
+        return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)
+
+    def _stream_through(self) -> float | None:
+        with transaction() as conn:
+            r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = 'pumpstream_status'").fetchone()
+        return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None
+
+    def ready_through(self, m1_epoch: float) -> float:
+        """End of the newest minute (at most ``m1``) the stream has written; A minute is
+        never read before it is written, so a lagging stream delays minutes instead of feeding them empty or partial."""
+        ft = self._stream_through()
+        return min(m1_epoch, ft + 60.0) if ft is not None else float("-inf")
+
+    def stream_fresh(self, m0_epoch: float) -> bool:
+        ft = self._stream_through()
+        return ft is not None and m0_epoch - ft <= STREAM_STALE_S
+
+    def _minute_rows(self, conn, agg: dict, m1_epoch: float) -> tuple[np.ndarray, list, list, dict]:
+        xs, infos, mints, bars = [], [], [], {}
+        self.market.add(m1_epoch, sum(float(a["buy"]) + float(a["sell"]) for a in agg.values()))
+        for mint, a in agg.items():
+            x, info = self._features(conn, mint, a, m1_epoch)
+            bars[mint] = (m1_epoch - 60.0, info["open"], info["price"], info["ec"])
+            if in_universe(x[None], X_COLS)[0] and self.eligible(info):        # the models' universe (train/selector.in_universe)
+                xs.append(x); infos.append(info); mints.append(mint)
+        return (np.stack(xs) if xs else np.zeros((0, len(X_COLS)), np.float32)), infos, mints, bars
+
+    def warm_up(self, m1_epoch: float, minutes: int = WARMUP_MIN) -> int:
+        """Feed the last ``minutes`` closed minutes through the feature states (and the books' bars) without trading."""
+        n = 0
+        with transaction() as conn:
+            for k in range(minutes, 0, -1):
+                t1 = m1_epoch - 60 * k
+                agg = self._aggregate(conn, datetime.fromtimestamp(t1 - 60, timezone.utc), datetime.fromtimestamp(t1, timezone.utc))
+                bars = {}
+                self.market.add(t1, sum(float(a["buy"]) + float(a["sell"]) for a in agg.values()))
+                for mint, a in agg.items():
+                    _x, info = self._features(conn, mint, a, t1); n += 1
+                    bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"])
+                for b in self.books:
+                    b.on_bars(t1 - 60.0, bars)
+        log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states))
+        return n
+
+    def _sweep(self, now_s: float, held: set[str]) -> None:
+        if now_s - self.last_sweep < 3600:
+            return
+        self.last_sweep = now_s
+        idle = [m for m, s in self.states.items() if m not in held and (s.st.last_ts is None or s.st.last_ts < now_s - IDLE_EVICT_S)]
+        for m in idle:
+            self.states.pop(m, None); self.meta_cache.pop(m, None)
+        if idle:
+            log.info("evicted %d mint states idle for more than %d h", len(idle), IDLE_EVICT_S // 3600)
+
+    def run_minute(self, m1_epoch: float, trade: bool = True) -> dict:
+        m1 = datetime.fromtimestamp(m1_epoch, timezone.utc); m0 = datetime.fromtimestamp(m1_epoch - 60, timezone.utc)
+        out: dict = {"minute": m1.isoformat()}
+        with transaction() as conn:
+            agg = self._aggregate(conn, m0, m1)
+            X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch)
+            ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars,
+                         prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()},
+                         fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self)
+            out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set()
+            for b in list(self.books):
+                try:
+                    with conn.transaction():                            # a savepoint per book
+                        b.on_bars(m1_epoch - 60.0, bars)
+                        out[b.name] = b.on_minute(ctx)
+                        held |= set(b.open_mints(conn))
+                except Exception:
+                    log.exception("%s: minute %s failed", b.name, m1.isoformat())
+                    record_event("error", b.name, f"minute failed: {m1.isoformat()}")
+            self._sweep(m1_epoch, held)
+        for b in [b for b in self.books if b.done]:
+            b.finish(); self.books.remove(b)
+        return out
+
+
+def _status(key: str, value: dict) -> None:
+    try:
+        with transaction() as conn:
+            conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                         (key, json.dumps({**value, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str)))
+    except Exception:
+        log.debug("status write failed", exc_info=True)
+
+
+def admit_books(engine: MinuteEngine, live: bool) -> list:
+    """Books that qualify now and are not running yet (the fly first: its presence decides whether the selector trades)."""
+    from . import fly_session, handover
+    from .selector_session import NoModel, SelectorBook
+    new = []
+    if not engine.has_book("fly"):
+        fb, why = fly_session.try_start(live)
+        if fb is not None:
+            new.append(fb)
+        else:
+            _status("fly_status", {"stage": "not trading", "detail": why})
+    fly_on = engine.has_book("fly") or bool(new)
+    with transaction() as conn:
+        handed = handover.state(conn) is not None
+    if not engine.has_book("selector") and not (handed and fly_on):
+        try:
+            new.append(SelectorBook())
+        except NoModel as e:
+            _status("selector_status", {"stage": "waiting for a model", "detail": str(e)})
+    return new
+
+
+def main(stop_event: threading.Event | None = None, live: bool = False) -> None:
+    engine = MinuteEngine(live=live)
+    while not (stop_event is not None and stop_event.is_set()):
+        engine.books = admit_books(engine, live)
+        if engine.books:
+            break
+        log.info("trading engine waiting: no book qualifies yet")
+        (stop_event or threading.Event()).wait(MODEL_CHECK_S)
+    else:
+        return
+    m_start = math.floor(time.time() / 60) * 60
+    engine.warm_up(m_start); engine.last_minute = m_start - 60; last_check = time.time(); n_min = 0
+    try:
+        while not (stop_event is not None and stop_event.is_set()):
+            now = time.time(); m1 = math.floor(now / 60) * 60
+            if now - last_check >= MODEL_CHECK_S:
+                last_check = now
+                for b in engine.books:
+                    try:
+                        b.maybe_reload()
+                    except Exception:
+                        log.exception("%s: model reload check failed", b.name)
+                try:
+                    engine.books += admit_books(engine, live)
+                except Exception:
+                    log.exception("book admission failed")
+            if engine.books and m1 > engine.last_minute and now - m1 >= 4.0 and (upto := engine.ready_through(m1)) > engine.last_minute:   # only minutes the stream has written
+                for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60):
+                    try:
+                        st = engine.run_minute(float(minute), trade=(minute == int(m1)))   # missed minutes update features (and the fly's learning) only
+                        n_min += 1
+                        if minute == int(m1) and n_min % 10 == 0:
+                            log.info("minute %s: traded %d eligible %d | %s", st["minute"], st.get("mints_traded", 0), st.get("eligible", 0),
+                                     " | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}"
+                                                for b in engine.books))
+                    except Exception:
+                        log.exception("minute %s failed", minute); record_event("error", "engine", f"minute failed: {minute}")
+                engine.last_minute = upto
+            time.sleep(0.5)
+    finally:
+        for b in engine.books:
+            try:
+                b.finish()
+            except Exception:
+                log.exception("%s: finish failed", b.name)

fly_trader/agent/paper_trading.py

@@ -0,0 +1,89 @@
+"""One trading minute of a paper book — the rules both race books share (the selector's and the fly's):
+
+exits: positions held their own hold (the strategy's, stored on the position) or ``horizon_s`` are sold at this minute's
+price (or the last traded price: the training label's exit); entries: scores at or above the book's line open a position sized from the book's certainty bands
+(``agent/sizing.py``), one position per token, unless blocked (global kill switch, paused entries, the book's own
+drawdown halt, or the caller's ``block``); marks and wealth per minute (positions net of exit cost, exposure at cost,
+drawdown ≥ 0); the book's drawdown halt is checked on its own peak (``rails.check_book_drawdown``). Every score, entry,
+exit and blocked pick is a ``decisions`` row.
+"""
+from __future__ import annotations
+
+import json
+
+import numpy as np
+
+from . import rails, sizing
+from ..execution import ledger
+from ..market.exit_cost import exit_cost_fraction
+
+
+def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold,
+                 table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None,
+                 tables: list | None = None) -> dict:
+    """Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row,
+    ``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason).
+    Left out, the call behaves exactly as the single-strategy engine."""
+    conn, m1 = ctx.conn, ctx.m1
+    beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id",
+                        (run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"]
+    n_exit = 0
+    for p in ledger.open_positions(conn, book):
+        held = (m1 - p["opened_at"]).total_seconds()
+        if held < float(p.get("hold_s") or horizon_s):
+            continue
+        did = conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, size_sol, forced, reason, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,false,%s,%s) RETURNING id",
+                           (beat, run_id, m1, p["mint"], p["pool"], f"{kind}_exit", float(p["cost_sol"]), f"held {held/60:.0f} min", json.dumps({"book": book}))).fetchone()["id"]
+        px = ctx.prices.get(p["mint"]) or float(p.get("last_mark_price") or p["entry_price"])
+        broker.sell(conn, position=p, decision_id=did, price=ctx.prices.get(p["mint"]), res_quote_sol=ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"]),
+                    mcap_sol=ctx.mcap(p["mint"], px), program_label=p.get("program_label"), forced_kind=None, pool_fee=ctx.fees.get(p["mint"]), ts=m1)
+        n_exit += 1
+    opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now}
+    cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = []
+    bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now)         # wealth at cost: the sizing base (agent/sizing.py)
+    circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = 1").fetchone()
+    blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None)
+               or ("book halted" if rails.book_halted(conn, book) else None))
+    thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([])
+    for k, (m, i, sc) in enumerate(zip(mints, infos, scores)):
+        thr_k = float(thr_v[k])
+        if not np.isfinite(sc) or sc < thr_k:
+            continue
+        picks.append((m, float(sc)))
+        tab_k = tables[k] if tables is not None else table
+        size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"])
+        denied = allow is not None and not bool(allow[k])
+        if m in open_mints or blocked or denied or size <= 0:
+            rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing")
+            conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, m_hat, size_sol, forced, rail, reason, detail) VALUES (%s,%s,%s,%s,%s,'blocked',%s,%s,false,%s,%s,%s)",
+                         (beat, run_id, m1, m, i["pool"], float(sc), size, rail, why if rail == "sizing" else f"{kind} pick not taken",
+                          json.dumps({"book": book, "score": float(sc), "threshold": thr_k, "sizing": why,
+                                      "strategy": None if strategies is None else strategies[k]})))
+            continue
+        hold_k = float(holds[k]) if holds is not None else None; strat_k = None if strategies is None else strategies[k]
+        did = conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, m_hat, size_sol, forced, reason, book_targets, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,false,%s,%s,%s) RETURNING id",
+                           (beat, run_id, m1, m, i["pool"], f"{kind}_enter", float(sc), size, f"{strat_k + ': ' if strat_k else ''}score {sc:.3f} ≥ {thr_k:.3f}; {why}", [book],
+                            json.dumps({"book": book, "score": float(sc), "threshold": thr_k, "size_sol": size, "sizing": why, "resq": i["resq"], "age_h": i["age_h"], "price": i["price"],
+                                        "strategy": strat_k, "hold_s": hold_k}))).fetchone()["id"]
+        fill = broker.buy(conn, decision_id=did, mint=m, pool=i["pool"], size_sol=size, price=i["price"], res_quote_sol=i["resq"],
+                          mcap_sol=i["mcap"], decimals=i["decimals"], program_label=i["program_label"], pool_fee=i["fee_rate"], ts=m1, hold_s=hold_k, strategy=strat_k)
+        if fill.ok:
+            n_enter += 1; open_mints.add(m); cash -= size
+            entries.append({"mint": m, "decision_id": did, "score": float(sc), "size_sol": size, "info": i, "threshold": thr_k, "table": tab_k, "hold_s": hold_k, "strategy": strat_k})
+        else:
+            conn.execute("UPDATE decisions SET rail = %s, reason = %s WHERE id = %s", ("paper_fill", fill.reason, did))
+    ledger.mark_positions(conn, book, ctx.prices, {}, m1)
+    opens = ledger.open_positions(conn, book); gross_v = 0.0; exit_cost = 0.0; exposure = 0.0
+    for p in opens:
+        px = float(p.get("last_mark_price") or p["entry_price"])
+        gross = int(p["qty"]) / 10 ** int(p.get("decimals") or 6) * px
+        rq = ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"])
+        gross_v += gross; exposure += float(p["cost_sol"])
+        exit_cost += gross * exit_cost_fraction(gross, rq, ctx.mcap(p["mint"], px), p.get("program_label"), ctx.fees.get(p["mint"]))
+    cash = ledger.paper_cash(conn, book); positions_net = gross_v - exit_cost; wealth = cash + positions_net
+    peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth)
+    conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+                 (beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens)))
+    rails.check_book_drawdown(conn, book, wealth, peak)        # the book's own kill switch: blocks its next minutes' entries
+    return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked,
+            "top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})}

fly_trader/agent/rails.py

@@ -0,0 +1,122 @@
+"""Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker
+(CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console
+toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+
+from .. import config
+from ..db.apilog import record_event
+from ..db.connection import transaction
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class CircuitState:
+    fail_count: int = 0
+    tripped: bool = False
+    kill_switch: bool = False
+    kill_reason: str | None = None
+    peak_wealth: float | None = None
+    entries_paused: bool = False
+
+
+def load_circuit(conn) -> CircuitState:
+    r = conn.execute("SELECT * FROM circuit_state WHERE id = 1").fetchone()
+    if not r:
+        return CircuitState()
+    return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"])
+
+
+def _event(conn, kind: str, detail: dict | None = None) -> None:
+    conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps(detail or {}, default=str)))
+
+
+def record_failure(conn, reason: str) -> CircuitState:
+    r = conn.execute(
+        "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = 1 RETURNING fail_count, tripped"
+    ).fetchone()
+    tripped = r["tripped"]
+    if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped:
+        conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = 1")
+        _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]})
+        record_event("error", "rails", "circuit breaker tripped", {"reason": reason})
+    return load_circuit(conn)
+
+
+def record_success(conn) -> None:
+    conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = 1")
+
+
+def check_drawdown(conn, wealth: float, peak: float) -> bool:
+    """Trip the kill switch when ``wealth`` has fallen KILL_SWITCH_DRAWDOWN below ``peak`` (the book's own peak: the
+    trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on."""
+    st = load_circuit(conn)
+    if st.kill_switch:
+        return True
+    if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN):
+        conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = 1",
+                     (f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL", peak))
+        _event(conn, "kill_switch", {"wealth": wealth, "peak": peak})
+        record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak})
+        return True
+    return False
+
+
+def check_book_drawdown(conn, book: str, wealth: float, peak: float) -> bool:
+    """A paper race book's own kill switch: KILL_SWITCH_DRAWDOWN below the book's peak halts that book's entries only,
+    so one book's drawdown never stops the other during the race. Returns whether the book is halted."""
+    if book_halted(conn, book):
+        return True
+    if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN):
+        reason = f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL"
+        conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) "
+                     "ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()",
+                     (book, reason, peak))
+        _event(conn, "book_halt", {"book": book, "wealth": wealth, "peak": peak})
+        record_event("error", "rails", f"{book}: drawdown halt, entries blocked", {"wealth": wealth, "peak": peak})
+        return True
+    return False
+
+
+def book_halted(conn, book: str) -> bool:
+    r = conn.execute("SELECT halted FROM book_state WHERE book = %s", (book,)).fetchone()
+    return bool(r and r["halted"])
+
+
+def clear_book_halt(book: str) -> None:
+    with transaction() as conn:
+        conn.execute("UPDATE book_state SET halted = false, reason = NULL, peak = NULL, updated_at = now() WHERE book = %s", (book,))
+        _event(conn, "book_resumed", {"book": book})
+    record_event("info", "rails", f"{book}: drawdown halt cleared")
+
+
+def notional_24h(conn) -> float:
+    r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours'").fetchone()
+    return float(r["s"])
+
+
+def record_notional(conn, sol: float, kind: str = "entry") -> None:
+    conn.execute("INSERT INTO notional_ledger (sol, kind) VALUES (%s, %s)", (sol, kind))
+
+
+def reset_circuit(kill: bool = False) -> None:
+    with transaction() as conn:
+        conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = 1")
+        _event(conn, "reset", {"kill": kill})
+        if kill:
+            conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = 1")
+    record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else ""))
+    print("circuit reset" + (" + kill switch cleared" if kill else ""))
+
+
+def set_entries_paused(paused: bool) -> None:
+    with transaction() as conn:
+        conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = 1", (paused,))
+        _event(conn, "paused" if paused else "resumed", {})
+    record_event("info", "rails", "entries paused" if paused else "entries resumed")
+    print("entries paused" if paused else "entries resumed")

fly_trader/agent/runner.py

@@ -0,0 +1,64 @@
+"""The `runner` worker: the selector session (``agent/selector_session.py``) trading the paper book every minute on
+the market feed. One runner per database (a Postgres advisory lock). ``LIVE_ENABLED=1`` passes the signing guard first."""
+from __future__ import annotations
+
+import logging
+
+from .. import config
+from ..db.apilog import record_event
+from ..db.connection import connect
+from ..logging_setup import setup
+
+log = logging.getLogger(__name__)
+
+
+RUNNER_LOCK_KEY = 0x666C795F72756E  # "fly_run": one runner per database, held for the runner's lifetime
+
+
+def acquire_runner_lock():
+    """Session advisory lock on a dedicated autocommit connection; closing the connection releases it.
+    Raises RuntimeError when another runner (process or console thread) already holds it."""
+    conn = connect(autocommit=True)
+    try:
+        ok = conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (RUNNER_LOCK_KEY,)).fetchone()["ok"]
+    except Exception:
+        conn.close()
+        raise
+    if not ok:
+        conn.close()
+        raise RuntimeError("another runner holds the runner lock; refusing to start a second one")
+    return conn
+
+
+def main(stop_event=None) -> None:
+    setup("runner")
+    try:
+        lock_conn = acquire_runner_lock()
+    except RuntimeError as e:
+        log.error("%s", e)
+        record_event("error", "runner", str(e))
+        raise
+    try:
+        _main(stop_event)
+    finally:
+        lock_conn.close()
+
+
+def _main(stop_event=None) -> None:
+    live = False
+    if config.LIVE_ENABLED:
+        from ..chain.cluster_guard import SigningRefused, assert_signing_allowed
+        try:
+            assert_signing_allowed(); live = True
+        except SigningRefused as e:
+            if config.SOLANA_CLUSTER != "mainnet-beta":
+                raise                                                   # a wrong cluster is a configuration error, never a wait
+            live = True                                                 # the per-minute gate (fly_session._live) keeps it in paper until the wallet is ready
+            log.error("LIVE_ENABLED=1 but %s -- trading paper only until that is fixed", e)
+            record_event("error", "runner", "live trading requested but not possible yet: holding in paper", {"why": str(e)})
+    if config.RESET_ON_START:                      # a fresh runner never carries over a previous run's books or stats
+        from ..ops.reset import reset_training_state
+        reset_training_state(reason="runner start")
+    from . import minute_engine
+    record_event("info", "runner", "runner started", {"live": live})
+    minute_engine.main(stop_event=stop_event, live=live)

fly_trader/agent/selector_session.py

@@ -0,0 +1,162 @@
+"""The selector's book: the GBM selector trading ``paper_selector`` on the minute engine (agent/minute_engine.py).
+
+Each minute the engine's eligible rows (full feature vectors) are scored by the deployed selector (``brain_snapshots``
+kind 'selector') with its columns picked by name; the book trades them with the shared paper rules
+(agent/paper_trading.py): scores at or above the model's line open positions sized from its measured certainty, held
+``horizon_min`` minutes, then sold at the last traded price (the training label's exit). A newer deployable model is
+picked up without a restart (``maybe_reload``). After the handover (agent/handover.py) the book stops entering while the
+fly trades, exits to flat and retires.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+from datetime import datetime, timezone
+
+import numpy as np
+
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..execution import ledger
+from ..execution.broker_paper import PaperBroker
+from ..train.decisions import X_COLS
+from . import handover, paper_trading
+
+log = logging.getLogger(__name__)
+BOOK, KIND = "paper_selector", "selector"
+
+
+class NoModel(RuntimeError):
+    """No selector has qualified to trade (current data and a profitable backtest after costs)."""
+
+
+def _columns(model) -> np.ndarray:
+    missing = [c for c in model.cols if c not in X_COLS]
+    if missing:
+        raise RuntimeError(f"selector expects features the live engine does not produce: {missing}")
+    return np.asarray([X_COLS.index(c) for c in model.cols], dtype=int)
+
+
+def pinned_snapshot() -> int | None:
+    """The selector snapshot the operator has pinned, or None. A pin freezes the book on one model: a forward test that
+    silently swapped models mid-run could not say which one earned the result."""
+    with transaction() as conn:
+        r = conn.execute("SELECT value FROM ui_settings WHERE key = 'pinned_selector_snapshot'").fetchone()
+    if not r:
+        return None
+    v = r["value"]
+    v = v.get("id") if isinstance(v, dict) else json.loads(v or "{}").get("id")
+    return int(v) if v is not None else None
+
+
+class SelectorBook:
+    name = "selector"
+
+    def __init__(self):
+        from ..train import selector as sel
+        pin = pinned_snapshot()
+        self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest()
+        if self.model is None:
+            raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)")
+        self.idx = _columns(self.model)
+        self.horizon_s = self.model.horizon_min * 60
+        self.broker = PaperBroker(BOOK); self.done = False
+        self.run_id = str(uuid.uuid4()); self.beat_no = 0
+        with transaction() as conn:
+            self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"]   # the snapshot actually loaded
+            conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')",
+                         (self.run_id, KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": BOOK}, default=str), self.snapshot_id))
+        record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min})
+        log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min)
+
+    def maybe_reload(self) -> bool:
+        """Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a
+        snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures
+        one model instead of a blend of every retrain that lands during it."""
+        import joblib
+        from ..train import selector as sel
+        if pinned_snapshot() is not None:
+            return False
+        with transaction() as conn:
+            r = sel.latest_current(conn)
+        if not r or r["id"] == self.snapshot_id:
+            return False
+        m = joblib.load(r["path"])
+        try:
+            idx = _columns(m)
+        except RuntimeError as e:
+            log.error("model #%d: %s; keeping #%s", r["id"], e, self.snapshot_id)
+            return False
+        old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60
+        with transaction() as conn:
+            conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s",
+                         (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": BOOK}, default=str), self.run_id))
+        record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold})
+        log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold)
+        return True
+
+    def on_bars(self, t_start: float, bars: dict) -> None:
+        pass
+
+    def open_mints(self, conn) -> set[str]:
+        return {p["mint"] for p in ledger.open_positions(conn, BOOK)}
+
+    def _decide(self, ctx) -> dict | None:
+        """The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose
+        inputs training had but live lacks are blocked — the skill table of the day missing while skill inputs are used, a
+        token graduated inside the archive whose curve facts are not known yet while rug inputs are used."""
+        if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints):
+            return None
+        d = self.model.decide(ctx.X, X_COLS, ctx.t_start)
+        groups = set((getattr(self.model, "metrics", None) or {}).get("groups") or [])
+        for k, i in enumerate(ctx.infos):
+            if not d["allow"][k]:
+                continue
+            if "skill" in groups and i.get("skill_missing"):
+                d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today"
+            elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"):
+                d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet"
+        return d
+
+    def on_minute(self, ctx) -> dict:
+        conn = ctx.conn
+        d = self._decide(ctx)
+        if d is not None:
+            scores = np.where(np.isfinite(d["score"]), d["score"], -1.0); thr = d["threshold"]
+        else:
+            scores = self.model.score(ctx.X[:, self.idx]) if len(ctx.mints) else np.array([]); thr = self.model.threshold
+        summary = {"minute": ctx.m1.isoformat(), "mints_traded": len(ctx.agg), "eligible": len(ctx.mints),
+                   "picks": int((scores >= thr).sum()) if len(scores) else 0,
+                   "score_p99": float(np.percentile(scores, 99)) if len(scores) else None, "score_max": float(scores.max()) if len(scores) else None,
+                   "threshold": self.model.threshold, "snapshot": self.snapshot_id,
+                   "strategies": sorted(((getattr(self.model, "stack", None) or {}).get("strategies") or {"ev": None}).keys())}
+        if not ctx.trade:
+            return summary
+        if not ctx.fresh:
+            status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()}
+            self._write_status(conn, status)
+            return status
+        handed = handover.state(conn) is not None and ctx.engine.has_book("fly")
+        self.beat_no += 1
+        extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]}
+        st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos,
+                                        scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s,
+                                        block="handed over to the fly" if handed else None, **extra)
+        if handed and st["open"] == 0:
+            self.done = True
+            record_event("info", "selector", "selector retired: the fly holds the seat and the book is flat", {"run_id": self.run_id})
+        status = {**summary, **st, "stage": "retired (handed over to the fly)" if self.done else ("exiting to flat (handed over to the fly)" if handed else "trading"),
+                  "latency_s": (datetime.now(timezone.utc) - ctx.m1).total_seconds(), "updated_at": datetime.now(timezone.utc).isoformat()}
+        self._write_status(conn, status)
+        return status
+
+    @staticmethod
+    def _write_status(conn, status: dict) -> None:
+        conn.execute("INSERT INTO ui_settings (key, value) VALUES ('selector_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                     (json.dumps(status, default=str),))
+
+    def finish(self) -> None:
+        with transaction() as conn:
+            conn.execute("UPDATE runs SET ended_at = now(), status = 'finished' WHERE run_id = %s", (self.run_id,))
+        record_event("info", "selector", "selector session stopped", {"run_id": self.run_id, "minutes": self.beat_no})

fly_trader/agent/sizing.py

@@ -0,0 +1,106 @@
+"""Position sizing: how much of the bankroll a buy gets, from how certain the model is that the trade will profit.
+
+Certainty is measured, not assumed. During training the backtest's out-of-sample trades are grouped by how far their
+score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal
+(Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the
+rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band →
+``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve), capped at
+``MAX_POSITION_FRACTION`` of it, ``MAX_POOL_SHARE`` of the pool's quote reserve (bigger buys pay more price impact than
+the 0.1 SOL the backtest measured) and the free cash above the reserve; below ``MIN_POSITION_SOL`` the buy is skipped.
+A model without a table (trained before sizing) trades the fixed ``MAX_POSITION_SOL``.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+from .. import config
+
+N_BANDS = 4
+MIN_BAND_TRADES = 50
+_F_GRID = np.linspace(0.0, 0.99, 100)
+
+
+def kelly_fraction(r: np.ndarray) -> float:
+    """Fraction of the bankroll per bet that maximises mean log growth over the empirical net returns ``r`` (0 if no edge)."""
+    r = np.asarray(r, dtype=float)
+    r = r[np.isfinite(r)]
+    if len(r) == 0 or r.mean() <= 0:
+        return 0.0
+    growth = np.log1p(np.clip(_F_GRID[:, None] * r[None, :], -0.999999, None)).mean(axis=1)
+    return float(_F_GRID[int(np.argmax(growth))])
+
+
+def build_table(margins: np.ndarray, returns: np.ndarray, n_bands: int = N_BANDS) -> list[dict]:
+    """Bands of score margin (score − the model's threshold) over out-of-sample trades, lowest first:
+    [{lo, n, mean, win, kelly}] where ``lo`` is the band's lowest margin (the first band starts at 0)."""
+    m = np.asarray(margins, dtype=float); r = np.asarray(returns, dtype=float)
+    ok = np.isfinite(m) & np.isfinite(r); m, r = m[ok], r[ok]
+    if len(r) < MIN_BAND_TRADES:
+        return []
+    k = max(1, min(n_bands, len(r) // MIN_BAND_TRADES))
+    edges = np.quantile(m, np.linspace(0, 1, k + 1)); edges[0] = 0.0
+    out = []
+    for b in range(k):
+        sel = (m >= edges[b]) & ((m < edges[b + 1]) if b < k - 1 else np.ones(len(m), bool))
+        rb = r[sel]
+        out.append({"lo": float(edges[b]), "n": int(len(rb)), "mean": float(rb.mean()) if len(rb) else None,
+                    "win": float((rb > 0).mean()) if len(rb) else None, "kelly": kelly_fraction(rb)})
+    return out
+
+
+def band_for(table: list[dict], margin: float) -> dict | None:
+    best = None
+    for b in table:
+        if margin >= b["lo"]:
+            best = b
+    return best
+
+
+def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float,
+                  res_quote_sol: float | None) -> tuple[float, str]:
+    """(size in SOL, why). Size 0 means skip the buy."""
+    reserve = config.GAS_RESERVE_SOL; free = cash_sol - reserve
+    if not table:
+        size = config.MAX_POSITION_SOL
+        return (size, f"fixed {size:g} SOL (model has no sizing table)") if free >= size else (0.0, "not enough free cash")
+    b = band_for(table, score - threshold)
+    if b is None or b["kelly"] <= 0:
+        return 0.0, "no edge in this score band"
+    deployable = max(0.0, bankroll_sol - reserve)
+    size = config.KELLY_FRACTION * b["kelly"] * deployable
+    caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free}
+    if res_quote_sol:
+        caps["pool depth"] = config.MAX_POOL_SHARE * res_quote_sol
+    limit = min(caps, key=caps.get)
+    why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL"
+    if caps[limit] < size:
+        size = caps[limit]; why += f", capped by {limit}"
+    if size < config.MIN_POSITION_SOL:
+        return 0.0, why + f" → {size:.3f} SOL is below the {config.MIN_POSITION_SOL:g} SOL minimum"
+    return float(size), why + f" → {size:.3f} SOL"
+
+
+def simulate_bankroll(ts: np.ndarray, horizon_s, margins: np.ndarray, returns: np.ndarray, table: list[dict] | None,
+                      start_sol: float | None = None, res_quote: np.ndarray | None = None, tables: list | None = None) -> dict:
+    """Replay out-of-sample trades in entry order with the sizing rule (positions overlap for ``horizon_s``; cash, the
+    reserve and — given ``res_quote``, each trade's pool quote reserve — the pool-depth cap are enforced, as live).
+    Returns the final bankroll, its multiple and the worst drawdown."""
+    start = float(start_sol or config.CAPITAL_SOL or 1.0); cash = start; open_: list[tuple[float, float, float]] = []   # (exit_ts, cost, return)
+    hv = np.asarray(horizon_s, dtype=np.float64); hold = np.full(len(ts), float(hv)) if hv.ndim == 0 else hv    # per-trade holds allowed
+    peak = start; mdd = 0.0; n = 0
+    for i in np.argsort(ts, kind="stable"):
+        t = float(ts[i])
+        still = []
+        for x in open_:
+            if x[0] <= t:
+                cash += x[1] * (1 + x[2])
+            else:
+                still.append(x)
+        open_ = still
+        bankroll = cash + sum(x[1] for x in open_)
+        size, _ = size_position(float(margins[i]), 0.0, tables[i] if tables is not None else table, bankroll, cash, float(res_quote[i]) if res_quote is not None else None)
+        if size > 0:
+            cash -= size; open_.append((t + float(hold[i]), size, float(returns[i]))); n += 1
+        w = cash + sum(x[1] for x in open_); peak = max(peak, w); mdd = max(mdd, 1 - w / peak if peak > 0 else 0.0)
+    final = cash + sum(x[1] * (1 + x[2]) for x in open_)
+    return {"start_sol": start, "final_sol": final, "multiple": final / start if start > 0 else None, "max_drawdown": mdd, "trades": n}

fly_trader/brain/__init__.py

@@ -0,0 +1 @@
+

fly_trader/brain/connectome.py

@@ -0,0 +1,171 @@
+"""The FlyWire connectome the fly (``train/fly_selector.py``) is built on: the artifact written by ``build-connectome``
+(``brain/connectome_build.py``), loaded as signed synapse counts with population ranges, and restricted to a set of
+populations by ``SubConnectome``.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from .. import config
+
+CONNECTOME_DIR = config.BRAIN_DIR / "connectome"
+CURRENT_FILE = CONNECTOME_DIR / "current.txt"
+SCALE_FILE = CONNECTOME_DIR / "scale.json"
+AFFERENT_POPS = ("ORN_FOOD", "ORN_DANGER", "GRN_SWEET", "GRN_BITTER", "GRN_OTHER", "MECH_JO", "MECH_BRISTLE",
+                 "MECH_OTHER", "THERMO_WARM", "THERMO_COOL", "THERMO_OTHER")     # sensory neurons: where the features enter
+EFFERENT_POP = "DESCENDING"                                                     # brain-to-body neurons: where the decoder reads
+
+
+def resolve_device(name: str | None = None) -> torch.device:
+    """config.DEVICE with automatic fallback to CPU when MPS (or CUDA) is unavailable."""
+    name = (name or config.DEVICE or "cpu").lower()
+    if name.startswith("mps"):
+        if torch.backends.mps.is_available():
+            return torch.device("mps")
+        print("[connectome] MPS unavailable; falling back to cpu", file=sys.stderr)
+        return torch.device("cpu")
+    if name.startswith("cuda"):
+        if torch.cuda.is_available():
+            return torch.device(name)
+        print("[connectome] CUDA unavailable; falling back to cpu", file=sys.stderr)
+        return torch.device("cpu")
+    return torch.device("cpu")
+
+
+def current_connectome_path() -> Path:
+    if not CURRENT_FILE.exists():
+        raise FileNotFoundError(f"{CURRENT_FILE} missing: run `fly-trader build-connectome` first")
+    name = CURRENT_FILE.read_text().strip()
+    path = Path(name) if Path(name).is_absolute() else CONNECTOME_DIR / name
+    if not path.exists():
+        raise FileNotFoundError(f"connectome {path} named in current.txt does not exist")
+    return path
+
+
+def _scale_for(connectome_file: str) -> dict | None:
+    """The calibrated scale if scale.json exists and refers to this artifact."""
+    if not SCALE_FILE.exists():
+        return None
+    try:
+        d = json.loads(SCALE_FILE.read_text())
+    except Exception:
+        return None
+    return d if d.get("connectome") in (connectome_file, Path(connectome_file).name) else None
+
+
+MB_POPS = ("KC", "MBON_APP", "MBON_AV", "MBON_OTHER")    # the mushroom-body block the build keeps apart from the sparse graph
+DAN_POPS = ("DAN_PAM", "DAN_PPL1", "DAN_OTHER")
+
+
+def mbon_compartments(indices: torch.Tensor, values_raw: torch.Tensor, pop_ranges: dict, cell_type=None) -> tuple[np.ndarray, list[str]]:
+    """Each MBON column's compartment: the dopamine cell type (or, without cell types, the DAN population) sending it the
+    most synapse mass in the raw graph — before scale.json's gains silence DAN→MBON as fast edges (Aso et al. 2014: a
+    compartment = the DAN type innervating that MBON's dendrites). −1 when no DAN reaches the MBON."""
+    if not all(p in pop_ranges for p in ("MBON_APP", "MBON_OTHER")):
+        return np.zeros(0, int), []
+    m0, m1 = pop_ranges["MBON_APP"][0], pop_ranges["MBON_OTHER"][1]
+    post, pre = indices[0].numpy(), indices[1].numpy(); w = np.abs(values_raw.numpy())
+    dan = np.zeros(len(pre), bool); label = np.full(len(pre), "", dtype=object)
+    for pop in DAN_POPS:
+        if pop in pop_ranges:
+            a, b = pop_ranges[pop]; sel = (pre >= a) & (pre < b); dan |= sel
+            label[sel] = np.asarray(cell_type, dtype=object)[pre[sel]] if cell_type is not None else pop
+    e = dan & (post >= m0) & (post < m1)
+    names = sorted({str(x) or "DAN" for x in label[e]})
+    comp = np.full(m1 - m0, -1, int)
+    if not e.any():
+        return comp, names
+    mass = np.zeros((m1 - m0, len(names))); col = {n: i for i, n in enumerate(names)}
+    np.add.at(mass, (post[e] - m0, np.array([col[str(x) or "DAN"] for x in label[e]])), w[e])
+    has = mass.sum(1) > 0; comp[has] = mass[has].argmax(1)
+    return comp, names
+
+
+def apply_pathway_gains(indices: torch.Tensor, values_raw: torch.Tensor, pop_ranges: dict, gains: dict) -> tuple[torch.Tensor, torch.Tensor]:
+    """Multiply the edges from population A to population B by ``gains['A->B']``; edges scaled to 0 are dropped.
+    scale.json silences the direct DAN→MBON/KC edges: dopamine acts on the mushroom body through KC→MBON plasticity,
+    not fast excitation (Hige 2015, Handler 2019)."""
+    post, pre = indices[0], indices[1]
+    v = values_raw.clone()
+    for key, g in (gains or {}).items():
+        a, _, b = key.partition("->")
+        if a not in pop_ranges or b not in pop_ranges:
+            continue
+        (a0, a1), (b0, b1) = pop_ranges[a], pop_ranges[b]
+        v[(pre >= a0) & (pre < a1) & (post >= b0) & (post < b1)] *= float(g)
+    keep = v != 0
+    return indices[:, keep], v[keep]
+
+
+class Connectome:
+    """Signed synapse counts ``values_raw`` [E] on edges ``indices`` [2, E] (row = post, col = pre) over ``N`` neurons,
+    population ``pop_ranges``, and the weight scale ``s`` (scale.json when it refers to this artifact, else the build's
+    s0). The KC→MBON synapses are not in the sparse graph: the build keeps them as the dense block ``W_KM0``
+    [n_KC, n_MBON] (synapse counts; MBON columns = MBON_APP, MBON_AV, MBON_OTHER in population order) with the mask
+    ``M_KM`` of the pairs that exist. scale.json's ``pathway_gains`` are applied to the sparse graph. Tensors stay on
+    the CPU; ``device`` is where models built on it should run."""
+
+    def __init__(self, path: str | Path | None = None, device: torch.device | str | None = None):
+        path = Path(path) if path else current_connectome_path()
+        z = np.load(path, allow_pickle=False)
+        self.path = path
+        self.N = int(z["N"])
+        self.pop_ranges = {k: (int(a), int(b)) for k, (a, b) in json.loads(str(z["pop_ranges"])).items()}
+        scale = _scale_for(path.name)
+        self.pathway_gains = dict((scale or {}).get("pathway_gains") or {})
+        raw_i = torch.from_numpy(np.ascontiguousarray(z["indices"])).to(torch.int64); raw_v = torch.from_numpy(np.ascontiguousarray(z["values_raw"])).to(torch.float32)
+        self.mbon_compartment, self.compartment_names = mbon_compartments(raw_i, raw_v, self.pop_ranges, z["cell_type"] if "cell_type" in z.files else None)
+        self.indices, self.values_raw = apply_pathway_gains(torch.from_numpy(np.ascontiguousarray(z["indices"])).to(torch.int64),
+                                                            torch.from_numpy(np.ascontiguousarray(z["values_raw"])).to(torch.float32),
+                                                            self.pop_ranges, self.pathway_gains)
+        self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32)
+        self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool)
+        self.spectral_radius_raw = float(z["spectral_radius_raw"])
+        self.content_sha256 = str(z["content_sha256"])
+        self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"])
+        self.device = device if isinstance(device, torch.device) else resolve_device(device)
+
+    @classmethod
+    def load(cls, path: str | Path | None = None, device: torch.device | str | None = None) -> "Connectome":
+        """The artifact named in data/brain/connectome/current.txt (or ``path``)."""
+        return cls(path, device=device)
+
+
+class SubConnectome:
+    """A connectome restricted to a set of populations (e.g. everything but the optic lobes)."""
+
+    def __init__(self, c, exclude: tuple[str, ...] = ("VISUAL",)):
+        keep = torch.ones(c.N, dtype=torch.bool)
+        for name in exclude:
+            if name in c.pop_ranges:
+                lo, hi = c.pop_ranges[name]
+                keep[lo:hi] = False
+        new_index = torch.cumsum(keep.to(torch.int64), 0) - 1
+        post, pre = c.indices[0], c.indices[1]
+        m = keep[post] & keep[pre]
+        self.indices = torch.stack([new_index[post[m]], new_index[pre[m]]])
+        self.values_raw = c.values_raw[m].clone()
+        self.N = int(keep.sum())
+        self.pop_ranges = {}
+        for name, (lo, hi) in c.pop_ranges.items():
+            if name in exclude or hi <= lo:
+                continue
+            self.pop_ranges[name] = (int(new_index[lo]), int(new_index[hi - 1]) + 1)
+        self.device = c.device
+        self.s = getattr(c, "s", None)
+        self.spectral_radius_raw = getattr(c, "spectral_radius_raw", None)
+        self.content_sha256 = getattr(c, "content_sha256", None)
+        lost = [p for p in MB_POPS if p in c.pop_ranges and not bool(keep[c.pop_ranges[p][0]:c.pop_ranges[p][1]].all())]
+        if lost:
+            raise ValueError(f"a sub-connectome must keep the whole mushroom body (KC→MBON block); excluded: {lost}")
+        self.W_KM0 = getattr(c, "W_KM0", None)          # unchanged: KC and MBON rows keep their order and ranges
+        self.mbon_compartment = getattr(c, "mbon_compartment", None); self.compartment_names = getattr(c, "compartment_names", [])
+        self.M_KM = getattr(c, "M_KM", None)
+        self.pathway_gains = getattr(c, "pathway_gains", {})
+        self.excluded = exclude
+        self.keep = keep

fly_trader/brain/connectome_build.py

@@ -0,0 +1,337 @@
+"""Build the FAFB v783 connectome artifact (``fly-trader build-connectome``).
+
+Inputs (data/brain/raw/): flybrain ``connections.csv.gz`` (pre_root_id, post_root_id, neuropil,
+syn_count, nt_type; one row per (pre, post, neuropil)), ``neurons.csv.gz``, ``classification.csv.gz``
+and the Schlegel et al. 2024 annotation TSV. Output: ``data/brain/connectome/fafb783_<sha12>.npz``
+(COO ``indices`` int64[2, nnz] with row = post, col = pre, sorted; ``values`` float32 pre-calibration
+weights; ``values_raw`` = signed synapse counts; the dense plastic KC->MBON block ``W_KM0``) plus
+``current.txt`` naming the artifact, the ``populations`` table and a ``runs`` row.
+
+Sign convention (config.NT_SIGN_MODE): ``shiu`` = GABA and glutamate inhibitory, everything else
+excitatory (Shiu et al. 2024 Nature 634:210); ``flybrain`` = only GABA inhibitory (flybrain's
+NT_SIGN). Weights are summed over neuropils per (pre, post) pair with the edge's own nt_type
+(nt_type varies within a pre neuron in FlyWire's connection table); zero sums are dropped, as
+flybrain does, which reproduces its 2,698,236 edges in ``flybrain`` mode.
+
+Scale: s0 = 0.99 / rho(|W_raw|) (spectral radius by shifted power iteration) is only the INITIAL
+guess, motivated by Costi et al. 2025 (PMC12109256), who ran the FlyWire connectome as an echo-state
+reservoir rescaled to rho = 0.99 (tanh units, glutamate treated as excitatory there). The fly
+(``train/fly_selector.py``) initialises its synaptic magnitudes from s (``scale.json`` when present, else s0).
+KC->MBON magnitudes are forced positive because KC output synapses are cholinergic (Barnstedt et
+al. 2016 Neuron), regardless of FlyWire's per-cell NT prediction.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import sys
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+import warnings
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.csv as pc
+import torch
+
+warnings.filterwarnings("ignore", message="Sparse CSR tensor support is in beta state")
+warnings.filterwarnings("ignore", message="Sparse invariant checks are implicitly disabled")
+
+from .. import config
+from . import populations as P
+
+CONNECTIONS_CSV = P.RAW_DIR / "connections.csv.gz"
+CONNECTOME_DIR = config.BRAIN_DIR / "connectome"
+CURRENT_FILE = CONNECTOME_DIR / "current.txt"
+EXPECTED_EDGES = 2_698_236
+W_CLIP = 1.0                 # = theta (plan section 1.7)
+TARGET_RADIUS = 0.99         # reservoir operating point (Vienna 2025)
+POWER_ITER_MAX = 5000
+POWER_ITER_TOL = 1e-9
+SOURCE_FILES = ["connections.csv.gz", "neurons.csv.gz", "classification.csv.gz",
+                "Supplemental_file1_neuron_annotations.tsv", "consolidated_cell_types.csv.gz", "neuron_meta.json"]
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def nt_sign(nt: np.ndarray, mode: str) -> np.ndarray:
+    """+1/-1 per edge from the edge's nt_type (flybrain vocabulary ACH/GABA/GLUT/DA/SER/OCT)."""
+    nt = np.char.upper(nt.astype(str))
+    if mode == "shiu":
+        neg = (nt == "GABA") | (nt == "GLUT")
+    elif mode == "flybrain":
+        neg = nt == "GABA"
+    else:
+        raise ValueError(f"unknown NT_SIGN_MODE {mode!r} (shiu | flybrain)")
+    return np.where(neg, -1.0, 1.0)
+
+
+def read_connections(path: Path = CONNECTIONS_CSV) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    tbl = pc.read_csv(str(path), convert_options=pc.ConvertOptions(column_types={
+        "pre_root_id": pa.int64(), "post_root_id": pa.int64(), "neuropil": pa.string(),
+        "syn_count": pa.int32(), "nt_type": pa.string()}))
+    pre = tbl["pre_root_id"].to_numpy()
+    post = tbl["post_root_id"].to_numpy()
+    syn = tbl["syn_count"].to_numpy().astype(np.int64)
+    nt = tbl["nt_type"].to_numpy(zero_copy_only=False).astype(str)
+    return pre, post, syn, nt
+
+
+def root_id_lookup(canonical_root_ids: np.ndarray):
+    order = np.argsort(canonical_root_ids)
+    srt = canonical_root_ids[order]
+
+    def lookup(x: np.ndarray) -> np.ndarray:
+        pos = np.searchsorted(srt, x)
+        pos[pos >= len(srt)] = 0
+        return np.where(srt[pos] == x, order[pos], -1)
+    return lookup
+
+
+def aggregate_edges(pre: np.ndarray, post: np.ndarray, syn: np.ndarray, nt: np.ndarray, N: int, mode: str) -> dict:
+    """Sum syn_count * sign over neuropils per (pre, post). Returns arrays over distinct pairs."""
+    key = pre.astype(np.int64) * N + post.astype(np.int64)
+    uniq, inv = np.unique(key, return_inverse=True)
+    n = len(uniq)
+    sgn = nt_sign(nt, mode)
+    signed = np.bincount(inv, weights=syn * sgn, minlength=n)
+    signed_fb = np.bincount(inv, weights=syn * nt_sign(nt, "flybrain"), minlength=n)
+    unsigned = np.bincount(inv, weights=syn, minlength=n)
+    return {
+        "pre": (uniq // N).astype(np.int64), "post": (uniq % N).astype(np.int64),
+        "signed": signed, "unsigned": unsigned,
+        "n_pairs": int(n), "nnz_mode": int((signed != 0).sum()), "nnz_flybrain": int((signed_fb != 0).sum()),
+    }
+
+
+def spectral_radius(row: np.ndarray, col: np.ndarray, values_abs: np.ndarray, N: int,
+                    max_iter: int = POWER_ITER_MAX, tol: float = POWER_ITER_TOL) -> dict:
+    """rho(|W|) by power iteration on the shifted matrix |W| + I (rho(|W| + I) = rho(|W|) + 1 for a
+    non-negative matrix; the shift makes the iteration converge for periodic components). Also
+    returns the Collatz-Wielandt bracket min_i (|W|x)_i / x_i <= rho <= max_i (|W|x)_i / x_i."""
+    idx = torch.from_numpy(np.stack([row, col]).astype(np.int64))
+    W = torch.sparse_coo_tensor(idx, torch.from_numpy(values_abs.astype(np.float64)), (N, N)).coalesce().to_sparse_csr()
+    x = torch.full((N, 1), 1.0 / math.sqrt(N), dtype=torch.float64)
+    lam_prev, it, converged = 0.0, 0, False
+    for it in range(1, max_iter + 1):
+        y = W @ x + x
+        lam = float(torch.linalg.vector_norm(y))
+        x = y / lam
+        if abs(lam - lam_prev) <= tol * lam:
+            converged = True
+            break
+        lam_prev = lam
+    wx = (W @ x).squeeze(1)
+    xs = x.squeeze(1)
+    ratio = wx / xs
+    return {"rho": lam - 1.0, "iterations": it, "converged": converged,
+            "cw_lower": float(ratio.min()), "cw_upper": float(ratio.max())}
+
+
+def content_hash(indices: np.ndarray, values_raw: np.ndarray, W_KM0: np.ndarray, root_ids: np.ndarray,
+                 pop_ranges_json: str, nt_sign_mode: str) -> str:
+    h = hashlib.sha256()
+    for part in (indices.tobytes(), values_raw.tobytes(), W_KM0.tobytes(), root_ids.tobytes(),
+                 pop_ranges_json.encode(), nt_sign_mode.encode()):
+        h.update(part)
+    return h.hexdigest()
+
+
+def _git_sha() -> str | None:
+    try:
+        import subprocess
+        out = subprocess.run(["git", "rev-parse", "HEAD"], cwd=config.REPO_ROOT, capture_output=True, text=True, timeout=5)
+        return out.stdout.strip() or None if out.returncode == 0 else None
+    except Exception:
+        return None
+
+
+def persist(pops: P.Populations, metrics: dict, build_cfg: dict, file_sha: str, started_at: datetime,
+            annotations_label: str) -> str:
+    """populations rows (id = order index) + runs row; returns the run_id."""
+    from psycopg.types.json import Jsonb
+    from ..db.connection import transaction
+    run_id = str(uuid.uuid4())
+    with transaction() as conn:
+        with conn.cursor() as cur:
+            for i, name in enumerate(pops.order):
+                cur.execute("DELETE FROM populations WHERE name = %s AND id <> %s", (name, i))
+                cur.execute(
+                    "INSERT INTO populations (id, name, n, kind, source) VALUES (%s, %s, %s, %s, %s) "
+                    "ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, n = EXCLUDED.n, kind = EXCLUDED.kind, "
+                    "source = EXCLUDED.source",
+                    (i, name, pops.n(name), pops.kind.get(name, "other"), annotations_label))
+            cur.execute(
+                "INSERT INTO runs (run_id, kind, started_at, ended_at, git_sha, config, connectome_sha256, status, metrics) "
+                "VALUES (%s, 'connectome_build', %s, now(), %s, %s, %s, 'done', %s)",
+                (run_id, started_at, _git_sha(), Jsonb(build_cfg), file_sha, Jsonb(metrics)))
+            if pops.source != "annotations":
+                cur.execute(
+                    "INSERT INTO events (level, source, message, detail) VALUES ('warning', 'connectome_build', %s, %s)",
+                    ("cell types from flybrain classification.csv fallback (annotation TSV unavailable); "
+                     "MBON valence by neurotransmitter, DAN valence by synapse counts",
+                     Jsonb({"run_id": run_id})))
+    return run_id
+
+
+def main(annotations: str | None = None, nt_sign_mode: str | None = None, source: str | None = None,
+         out_dir: str | Path | None = None, persist_db: bool = True) -> Path:
+    started_at = datetime.now(timezone.utc)
+    t0 = time.perf_counter()
+    mode = nt_sign_mode or config.NT_SIGN_MODE
+    out_dir = Path(out_dir) if out_dir else CONNECTOME_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # ---- source hashes ----
+    source_sha = {}
+    for name in SOURCE_FILES:
+        p = P.RAW_DIR / name
+        if p.exists():
+            source_sha[name] = sha256_file(p)
+    if annotations:
+        source_sha[Path(annotations).name] = sha256_file(Path(annotations))
+
+    # ---- connections (canonical flybrain indices) ----
+    pre_rid, post_rid, syn, nt = read_connections()
+    n_rows = len(pre_rid)
+    neu_ids = np.asarray(__import__("pandas").read_csv(P.NEURONS_CSV, usecols=["root_id"], dtype={"root_id": np.int64})["root_id"])
+    N = len(neu_ids)
+    lookup = root_id_lookup(neu_ids)
+    pre_c, post_c = lookup(pre_rid), lookup(post_rid)
+    ok = (pre_c >= 0) & (post_c >= 0)
+    n_skipped = int((~ok).sum())
+    unknown_nt = int((~np.isin(np.char.upper(nt), ["ACH", "GABA", "GLUT", "DA", "SER", "OCT", "OA"])).sum())
+    pre_c, post_c, syn, nt = pre_c[ok], post_c[ok], syn[ok], nt[ok]
+    print(f"[build] connections rows={n_rows:,} skipped_unknown_root_id={n_skipped} unknown_nt_rows={unknown_nt} "
+          f"({time.perf_counter() - t0:.1f}s)")
+
+    # ---- populations ----
+    pops = P.build_populations(annotations=annotations, source=source, edges=(pre_c, post_c, syn.astype(np.float64)))
+    assert pops.N == N
+    inv = pops.inverse_index()          # canonical -> sorted position
+    print(f"[build] populations source={pops.source} N={N:,} ({time.perf_counter() - t0:.1f}s)")
+
+    # ---- aggregate over neuropils ----
+    agg = aggregate_edges(pre_c, post_c, syn, nt, N, mode)
+    nnz_fb, nnz_mode, n_pairs = agg["nnz_flybrain"], agg["nnz_mode"], agg["n_pairs"]
+    edges_match = nnz_fb == EXPECTED_EDGES
+    print(f"[build] distinct (pre,post) pairs={n_pairs:,}; nonzero edges: flybrain-sign={nnz_fb:,} "
+          f"(expected {EXPECTED_EDGES:,} -> {'MATCH' if edges_match else 'MISMATCH'}), {mode}-sign={nnz_mode:,}")
+    if not edges_match:
+        print(f"[build] WARNING flybrain-mode edge count {nnz_fb:,} != {EXPECTED_EDGES:,}; continuing", file=sys.stderr)
+    keep = agg["signed"] != 0
+    pre_s = inv[agg["pre"][keep]]        # sorted (population-order) indices
+    post_s = inv[agg["post"][keep]]
+    signed = agg["signed"][keep]
+    unsigned = agg["unsigned"][keep]
+
+    # ---- KC -> MBON block (dense, positive magnitudes) ----
+    kc0, kc1 = pops.ranges["KC"]
+    m0 = pops.ranges["MBON_APP"][0]
+    m1 = pops.ranges["MBON_OTHER"][1]
+    assert pops.ranges["MBON_APP"][1] == pops.ranges["MBON_AV"][0] and pops.ranges["MBON_AV"][1] == pops.ranges["MBON_OTHER"][0]
+    n_kc, n_mbon = kc1 - kc0, m1 - m0
+    is_km = (pre_s >= kc0) & (pre_s < kc1) & (post_s >= m0) & (post_s < m1)
+    W_KM0 = np.zeros((n_kc, n_mbon), dtype=np.float32)
+    W_KM0[pre_s[is_km] - kc0, post_s[is_km] - m0] = np.abs(unsigned[is_km]).astype(np.float32)
+    M_KM = W_KM0 != 0
+    n_km_neg = int((signed[is_km] < 0).sum())
+    rest = ~is_km
+    row = post_s[rest]
+    col = pre_s[rest]
+    values_raw = signed[rest].astype(np.float32)
+    order = np.lexsort((col, row))       # coalesced: sorted by (row, col)
+    row, col, values_raw = row[order], col[order], values_raw[order]
+    indices = np.stack([row, col]).astype(np.int64)
+    nnz = int(len(values_raw))
+    print(f"[build] KC->MBON pairs={int(M_KM.sum()):,} (non-cholinergic sign on {n_km_neg} pairs ignored, magnitudes kept); "
+          f"sparse nnz={nnz:,} ({time.perf_counter() - t0:.1f}s)")
+
+    # ---- spectral radius and initial scale ----
+    sr = spectral_radius(row, col, np.abs(values_raw), N)
+    rho = sr["rho"]
+    s0 = TARGET_RADIUS / rho
+    values = np.clip(s0 * values_raw, -W_CLIP, W_CLIP).astype(np.float32)
+    print(f"[build] rho(|W_raw|)={rho:.3f} (Collatz-Wielandt upper bound {sr['cw_upper']:.3f}, "
+          f"{sr['iterations']} iters, converged={sr['converged']}) s0={s0:.6g} ({time.perf_counter() - t0:.1f}s)")
+
+    # ---- artifact ----
+    pop_ranges_json = json.dumps(pops.ranges)
+    chash = content_hash(indices, values_raw, W_KM0, pops.root_ids, pop_ranges_json, mode)
+    fname = f"fafb783_{chash[:12]}.npz"
+    path = out_dir / fname
+    build_cfg = {
+        "nt_sign_mode": mode, "cell_types_source": pops.source,
+        "annotations": str(Path(annotations) if annotations else P.ANNOTATIONS_TSV),
+        "w_clip": W_CLIP, "target_radius": TARGET_RADIUS, "power_iter_max": POWER_ITER_MAX, "power_iter_tol": POWER_ITER_TOL,
+        "expected_edges": EXPECTED_EDGES, "torch": torch.__version__, "numpy": np.__version__,
+        "built_at": started_at.isoformat(), "content_sha256": chash, "file": fname,
+        "population_order": pops.order,
+    }
+    counts = pops.counts()
+    metrics = {
+        "N": N, "rows": n_rows, "distinct_pairs": n_pairs, "nnz_flybrain_mode": nnz_fb, "nnz_mode": nnz_mode,
+        "edges_expected": EXPECTED_EDGES, "edges_match": edges_match, "nnz_sparse": nnz,
+        "skipped_rows": n_skipped, "unknown_nt_rows": unknown_nt,
+        "kc_mbon_pairs": int(M_KM.sum()), "kc_mbon_max_syn": float(W_KM0.max()), "kc_mbon_mean_syn": float(W_KM0[M_KM].mean()),
+        "spectral_radius_raw": rho, "spectral_radius_bracket": [sr["cw_lower"], sr["cw_upper"]],
+        "power_iterations": sr["iterations"], "power_converged": sr["converged"], "s0": s0,
+        "n_glomeruli": len(pops.glomerulus_names), "orn_food_untyped": int((pops.glomerulus_of_orn < 0).sum()),
+        "populations": counts, "cell_types_source": pops.source,
+        "build_seconds": None,
+    }
+    if path.exists():
+        print(f"[build] {path} already exists with identical content hash; keeping it")
+    else:
+        tmp = out_dir / f"{fname}.tmp.npz"   # numpy appends .npz to names that lack it
+        np.savez_compressed(
+            tmp, indices=indices, values=values, values_raw=values_raw, N=np.int64(N),
+            pop_ranges=np.array(pop_ranges_json), pop_order=np.array(pops.order),
+            root_ids=pops.root_ids, nt=pops.nt.astype(str), flybrain_group=pops.flybrain_group,
+            flybrain_group_names=np.array(pops.flybrain_group_names), cell_type=pops.cell_type.astype(str),
+            W_KM0=W_KM0, M_KM=M_KM,
+            kc_idx=pops.index("KC"), mbon_app_idx=pops.index("MBON_APP"), mbon_av_idx=pops.index("MBON_AV"),
+            mbon_other_idx=pops.index("MBON_OTHER"), dan_pam_idx=pops.index("DAN_PAM"), dan_ppl1_idx=pops.index("DAN_PPL1"),
+            glomerulus_of_orn=pops.glomerulus_of_orn, glomerulus_names=np.array(pops.glomerulus_names, dtype=str),
+            spectral_radius_raw=np.float64(rho), s0=np.float64(s0), nt_sign_mode=np.array(mode),
+            w_clip=np.float32(W_CLIP), source_sha256=np.array(json.dumps(source_sha)),
+            build_config=np.array(json.dumps(build_cfg)), content_sha256=np.array(chash),
+            canonical_index=pops.canonical_index,
+        )
+        os.replace(tmp, path)
+    file_sha = sha256_file(path)
+    CURRENT_FILE.parent.mkdir(parents=True, exist_ok=True)
+    if out_dir == CONNECTOME_DIR:
+        CURRENT_FILE.write_text(fname + "\n")
+    metrics["build_seconds"] = round(time.perf_counter() - t0, 1)
+    metrics["file_sha256"] = file_sha
+
+    # ---- report ----
+    print(f"[build] N={N:,} nnz={nnz:,} KC={counts['KC']} MBON_APP={counts['MBON_APP']} MBON_AV={counts['MBON_AV']} "
+          f"MBON_OTHER={counts['MBON_OTHER']} DAN_PAM={counts['DAN_PAM']} DAN_PPL1={counts['DAN_PPL1']} "
+          f"glomeruli={len(pops.glomerulus_names)}")
+    for name in pops.order:
+        print(f"    {name:<13} {counts[name]:>7}  kind={pops.kind.get(name, 'other')}")
+    print(f"[build] wrote {path} ({path.stat().st_size / 1e6:.1f} MB, sha256 {file_sha[:12]}); current.txt -> {fname}")
+    if counts["KC"] != P.EXPECTED_KC or (counts["MBON_APP"] + counts["MBON_AV"] + counts["MBON_OTHER"]) != P.EXPECTED_MBON:
+        raise RuntimeError("KC/MBON counts do not match FAFB v783 expectations")  # build_populations already checks
+    if persist_db:
+        label = f"{pops.source}:{Path(build_cfg['annotations']).name if pops.source == 'annotations' else 'classification.csv.gz'}"
+        run_id = persist(pops, metrics, {"build": build_cfg, "env": config.summary()}, file_sha, started_at, label)
+        print(f"[build] runs row {run_id} (kind=connectome_build) and {len(pops.order)} populations rows written")
+    return path
+
+
+if __name__ == "__main__":
+    main()

fly_trader/brain/plastic.py

@@ -0,0 +1,275 @@
+"""Mushroom-body plasticity: after its bootstrap the fly keeps learning from the market, at the KC→MBON synapses only.
+
+Rule (dopamine-gated three-factor; Hige 2015, Cohn 2015, Bennett 2021; masked like the connectome, FlyModel-style
+associative updates, Shen/Dasgupta/Navlakha 2023). A plastic change ``D`` [n_KC, n_MBON] of the KC→MBON weights acts
+at the network's last step (``FlyNet.readout``). For a scored minute with KC code ``k`` (presynaptic activity), MBON
+activity ``m`` and, 120 minutes later, its realized net return ``r``:
+
+    δ   = clip(clip(r, ±1) − ŷ, ±DELTA_CLIP)                  dopamine: the prediction error (ŷ recomputed with the current D)
+    g_j = c_j γ_j (1 − m_j²) / (σ_j · scale)                   how MBON j moves the prediction (∂ŷ/∂D_ij = k_i g_j, exact)
+    ΔD  = (α/ν) · M ⊙ Σ_rows w δ k gᵀ                          w = 1 on minutes scored at/above the line, else 0
+
+which is gradient descent on the weighted squared prediction error: reward (δ > 0) depresses KC→avoid-MBON synapses
+(c < 0) and potentiates KC→approach ones; punishment the reverse. ``ν`` (the median squared norm of one row's update,
+fixed at bootstrap) makes ``α`` the share of a row's error one update corrects. Changes decay toward the bootstrap weights
+with half-life ``τ½`` in event time (dopamine-gated forgetting, Berry 2012), each update is capped at ``STEP_CAP`` of the
+bootstrap weights' norm, and ``D`` stays within [−w0, 2·w0] (effective weights within [0, 3·w0]). ``β_pot`` < 1 damps
+potentiation (depression-dominant; still a descent direction). ``D = 0`` is the frozen fly (the shadow).
+
+The label arrives two hours after the decision, far beyond any biological eligibility window (seconds: Handler 2019),
+so each scored minute leaves a synaptic tag (``PendingTags``: its KC code, MBON pre-activation and descending output)
+that is captured when the label resolves (synaptic tag-and-capture, Frey & Morris 1997).
+
+``PlasticBank`` holds ``C`` configurations side by side (the replay's grid; 1 live).
+"""
+from __future__ import annotations
+
+import bisect
+import itertools
+import math
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+
+DELTA_CLIP = 0.5
+RET_CLIP = 1.0
+TOP_WEIGHT = 10.0
+STEP_CAP = 1e-3              # ‖ΔD‖_F per update ≤ STEP_CAP · ‖w0‖_F
+DAY_S = 86400.0
+
+
+@dataclass
+class Tags:
+    """A batch of captured tags: row keys, the parts cached at scoring and each configuration's weight [C, B]."""
+    keys: list
+    ts: np.ndarray
+    y_dn: torch.Tensor      # [B]
+    u0: torch.Tensor        # [B, n_MBON]
+    k: torch.Tensor         # [B, n_KC] dense KC code
+    w: torch.Tensor         # [C, B]
+    s: np.ndarray = None    # [B] strategy (dopamine channel) of each tag; None = channel 0
+
+    def __post_init__(self):
+        if self.s is None:
+            self.s = np.zeros(len(self.keys), dtype=np.int64)
+
+    def __len__(self) -> int:
+        return len(self.keys)
+
+    def subset(self, keep: np.ndarray) -> "Tags":
+        keep = np.asarray(keep, dtype=bool); i = torch.as_tensor(np.flatnonzero(keep), device=self.y_dn.device)
+        return Tags([k for k, m in zip(self.keys, keep) if m], self.ts[keep], self.y_dn[i], self.u0[i], self.k[i], self.w[:, i], self.s[keep])
+
+
+class PendingTags:
+    """Scored minutes waiting for their labels, kept in due order (strategies hold for different times). The KC code is
+    stored sparsely (its active cells)."""
+
+    def __init__(self, n_kc: int, k_active: int):
+        self.n_kc, self.k_active = n_kc, k_active
+        self.q: list = []; self.n = 0; self._seq = itertools.count()
+
+    def __len__(self) -> int:
+        return self.n
+
+    def push(self, keys: list, ts: np.ndarray, due: float, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor, w: torch.Tensor, s=None) -> None:
+        """Tags capturable from event time ``due`` on (all rows of one push share it); ``s``: their strategy channel(s)."""
+        if len(keys) == 0:
+            return
+        vals, idx = k.detach().topk(min(self.k_active, k.shape[1]), dim=1)
+        ch = np.broadcast_to(np.asarray(0 if s is None else s, dtype=np.int64), (len(keys),)).copy()
+        chunk = {"due": float(due), "keys": list(keys), "ts": np.asarray(ts, dtype=np.float64), "y_dn": y_dn.detach().float().cpu(), "s": ch,
+                 "u0": u0.detach().float().cpu(), "idx": idx.to(torch.int32).cpu(), "vals": vals.float().cpu(), "w": w.detach().float().cpu()}
+        bisect.insort(self.q, (float(due), next(self._seq), chunk))
+        self.n += len(keys)
+
+    def pop_due(self, now: float, device=None) -> Tags | None:
+        out = []
+        while self.q and self.q[0][0] <= now:
+            out.append(self.q.pop(0)[2])
+        if not out:
+            return None
+        self.n -= sum(len(c["keys"]) for c in out)
+        return self._tags(out, device)
+
+    def clear(self) -> None:
+        self.q.clear(); self.n = 0
+
+    def _tags(self, chunks: list[dict], device=None) -> Tags:
+        idx = torch.cat([c["idx"] for c in chunks]).long(); vals = torch.cat([c["vals"] for c in chunks])
+        k = torch.zeros(len(idx), self.n_kc).scatter_(1, idx, vals)
+        dev = device or "cpu"
+        return Tags(keys=[key for c in chunks for key in c["keys"]], ts=np.concatenate([c["ts"] for c in chunks]),
+                    y_dn=torch.cat([c["y_dn"] for c in chunks]).to(dev), u0=torch.cat([c["u0"] for c in chunks]).to(dev), k=k.to(dev),
+                    w=torch.cat([c["w"] for c in chunks], dim=1).to(dev), s=np.concatenate([c["s"] for c in chunks]))
+
+
+def assign_channels(comp: np.ndarray, km_mass: np.ndarray, c_sign: np.ndarray, n_strategies: int) -> tuple[np.ndarray, np.ndarray]:
+    """Each strategy's own dopamine channel: its mushroom-body compartments. ``comp`` [J]: each MBON's compartment (−1: no
+    DAN), ``km_mass`` [J]: KC→MBON synapse mass per MBON, ``c_sign`` [J]: readout sign (+1 approach, −1 avoid, 0 other).
+    Compartments are split by their MBONs' majority valence and, within each valence, dealt in descending synapse mass to
+    the strategy with the least mass so far (deterministic, balanced, every strategy gets both valences when there are
+    enough). Returns ``learn`` [S, J] (the strategy's own MBONs: its outcomes change only their KC synapses) and ``read``
+    [S, J] (its own + the MBONs no compartment claims, which stay fixed). One strategy: everything, as before; no DAN
+    data: every MBON is its own compartment."""
+    J = len(c_sign); S = int(n_strategies)
+    if S <= 1:
+        return np.ones((1, J), bool), np.ones((1, J), bool)
+    comp = np.asarray(comp, int) if comp is not None and len(comp) == J else np.full(J, -1)
+    if (comp < 0).all():
+        comp = np.arange(J)
+    learn = np.zeros((S, J), bool)
+    ids = sorted(set(comp[comp >= 0].tolist()))
+    val = {k: int(np.sign(np.sum(np.sign(c_sign[comp == k])))) for k in ids}
+    mass = {k: float(np.asarray(km_mass)[comp == k].sum()) for k in ids}
+    for v in (1, -1, 0):
+        load = np.zeros(S)
+        for k in sorted([k for k in ids if val[k] == v], key=lambda k: (-mass[k], k)):
+            s_ = int(np.argmin(load)); learn[s_, comp == k] = True; load[s_] += mass[k]
+    free = ~learn.any(0)
+    return learn, learn | free[None, :]
+
+
+def row_weights(scores: torch.Tensor, lines: torch.Tensor) -> torch.Tensor:
+    """[C, B]: 1 where a configuration scored the row at or above its own line, else 0 — it learns from the outcome of
+    its own decisions and nothing else.
+
+    Until 2026-09-21 every candidate row was tagged, weighting the ones above the line 10x. Candidates are overwhelmingly
+    losers (4.58M of them averaged -5.70% over the 2026-07-27 replay while the rows actually traded averaged +6.30%), so
+    even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction
+    toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row
+    below the line is not a decision, so it carries no weight."""
+    return torch.where(scores >= lines[:, None], 1.0, 0.0)
+
+
+class PlasticBank:
+    """Plastic changes ``D`` [C, n_KC, n_MBON] of a FlyNet's KC→MBON weights under ``C`` configurations (α, τ½ in days;
+    τ½ = inf: no decay). The net is read, never modified."""
+
+    def __init__(self, net, configs: list[tuple[float, float]], scale: float, nu: float = 1.0, beta_pot: float = 1.0, step_cap: float = STEP_CAP,
+                 learn: np.ndarray | None = None, read: np.ndarray | None = None):
+        self.net, self.scale, self.nu, self.beta_pot, self.step_cap = net, float(scale), float(nu), float(beta_pot), float(step_cap)
+        dev = net.dev
+        with torch.no_grad():
+            self.w0 = net.w_km().detach().clone()
+            self.mask = net.mask.float()
+            self.c = net.c.detach().clone(); self.gamma = net.gain[net.mb0:net.mb1].detach().clone()
+            mu, sd = net.mbon_stats(); self.mu, self.sigma = mu.detach().clone(), sd.detach().clone()
+        self.w0_norm = float(torch.linalg.norm(self.w0))
+        self.configs = [(float(a), float(h)) for a, h in configs]
+        C = len(self.configs)
+        J = self.w0.shape[1]
+        self.learn = torch.as_tensor(np.ones((1, J), bool) if learn is None else learn, device=dev)          # [S, J] per-strategy channels
+        self.read = torch.as_tensor(np.ones((1, J), bool) if read is None else read, device=dev)
+        self.alpha = torch.tensor([a for a, _ in self.configs], device=dev)[:, None].repeat(1, J)            # [C, J]: per-channel rates
+        self.half_life_s = torch.tensor([h * DAY_S if math.isfinite(h) else float("inf") for _, h in self.configs], device=dev)[:, None].repeat(1, J)
+        self.D = torch.zeros(C, *self.w0.shape, device=dev)
+        self.t_last: float | None = None
+        self.dev = dev
+
+    @property
+    def C(self) -> int:
+        return len(self.configs)
+
+    # ---- prediction ----
+    def _read(self, s, B: int) -> torch.Tensor:
+        """[B, J] the MBONs each row's strategy reads (all MBONs with one strategy)."""
+        if self.read.shape[0] == 1 or s is None:
+            return self.read[0].float()[None].expand(B, -1)
+        return self.read[torch.as_tensor(np.asarray(s), device=self.dev)].float()
+
+    def predict(self, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor, D: torch.Tensor | None = None, s=None) -> tuple[torch.Tensor, torch.Tensor]:
+        """(scores [C, B] in net-return units, MBON activity [C, B, n_MBON]) under each configuration's ``D``; ``s``: each
+        row's strategy (its head output is ``y_dn``; it reads its own compartments and the unclaimed ones)."""
+        D = self.D if D is None else D
+        u = u0[None] + self.gamma * torch.einsum("bi,cij->cbj", k, D * self.mask)
+        m = torch.tanh(u)
+        return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m
+
+    def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor:
+        """The shadow: the bootstrap fly's score (D = 0) [B]."""
+        return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale
+
+    def grad_factor(self, m: torch.Tensor, s=None) -> torch.Tensor:
+        """g [C, B, n_MBON] with ∂ŷ/∂D_ij = k_i g_j (zero on MBONs the row's strategy does not read)."""
+        return self.c * self.gamma * (1.0 - m * m) / (self.sigma * self.scale) * self._read(s, m.shape[1])
+
+    # ---- learning ----
+    def decay_to(self, t: float) -> None:
+        """Decay toward the bootstrap weights up to event time ``t`` (seconds)."""
+        if self.t_last is not None and t > self.t_last:
+            f = torch.pow(2.0, -(t - self.t_last) / self.half_life_s)            # [C, J]; inf half-life → 1
+            self.D *= f[:, None, :]
+        self.t_last = t if self.t_last is None else max(self.t_last, t)
+
+    def estimate_nu(self, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor) -> float:
+        """Median squared norm of one row's update direction at D = 0 (sets the scale ``α`` is measured in)."""
+        m = torch.tanh(u0)
+        g = self.c * self.gamma * (1.0 - m * m) / (self.sigma * self.scale)          # [B, J]
+        sq = ((k * k) @ self.mask * g * g).sum(1)
+        v = float(sq.median()) if len(sq) else 0.0
+        self.nu = v if v > 0 else 1.0
+        return self.nu
+
+    def update(self, tags: Tags, r: torch.Tensor, t: float) -> dict:
+        """Capture ``tags`` with realized net returns ``r`` [B] at event time ``t``; returns per-configuration statistics."""
+        self.decay_to(t)
+        if len(tags) == 0:
+            return {"n": 0}
+        r = r.to(self.dev).float().clamp(-RET_CLIP, RET_CLIP)
+        yhat, m = self.predict(tags.y_dn, tags.u0, tags.k, s=tags.s)
+        delta = (r[None] - yhat).clamp(-DELTA_CLIP, DELTA_CLIP)                     # [C, B]
+        own = self.learn.float()[torch.as_tensor(tags.s, device=self.dev)] if self.learn.shape[0] > 1 else self.learn[0].float()[None]
+        g = self.grad_factor(m, tags.s) * own                                       # [C, B, J]: only the row's own channel learns
+        dD = torch.einsum("bi,cbj->cij", tags.k, (tags.w * delta)[:, :, None] * g) * self.mask
+        if self.beta_pot != 1.0:
+            dD = torch.where(dD > 0, dD * self.beta_pot, dD)
+        dD = dD * (self.alpha / self.nu)[:, None, :]
+        norms = torch.linalg.norm(dD.flatten(1), dim=1)
+        cap = self.step_cap * self.w0_norm
+        capped = norms > cap
+        dD = dD * torch.where(capped, cap / norms.clamp(min=1e-30), torch.ones_like(norms))[:, None, None]
+        D = self.D + dD
+        self.D = torch.maximum(torch.minimum(D, 2.0 * self.w0), -self.w0) * self.mask
+        return {"n": len(tags), "mean_delta": delta.mean(1).tolist(), "mean_abs_delta": delta.abs().mean(1).tolist(),
+                "step": torch.linalg.norm(dD.flatten(1), dim=1).tolist(), "capped": capped.tolist(), "drift": self.drift().tolist()}
+
+    def drift(self, s: int | None = None) -> torch.Tensor:
+        """‖D‖_F / ‖w0‖_F per configuration (``s``: over that strategy's own channel only)."""
+        if s is None:
+            return torch.linalg.norm(self.D.flatten(1), dim=1) / max(self.w0_norm, 1e-30)
+        cols = self.learn[s]
+        return torch.linalg.norm(self.D[:, :, cols].flatten(1), dim=1) / max(float(torch.linalg.norm(self.w0[:, cols])), 1e-30)
+
+    # ---- state ----
+    def state(self) -> dict:
+        m = self.mask.bool()
+        return {"D": self.D[:, m].cpu(), "t_last": self.t_last, "configs": self.configs, "nu": self.nu, "beta_pot": self.beta_pot, "step_cap": self.step_cap,
+                "alpha": self.alpha.cpu(), "half_life_s": self.half_life_s.cpu(), "learn": self.learn.cpu(), "read": self.read.cpu()}
+
+    def load_state(self, s: dict) -> None:
+        if [tuple(c) for c in s["configs"]] != self.configs:
+            raise ValueError(f"plastic state for configurations {s['configs']} does not match {self.configs}")
+        D = torch.zeros_like(self.D); D[:, self.mask.bool()] = s["D"].to(self.dev).float()
+        self.D, self.t_last, self.nu = D, s["t_last"], float(s["nu"])
+        if "alpha" in s:
+            self.alpha, self.half_life_s = s["alpha"].to(self.dev), s["half_life_s"].to(self.dev)
+            self.learn, self.read = s["learn"].to(self.dev), s["read"].to(self.dev)
+        self.beta_pot, self.step_cap = float(s.get("beta_pot", self.beta_pot)), float(s.get("step_cap", self.step_cap))
+
+    def select(self, i: int) -> "PlasticBank":
+        """A single-configuration bank holding configuration ``i``'s current ``D`` (the replay's winner, going live)."""
+        b = PlasticBank(self.net, [self.configs[i]], self.scale, self.nu, self.beta_pot, self.step_cap, self.learn.cpu().numpy(), self.read.cpu().numpy())
+        b.D = self.D[i:i + 1].clone(); b.t_last = self.t_last
+        return b
+
+    def select_per_strategy(self, best: list[int]) -> "PlasticBank":
+        """One bank with each strategy's own winning configuration: strategy s's channel columns (D, α, half-life) come from
+        configuration ``best[s]``. Exact, because a strategy's value depends only on its own channel."""
+        b = self.select(best[0])
+        for s_, i in enumerate(best):
+            cols = self.learn[s_]
+            b.D[0][:, cols] = self.D[i][:, cols]; b.alpha[0, cols] = self.alpha[i, cols]; b.half_life_s[0, cols] = self.half_life_s[i, cols]
+        b.configs = [("per-strategy", tuple(best))]
+        return b

fly_trader/brain/populations.py

@@ -0,0 +1,611 @@
+"""Population index sets for the FlyWire FAFB v783 connectome.
+
+The neuron table is flybrain's ``neurons.csv.gz`` (139,255 canonical root_ids, the same set and
+order the connectome edges refer to) left-joined on ``root_id`` with the Schlegel et al. 2024
+annotation TSV (``Supplemental_file1_neuron_annotations.tsv``: cell_class / cell_sub_class /
+cell_type / top_nt). Neurotransmitter = ``top_nt`` when annotated, else flybrain's ``nt_type``
+mapped to the same vocabulary.
+
+Populations are *contiguous index ranges*: neurons are sorted by (population_id, root_id) so a
+population is a slice, never a gather, on the GPU. Membership is first-match in
+``POPULATION_ORDER``; everything unmatched is ``OTHER``.
+
+Valence tables (config/mbon_valence.json): Aso et al. 2014 eLife 3:e04580 (aversion-driving MBONs
+are glutamatergic, attraction-driving MBONs GABAergic/cholinergic) and 3:e04577 (PAM = reward,
+PPL1 = punishment). KC sparse code: Dasgupta, Stevens, Navlakha 2017 Science.
+
+flybrain's 63 behavioural groups (``determine_group`` from snedea/flybrain
+``scripts/build_connectome.py``) are recomputed verbatim so group rates stay comparable with the
+JavaScript demo; the DRIVE_HUNGER group (46 pars-intercerebralis neurons) is our HUNGER population.
+
+Fallback (``config.CELL_TYPES_SOURCE == 'flybrain'`` or the TSV is absent): flybrain's
+``classification.csv.gz`` supplies the classes; MBON valence is assigned by neurotransmitter
+(glutamate -> avoid, acetylcholine/gaba -> approach) and DAN valence by synapse counts onto avoid
+vs approach MBONs (reward if more onto avoid MBONs: reward DANs depress avoidance synapses);
+glomeruli come from the Codex ``consolidated_cell_types.csv.gz`` if it types >= 30 ORN
+glomeruli, else a seeded random partition into 50 pseudo-glomeruli.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from .. import config
+
+RAW_DIR = config.BRAIN_DIR / "raw"
+NEURONS_CSV = RAW_DIR / "neurons.csv.gz"
+CLASSIFICATION_CSV = RAW_DIR / "classification.csv.gz"
+ANNOTATIONS_TSV = RAW_DIR / "Supplemental_file1_neuron_annotations.tsv"
+CONSOLIDATED_TYPES_CSV = RAW_DIR / "consolidated_cell_types.csv.gz"
+MBON_VALENCE_JSON = config.REPO_ROOT / "config" / "mbon_valence.json"
+
+EXPECTED_N = 139_255
+EXPECTED_KC = 5_177
+EXPECTED_MBON = 96
+
+# Local constants (not in config.py): pseudo-glomerulus fallback.
+MIN_TYPED_GLOMERULI = 30
+PSEUDO_GLOMERULI = 50
+
+# First match wins; the remainder is OTHER.
+POPULATION_ORDER: list[str] = [
+    "KC", "MBON_APP", "MBON_AV", "MBON_OTHER",
+    "DAN_PAM", "DAN_PPL1", "DAN_OTHER",
+    "ORN_DANGER", "ORN_FOOD", "ALPN", "ALLN", "LH",
+    "GRN_SWEET", "GRN_BITTER", "GRN_OTHER",
+    "MECH_JO", "MECH_BRISTLE", "MECH_OTHER",
+    "THERMO_WARM", "THERMO_COOL", "THERMO_OTHER",
+    "CX", "HUNGER", "DESCENDING", "VISUAL", "OTHER",
+]
+
+# kind for the ``populations`` table: sensory | central | mb | motor | other
+POPULATION_KIND: dict[str, str] = {
+    "KC": "mb", "MBON_APP": "mb", "MBON_AV": "mb", "MBON_OTHER": "mb",
+    "DAN_PAM": "mb", "DAN_PPL1": "mb", "DAN_OTHER": "mb",
+    "ORN_DANGER": "sensory", "ORN_FOOD": "sensory", "ALPN": "central", "ALLN": "central", "LH": "central",
+    "GRN_SWEET": "sensory", "GRN_BITTER": "sensory", "GRN_OTHER": "sensory",
+    "MECH_JO": "sensory", "MECH_BRISTLE": "sensory", "MECH_OTHER": "sensory",
+    "THERMO_WARM": "sensory", "THERMO_COOL": "sensory", "THERMO_OTHER": "sensory",
+    "CX": "central", "HUNGER": "central", "DESCENDING": "motor", "VISUAL": "sensory", "OTHER": "other",
+}
+
+NT_FLYBRAIN_TO_NAME: dict[str, str] = {
+    "ACH": "acetylcholine", "GABA": "gaba", "GLUT": "glutamate", "DA": "dopamine",
+    "SER": "serotonin", "OA": "octopamine", "OCT": "octopamine",
+}
+
+# ---------------------------------------------------------------------------------------------
+# flybrain groups, ported verbatim from snedea/flybrain scripts/build_connectome.py
+# ---------------------------------------------------------------------------------------------
+FLYBRAIN_GROUPS: list[tuple[str, str]] = [
+    ("VIS_R1R6", "sensory"), ("VIS_R7R8", "sensory"), ("VIS_ME", "sensory"), ("VIS_LO", "sensory"),
+    ("VIS_LC", "sensory"), ("VIS_LPTC", "sensory"), ("OLF_ORN_FOOD", "sensory"), ("OLF_ORN_DANGER", "sensory"),
+    ("OLF_LN", "sensory"), ("OLF_PN", "sensory"), ("MECH_BRISTLE", "sensory"), ("MECH_JO", "sensory"),
+    ("MECH_CHORD", "sensory"), ("ANTENNAL_MECH", "sensory"), ("THERMO_WARM", "sensory"), ("THERMO_COOL", "sensory"),
+    ("NOCI", "sensory"),
+    ("MB_KC", "central"), ("MB_APL", "central"), ("MB_MBON_APP", "central"), ("MB_MBON_AV", "central"),
+    ("MB_DAN_REW", "central"), ("MB_DAN_PUN", "central"), ("LH_APP", "central"), ("LH_AV", "central"),
+    ("CX_EPG", "central"), ("CX_PFN", "central"), ("CX_FC", "central"), ("CX_HDELTA", "central"),
+    ("SEZ_FEED", "central"), ("SEZ_GROOM", "central"), ("SEZ_WATER", "central"), ("GUS_GRN_SWEET", "central"),
+    ("GUS_GRN_BITTER", "central"), ("GUS_GRN_WATER", "central"), ("GNG_DESC", "central"), ("CLOCK_DN", "central"),
+    ("DRIVE_HUNGER", "drives"), ("DRIVE_FEAR", "drives"), ("DRIVE_FATIGUE", "drives"), ("DRIVE_CURIOSITY", "drives"),
+    ("DRIVE_GROOM", "drives"),
+    ("DN_WALK", "motor"), ("DN_FLIGHT", "motor"), ("DN_TURN", "motor"), ("DN_BACKUP", "motor"),
+    ("DN_STARTLE", "motor"), ("VNC_CPG", "motor"), ("MN_LEG_L1", "motor"), ("MN_LEG_R1", "motor"),
+    ("MN_LEG_L2", "motor"), ("MN_LEG_R2", "motor"), ("MN_LEG_L3", "motor"), ("MN_LEG_R3", "motor"),
+    ("MN_WING_L", "motor"), ("MN_WING_R", "motor"), ("MN_PROBOSCIS", "motor"), ("MN_HEAD", "motor"),
+    ("MN_ABDOMEN", "motor"),
+    ("GENERIC_SENSORY", "sensory"), ("GENERIC_CENTRAL", "central"), ("GENERIC_DRIVES", "drives"),
+    ("GENERIC_MOTOR", "motor"),
+]
+FLYBRAIN_GROUP_NAMES: list[str] = [name for name, _ in FLYBRAIN_GROUPS]
+FLYBRAIN_GROUP_ID: dict[str, int] = {name: i for i, name in enumerate(FLYBRAIN_GROUP_NAMES)}
+assert len(FLYBRAIN_GROUPS) == 63
+
+
+def determine_group(flow: str, super_class: str, cls: str, sub_class: str, region: str, side: str = "") -> str:
+    """Map classification fields to one of the 63 group names (59 named + 4 generic).
+
+    Verbatim port of flybrain ``scripts/build_connectome.py::determine_group`` (inputs are the
+    lower-cased flow / super_class / class / sub_class / side columns of classification.csv).
+    """
+    # --- Visual system ---
+    if "visual" in cls or "optic" in cls or "visual" in super_class or "optic" in super_class:
+        if "photo_receptor" in sub_class or "photo" in sub_class:
+            if "r7" in sub_class or "r8" in sub_class or "uv" in sub_class or "pale" in sub_class:
+                return "VIS_R7R8"
+            return "VIS_R1R6"
+        if "ocellar" in sub_class or "ocellar" in cls:
+            return "VIS_R1R6"
+        if "lptc" in sub_class or "tangential" in sub_class:
+            return "VIS_LPTC"
+        if "lc" in sub_class or "loom" in sub_class or "lobula_columnar" in sub_class:
+            return "VIS_LC"
+        if "lobula" in sub_class and "plate" in sub_class:
+            return "VIS_LPTC"
+        if "lobula" in sub_class or sub_class == "lo":
+            return "VIS_LO"
+        if "medulla" in sub_class or "tm" in sub_class or "mi" in sub_class:
+            return "VIS_ME"
+        return "VIS_ME"
+
+    # --- Olfactory system ---
+    if "olfact" in cls:
+        if "pheromone" in sub_class or "avers" in sub_class or "danger" in sub_class:
+            return "OLF_ORN_DANGER"
+        return "OLF_ORN_FOOD"
+    if cls == "alpn":
+        return "OLF_PN"
+    if cls == "alln":
+        return "OLF_LN"
+
+    # --- Gustatory system ---
+    if "gustat" in cls:
+        if "bitter" in sub_class:
+            return "GUS_GRN_BITTER"
+        if "water" in sub_class:
+            return "GUS_GRN_WATER"
+        return "GUS_GRN_SWEET"
+
+    # --- Mechanosensory system ---
+    if "mechano" in cls:
+        if "wind" in sub_class or "gravity" in sub_class or "auditory" in sub_class:
+            return "MECH_JO"
+        if "groom" in sub_class:
+            return "MECH_BRISTLE"
+        if "bristle" in sub_class or "taste_peg" in sub_class:
+            return "MECH_BRISTLE"
+        if "chord" in sub_class or "propriocep" in sub_class:
+            return "MECH_CHORD"
+        if "antenna" in sub_class:
+            return "ANTENNAL_MECH"
+        return "MECH_BRISTLE"
+
+    # --- Thermosensory ---
+    if "thermo" in cls:
+        if "cool" in sub_class or "cold" in sub_class:
+            return "THERMO_COOL"
+        return "THERMO_WARM"
+
+    # --- Hygrosensory -> thermosensory ---
+    if "hygro" in cls:
+        if "dry" in sub_class:
+            return "THERMO_WARM"
+        return "THERMO_COOL"
+
+    # --- Nociceptive ---
+    if "nocicep" in cls:
+        return "NOCI"
+
+    # --- Unknown sensory -> generic sensory ---
+    if "unknown_sensory" in cls:
+        return "MECH_BRISTLE"
+
+    # --- Mushroom body: Kenyon cells ---
+    if "kenyon" in cls or cls == "kc":
+        return "MB_KC"
+
+    # --- Mushroom body output neurons (MBONs) ---
+    if cls == "mbon":
+        return "MB_MBON_APP"
+
+    # --- Mushroom body input neurons (MBINs) ---
+    if cls == "mbin":
+        return "MB_DAN_REW"
+
+    # --- Dopaminergic neurons (DANs) ---
+    if cls == "dan":
+        return "MB_DAN_REW"
+
+    # --- Lateral horn ---
+    if cls == "lhln" or cls == "lhcent":
+        return "LH_APP"
+
+    # --- Central complex ---
+    if cls == "cx":
+        if "ring" in sub_class:
+            return "CX_EPG"
+        if "tangential" in sub_class:
+            return "CX_HDELTA"
+        if "columnar" in sub_class:
+            return "CX_PFN"
+        return "CX_FC"
+
+    # --- Antennal lobe interneurons ---
+    if cls == "alin":
+        return "OLF_LN"
+    if cls == "alon":
+        return "OLF_PN"
+
+    # --- TuBu neurons (tubercle to bulb, visual -> CX pathway) ---
+    if cls == "tubu":
+        return "CX_EPG"
+
+    # --- TPN (taste projection neurons) ---
+    if cls == "tpn":
+        if "water" in sub_class:
+            return "GUS_GRN_WATER"
+        return "GUS_GRN_SWEET"
+
+    # --- MAL (medial accessory lobe) neurons ---
+    if cls == "mal":
+        return "CX_FC"
+
+    # --- Ascending neurons ---
+    if "ascending" in super_class or cls == "an":
+        return "GNG_DESC"
+
+    # --- Descending neurons ---
+    if "descend" in super_class or "descend" in cls:
+        if "dn1p" in sub_class:
+            return "DN_STARTLE"
+        if "dn3" in sub_class:
+            return "DN_WALK"
+        if "walk" in sub_class or "locomot" in sub_class:
+            return "DN_WALK"
+        if "flight" in sub_class:
+            return "DN_FLIGHT"
+        if "turn" in sub_class:
+            return "DN_TURN"
+        if "back" in sub_class:
+            return "DN_BACKUP"
+        if "startle" in sub_class or "escape" in sub_class or "giant" in sub_class:
+            return "DN_STARTLE"
+        return "GNG_DESC"
+
+    # --- Motor neurons ---
+    if "motor" in cls or "motor" in super_class:
+        if "proboscis" in sub_class:
+            return "MN_PROBOSCIS"
+        if "neck" in sub_class or "head" in sub_class:
+            return "MN_HEAD"
+        if "abdom" in sub_class or "crop" in sub_class:
+            return "MN_ABDOMEN"
+        if "ingestion" in sub_class or "haustellum" in sub_class or "salivary" in sub_class:
+            return "SEZ_FEED"
+        if "eye" in sub_class or "antenna" in sub_class:
+            return "MN_HEAD"
+        return "VNC_CPG"
+
+    # --- Clock neurons ---
+    if "clock" in cls or "circadian" in cls:
+        return "CLOCK_DN"
+
+    # --- Endocrine / neurosecretory ---
+    if "endocrine" in super_class:
+        if "pars_lateralis" in cls:
+            return "DRIVE_FATIGUE"
+        return "DRIVE_HUNGER"
+
+    # --- Fallback: use flow to pick a region-appropriate generic ---
+    if flow == "afferent":
+        return "GENERIC_SENSORY"
+    if flow == "efferent":
+        return "GENERIC_MOTOR"
+    return "GENERIC_CENTRAL"
+
+
+# ---------------------------------------------------------------------------------------------
+# Loading
+# ---------------------------------------------------------------------------------------------
+def load_neuron_table(annotations: str | Path | None = None, source: str | None = None) -> tuple[pd.DataFrame, str]:
+    """flybrain neurons (canonical order) + classification + annotations (left join).
+
+    Returns (table, source) where source is 'annotations' or 'flybrain'. Columns:
+    root_id (int64), fb_nt, fb_flow, fb_super_class, fb_class, fb_sub_class, fb_side (lower-cased,
+    as flybrain sees them), flybrain_group (uint16), annotated (bool), super_class, cell_class,
+    cell_sub_class, cell_type, top_nt, nt.
+    """
+    source = source or config.CELL_TYPES_SOURCE
+    neu = pd.read_csv(NEURONS_CSV, usecols=["root_id", "nt_type"], dtype={"root_id": np.int64, "nt_type": str},
+                      keep_default_na=False)
+    if len(neu) != EXPECTED_N:
+        print(f"[populations] WARNING neurons.csv has {len(neu)} rows, expected {EXPECTED_N}", file=sys.stderr)
+    cls = pd.read_csv(CLASSIFICATION_CSV, dtype=str, keep_default_na=False)
+    cls["root_id"] = cls["root_id"].astype(np.int64)
+    for c in ("flow", "super_class", "class", "sub_class", "side"):
+        cls[c] = cls[c].str.strip().str.lower()
+    cls = cls.rename(columns={"flow": "fb_flow", "super_class": "fb_super_class", "class": "fb_class",
+                              "sub_class": "fb_sub_class", "side": "fb_side"})
+    cls = cls[["root_id", "fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"]]
+    df = neu.rename(columns={"nt_type": "fb_nt"}).merge(cls, on="root_id", how="left")
+    for c in ("fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"):
+        df[c] = df[c].fillna("")
+    df["fb_nt"] = df["fb_nt"].str.strip().str.upper()
+
+    # flybrain group labels (classification.csv rows; neurons without a row keep GENERIC_CENTRAL)
+    groups = np.full(len(df), FLYBRAIN_GROUP_ID["GENERIC_CENTRAL"], dtype=np.uint16)
+    keyed = df[["fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"]].drop_duplicates()
+    cache: dict[tuple, int] = {}
+    for row in keyed.itertuples(index=False):
+        cache[tuple(row)] = FLYBRAIN_GROUP_ID[determine_group(row[0], row[1], row[2], row[3], "central", row[4])]
+    keys = list(zip(df["fb_flow"], df["fb_super_class"], df["fb_class"], df["fb_sub_class"], df["fb_side"]))
+    has_row = df["root_id"].isin(cls["root_id"]).to_numpy()
+    groups[has_row] = np.array([cache[k] for k, h in zip(keys, has_row) if h], dtype=np.uint16)
+    df["flybrain_group"] = groups
+
+    ann_path = Path(annotations) if annotations else ANNOTATIONS_TSV
+    use_ann = source == "annotations" and ann_path.exists()
+    if source == "annotations" and not ann_path.exists():
+        print(f"[populations] WARNING annotations TSV missing at {ann_path}; falling back to flybrain "
+              f"classification", file=sys.stderr)
+    if use_ann:
+        ann = pd.read_csv(ann_path, sep="\t", dtype=str, keep_default_na=False, low_memory=False,
+                          usecols=["root_id", "super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"])
+        ann["root_id"] = ann["root_id"].astype(np.int64)
+        ann = ann.drop_duplicates("root_id")
+        ann["annotated"] = True
+        df = df.merge(ann, on="root_id", how="left")
+        df["annotated"] = df["annotated"].fillna(False).astype(bool)
+        for c in ("super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"):
+            df[c] = df[c].fillna("").str.strip()
+        src = "annotations"
+    else:
+        df["annotated"] = False
+        for c in ("super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"):
+            df[c] = ""
+        src = "flybrain"
+    fb_nt_named = df["fb_nt"].map(NT_FLYBRAIN_TO_NAME).fillna("")
+    df["nt"] = np.where(df["top_nt"] != "", df["top_nt"].str.lower(), fb_nt_named)
+    return df, src
+
+
+def load_mbon_valence(path: str | Path | None = None) -> dict:
+    with open(path or MBON_VALENCE_JSON) as f:
+        return json.load(f)
+
+
+def mbon_base_type(cell_type: str) -> str:
+    """Exact type only: '-like' annotations and combined labels ('MBON25,MBON34') stay unassignable
+    (no valence evidence for '-like' cells; literature review 2026-09-12)."""
+    return cell_type.strip()
+
+
+# ---------------------------------------------------------------------------------------------
+# Populations
+# ---------------------------------------------------------------------------------------------
+@dataclass
+class Populations:
+    order: list[str]
+    ranges: dict[str, tuple[int, int]]
+    root_ids: np.ndarray                 # int64 [N], population order
+    nt: np.ndarray                       # str [N]
+    flybrain_group: np.ndarray           # uint16 [N]
+    flybrain_group_names: list[str]
+    glomerulus_of_orn: np.ndarray        # int32 [n_ORN_FOOD], -1 unknown
+    glomerulus_names: list[str]
+    canonical_index: np.ndarray          # int64 [N]: sorted position i holds flybrain row canonical_index[i]
+    source: str = "annotations"
+    cell_type: np.ndarray = field(default_factory=lambda: np.zeros(0, dtype=object))
+    kind: dict[str, str] = field(default_factory=lambda: dict(POPULATION_KIND))
+
+    @property
+    def N(self) -> int:
+        return int(len(self.root_ids))
+
+    def n(self, name: str) -> int:
+        a, b = self.ranges[name]
+        return b - a
+
+    def index(self, name: str) -> np.ndarray:
+        a, b = self.ranges[name]
+        return np.arange(a, b, dtype=np.int64)
+
+    def slice(self, name: str) -> slice:
+        a, b = self.ranges[name]
+        return slice(a, b)
+
+    def population_id(self) -> np.ndarray:
+        pid = np.empty(self.N, dtype=np.int16)
+        for i, name in enumerate(self.order):
+            a, b = self.ranges[name]
+            pid[a:b] = i
+        return pid
+
+    def counts(self) -> dict[str, int]:
+        return {name: self.n(name) for name in self.order}
+
+    def inverse_index(self) -> np.ndarray:
+        """canonical flybrain row -> sorted position."""
+        inv = np.empty(self.N, dtype=np.int64)
+        inv[self.canonical_index] = np.arange(self.N, dtype=np.int64)
+        return inv
+
+
+def _assign_annotations(df: pd.DataFrame, valence: dict) -> np.ndarray:
+    cc = df["cell_class"].to_numpy(dtype=object)
+    sc = df["cell_sub_class"].to_numpy(dtype=object)
+    ct = df["cell_type"].to_numpy(dtype=object)
+    base = np.array([mbon_base_type(t) for t in ct], dtype=object)
+    sup = np.where(df["annotated"].to_numpy(), df["super_class"].to_numpy(dtype=object),
+                   df["fb_super_class"].to_numpy(dtype=object))
+    approach = set(valence["approach"])
+    avoid = set(valence["avoid"])
+    rew = valence.get("dan_reward_prefix", "PAM")
+    pun = valence.get("dan_punishment_prefix", "PPL1")
+    is_mbon = cc == "MBON"
+    is_dan = cc == "DAN"
+    is_olf = cc == "olfactory"
+    is_gus = cc == "gustatory"
+    is_mech = cc == "mechanosensory"
+    is_thermo = cc == "thermosensory"
+    is_hygro = cc == "hygrosensory"
+    starts = np.char.startswith
+    ct_s = ct.astype(str)
+    rules = {
+        "KC": cc == "Kenyon_Cell",
+        "MBON_APP": is_mbon & np.isin(base, list(approach)),
+        "MBON_AV": is_mbon & np.isin(base, list(avoid)),
+        "MBON_OTHER": is_mbon,
+        "DAN_PAM": is_dan & starts(ct_s, rew),
+        "DAN_PPL1": is_dan & starts(ct_s, pun),
+        "DAN_OTHER": is_dan,
+        "ORN_DANGER": is_olf & (sc == "pheromone"),
+        "ORN_FOOD": is_olf,
+        "ALPN": cc == "ALPN",
+        "ALLN": cc == "ALLN",
+        "LH": np.isin(cc, ["LHLN", "LHCENT"]),
+        "GRN_SWEET": is_gus & (sc == "sugar/water"),
+        "GRN_BITTER": is_gus & (sc == "bitter"),
+        "GRN_OTHER": is_gus,
+        "MECH_JO": is_mech & np.isin(sc, ["wind_gravity", "auditory"]),
+        "MECH_BRISTLE": is_mech & np.isin(sc, ["eye bristle", "head bristle"]),
+        "MECH_OTHER": is_mech,
+        "THERMO_WARM": (is_thermo & (sc == "heating")) | (is_hygro & (sc == "dry")),
+        "THERMO_COOL": (is_thermo & (sc == "cold")) | (is_hygro & np.isin(sc, ["cooling", "moist", "evaporative_cooling"])),
+        "THERMO_OTHER": is_thermo | is_hygro,
+        "CX": cc == "CX",
+        "HUNGER": df["flybrain_group"].to_numpy() == FLYBRAIN_GROUP_ID["DRIVE_HUNGER"],
+        "DESCENDING": sup == "descending",
+        "VISUAL": np.isin(sup, ["optic", "visual_projection", "visual_centrifugal"]) | (cc == "visual"),
+    }
+    return _first_match(rules, len(df))
+
+
+def _contains(arr: np.ndarray, needle: str) -> np.ndarray:
+    return np.array([needle in s for s in arr], dtype=bool)
+
+
+def _assign_flybrain(df: pd.DataFrame, dan_valence: np.ndarray | None) -> np.ndarray:
+    """Fallback partition from flybrain classification.csv (lower-cased fields)."""
+    cls = df["fb_class"].to_numpy(dtype=object)
+    sub = df["fb_sub_class"].to_numpy(dtype=object)
+    sup = df["fb_super_class"].to_numpy(dtype=object)
+    nt = df["nt"].to_numpy(dtype=object)
+    is_kc = _contains(cls, "kenyon") | (cls == "kc")
+    is_mbon = cls == "mbon"
+    is_dan = cls == "dan"
+    is_olf = _contains(cls, "olfact")
+    is_gus = _contains(cls, "gustat")
+    is_mech = _contains(cls, "mechano")
+    is_thermo = _contains(cls, "thermo")
+    is_hygro = _contains(cls, "hygro")
+    if dan_valence is None:
+        dan_valence = np.zeros(len(df), dtype=np.int8)
+    rules = {
+        "KC": is_kc,
+        "MBON_APP": is_mbon & np.isin(nt, ["acetylcholine", "gaba"]),
+        "MBON_AV": is_mbon & (nt == "glutamate"),
+        "MBON_OTHER": is_mbon,
+        "DAN_PAM": is_dan & (dan_valence > 0),
+        "DAN_PPL1": is_dan & (dan_valence < 0),
+        "DAN_OTHER": is_dan,
+        "ORN_DANGER": is_olf & (_contains(sub, "pheromone") | _contains(sub, "avers") | _contains(sub, "danger")),
+        "ORN_FOOD": is_olf,
+        "ALPN": cls == "alpn",
+        "ALLN": cls == "alln",
+        "LH": (cls == "lhln") | (cls == "lhcent"),
+        "GRN_SWEET": is_gus & (_contains(sub, "sugar") | _contains(sub, "water")),
+        "GRN_BITTER": is_gus & _contains(sub, "bitter"),
+        "GRN_OTHER": is_gus,
+        "MECH_JO": is_mech & (_contains(sub, "wind") | _contains(sub, "gravity") | _contains(sub, "auditory")),
+        "MECH_BRISTLE": is_mech & _contains(sub, "bristle"),
+        "MECH_OTHER": is_mech,
+        "THERMO_WARM": (is_thermo & (_contains(sub, "heat") | _contains(sub, "warm"))) | (is_hygro & _contains(sub, "dry")),
+        "THERMO_COOL": (is_thermo & (_contains(sub, "cold") | _contains(sub, "cool")))
+                       | (is_hygro & (_contains(sub, "cool") | _contains(sub, "moist") | _contains(sub, "evap"))),
+        "THERMO_OTHER": is_thermo | is_hygro,
+        "CX": cls == "cx",
+        "HUNGER": df["flybrain_group"].to_numpy() == FLYBRAIN_GROUP_ID["DRIVE_HUNGER"],
+        "DESCENDING": _contains(sup, "descend"),
+        "VISUAL": np.isin(sup, ["optic", "visual_projection", "visual_centrifugal"]) | (cls == "visual"),
+    }
+    return _first_match(rules, len(df))
+
+
+def _first_match(rules: dict[str, np.ndarray], n: int) -> np.ndarray:
+    pid = np.full(n, -1, dtype=np.int16)
+    for name in POPULATION_ORDER:
+        if name == "OTHER":
+            continue
+        m = rules[name] & (pid < 0)
+        pid[m] = POPULATION_ORDER.index(name)
+    pid[pid < 0] = POPULATION_ORDER.index("OTHER")
+    return pid
+
+
+def fallback_dan_valence(df: pd.DataFrame, pid_partial: np.ndarray, edges: tuple[np.ndarray, np.ndarray, np.ndarray]) -> np.ndarray:
+    """+1 reward / -1 punishment / 0 unknown per canonical neuron, from synapse counts onto avoid vs
+    approach MBONs (reward DANs innervate the compartments whose MBONs drive avoidance)."""
+    pre, post, syn = edges
+    app = POPULATION_ORDER.index("MBON_APP")
+    av = POPULATION_ORDER.index("MBON_AV")
+    n = len(df)
+    onto_app = np.bincount(pre, weights=syn * (pid_partial[post] == app), minlength=n)
+    onto_av = np.bincount(pre, weights=syn * (pid_partial[post] == av), minlength=n)
+    val = np.zeros(n, dtype=np.int8)
+    val[onto_av > onto_app] = 1
+    val[(onto_app > 0) & (onto_app >= onto_av)] = -1
+    return val
+
+
+def _glomeruli_annotations(cell_types: np.ndarray) -> tuple[np.ndarray, list[str]]:
+    names: list[str] = sorted({t[len("ORN_"):] for t in cell_types if t.startswith("ORN_") and len(t) > 4})
+    lut = {n: i for i, n in enumerate(names)}
+    gid = np.array([lut.get(t[len("ORN_"):], -1) if t.startswith("ORN_") else -1 for t in cell_types], dtype=np.int32)
+    return gid, names
+
+
+def _glomeruli_fallback(root_ids: np.ndarray, seed: int) -> tuple[np.ndarray, list[str]]:
+    if CONSOLIDATED_TYPES_CSV.exists():
+        ct = pd.read_csv(CONSOLIDATED_TYPES_CSV, dtype=str, keep_default_na=False, usecols=["root_id", "primary_type"])
+        ct["root_id"] = ct["root_id"].astype(np.int64)
+        lut = dict(zip(ct["root_id"].to_numpy(), ct["primary_type"].to_numpy()))
+        types = np.array([lut.get(r, "") for r in root_ids], dtype=object)
+        gid, names = _glomeruli_annotations(types)
+        if len(names) >= MIN_TYPED_GLOMERULI:
+            return gid, names
+    rng = np.random.default_rng(seed)
+    gid = rng.integers(0, PSEUDO_GLOMERULI, size=len(root_ids)).astype(np.int32)
+    return gid, [f"pseudo_{i:02d}" for i in range(PSEUDO_GLOMERULI)]
+
+
+def build_populations(annotations: str | Path | None = None, source: str | None = None,
+                      edges: tuple[np.ndarray, np.ndarray, np.ndarray] | None = None,
+                      valence_path: str | Path | None = None, seed: int = 0, verbose: bool = True) -> Populations:
+    """Partition the neuron table into contiguous populations.
+
+    ``edges`` = (pre_canonical, post_canonical, syn_count) arrays; only needed for the flybrain
+    fallback (DAN valence by synapse counts).
+    """
+    df, src = load_neuron_table(annotations, source)
+    valence = load_mbon_valence(valence_path)
+    if src == "annotations":
+        pid = _assign_annotations(df, valence)
+    else:
+        pid = _assign_flybrain(df, None)
+        if edges is not None:
+            pid = _assign_flybrain(df, fallback_dan_valence(df, pid, edges))
+        elif verbose:
+            print("[populations] WARNING flybrain fallback without edges: DANs are all DAN_OTHER", file=sys.stderr)
+    root_ids = df["root_id"].to_numpy(dtype=np.int64)
+    perm = np.lexsort((root_ids, pid))  # by population, then root_id
+    pid_sorted = pid[perm]
+    ranges: dict[str, tuple[int, int]] = {}
+    for i, name in enumerate(POPULATION_ORDER):
+        a = int(np.searchsorted(pid_sorted, i, side="left"))
+        b = int(np.searchsorted(pid_sorted, i, side="right"))
+        ranges[name] = (a, b)
+    cell_type = df["cell_type"].to_numpy(dtype=object)[perm]
+    a, b = ranges["ORN_FOOD"]
+    if src == "annotations":
+        gid, gnames = _glomeruli_annotations(cell_type[a:b])
+    else:
+        gid, gnames = _glomeruli_fallback(root_ids[perm][a:b], seed)
+    pops = Populations(
+        order=list(POPULATION_ORDER), ranges=ranges, root_ids=root_ids[perm],
+        nt=df["nt"].to_numpy(dtype=object)[perm].astype(str), flybrain_group=df["flybrain_group"].to_numpy()[perm],
+        flybrain_group_names=list(FLYBRAIN_GROUP_NAMES), glomerulus_of_orn=gid, glomerulus_names=gnames,
+        canonical_index=perm.astype(np.int64), source=src, cell_type=cell_type,
+    )
+    n_kc = pops.n("KC")
+    n_mbon = pops.n("MBON_APP") + pops.n("MBON_AV") + pops.n("MBON_OTHER")
+    if n_kc != EXPECTED_KC or n_mbon != EXPECTED_MBON:
+        raise RuntimeError(f"population sanity check failed: KC={n_kc} (expected {EXPECTED_KC}), "
+                           f"MBON={n_mbon} (expected {EXPECTED_MBON}); source={src}")
+    return pops

fly_trader/chain/__init__.py

@@ -0,0 +1 @@
+

fly_trader/chain/balances.py

@@ -0,0 +1,49 @@
+"""Wallet balance snapshots and deltas (pattern: flywheel utils/balance_tracker.py:41-80).
+
+A fill is verified ONLY by comparing a snapshot taken before the order with one taken after
+confirmation (plan §9). SOL is tracked as lamports of the wallet itself; SPL balances are raw
+integer amounts summed per mint across all accounts under both token programs (Jupiter wraps and
+unwraps WSOL inside the transaction, so a SOL leg shows up as a lamport delta, not a WSOL delta).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from .. import config
+
+
+@dataclass
+class Snapshot:
+    lamports: int
+    tokens: dict[str, int]
+    ts: datetime
+    decimals: dict[str, int] = field(default_factory=dict)
+
+    def token(self, mint: str) -> int:
+        return int(self.tokens.get(mint, 0))
+
+    def to_json(self) -> dict:
+        return {"lamports": int(self.lamports), "tokens": {m: int(a) for m, a in self.tokens.items()},
+                "decimals": {m: int(d) for m, d in self.decimals.items()}, "ts": self.ts.isoformat()}
+
+
+def snapshot_balances(rpc, pubkey) -> Snapshot:
+    lamports = int(rpc.get_balance(str(pubkey)))
+    tokens: dict[str, int] = {}
+    decimals: dict[str, int] = {}
+    for acct in rpc.get_token_accounts_by_owner(str(pubkey)):
+        mint = acct["mint"]
+        tokens[mint] = tokens.get(mint, 0) + int(acct["amount"])
+        if acct.get("decimals") is not None:
+            decimals[mint] = int(acct["decimals"])
+    return Snapshot(lamports=lamports, tokens=tokens, ts=config.utcnow(), decimals=decimals)
+
+
+def compute_delta(pre: Snapshot, post: Snapshot) -> dict:
+    """``{"lamports_delta": int, "token_deltas": {mint: post - pre}}`` over every mint seen in either snapshot."""
+    mints = set(pre.tokens) | set(post.tokens)
+    return {
+        "lamports_delta": int(post.lamports) - int(pre.lamports),
+        "token_deltas": {m: int(post.tokens.get(m, 0)) - int(pre.tokens.get(m, 0)) for m in sorted(mints)},
+    }

fly_trader/chain/cluster_guard.py

@@ -0,0 +1,35 @@
+"""Default-deny signing gate (plan §9; pattern from better_bot code/solana_client.py:28-52).
+
+Every path that signs a transaction calls ``assert_signing_allowed()`` first. Reads (balances,
+quotes, signature statuses) never do. Signing is allowed only when ALL of these hold:
+``SOLANA_CLUSTER == 'mainnet-beta'`` (the only cluster where the graduated-memecoin universe
+exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position
+cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a
+test can monkeypatch them.
+"""
+from __future__ import annotations
+
+from .. import config
+
+
+class SigningRefused(RuntimeError):
+    """Raised instead of signing when the live gate is not fully open."""
+
+
+def assert_signing_allowed() -> None:
+    cluster = config.SOLANA_CLUSTER
+    if cluster != "mainnet-beta":
+        raise SigningRefused(f"refusing to sign on cluster {cluster!r}: only mainnet-beta is allowed")
+    if not config.LIVE_ENABLED:
+        raise SigningRefused("refusing to sign: LIVE_ENABLED is not set (default-deny)")
+    missing = config.live_prerequisites_missing()
+    if missing:
+        raise SigningRefused("refusing to sign: live prerequisites missing: " + ", ".join(missing))
+
+
+def signing_allowed() -> bool:
+    try:
+        assert_signing_allowed()
+        return True
+    except SigningRefused:
+        return False

fly_trader/chain/jupiter_swap.py

@@ -0,0 +1,221 @@
+"""Jupiter Swap API v2 client: ``GET /order``, ``POST /execute``, ``GET /program-id-to-label``.
+
+Base ``config.JUPITER_SWAP_BASE`` (https://api.jup.ag/swap/v2), header ``x-api-key``. Token-bucket
+rate limits: one bucket (``config.JUPITER_RPS``) for ``/order`` and a separate one for ``/execute``
+(Jupiter meters ``/execute`` separately). Every call is persisted to ``api_calls``
+(service='jupiter'); the key is never stored (the request jsonb holds query params / a summary).
+
+``GET /order`` verified 2026-09-12 against mainnet (0.01 SOL -> USDC):
+  quote-only (no taker) -> HTTP 200 with fields: swapType ("aggregator"), inAmount, outAmount,
+    otherAmountThreshold, swapMode ("ExactIn"), slippageBps (0 when not requested), priceImpactPct
+    (string), routePlan [{percent, bps, usdValue, swapInfo {ammKey, label, inputMint, outputMint,
+    inAmount, outAmount}}], feeMint, feeBps (2 for SOL/USDC), platformFee {feeBps, feeMint},
+    signatureFeeLamports (0), signatureFeePayer (null), prioritizationFeeLamports (0),
+    prioritizationFeePayer (null), rentFeeLamports, rentFeePayer, transaction (null), gasless,
+    jitOptimized, taker (null), inputMint, outputMint, router ("metis"), guaranteedPrice, requestId,
+    inUsdValue, outUsdValue, swapUsdValue, priceImpact (number, percentage points), mode ("ultra" and
+    "manual" both observed), totalTime. No lastValidBlockHeight / expireAt / errorCode in the
+    quote-only response. feeBps was 2 for SOL/USDC (docs: 10 bps most pairs, 50 bps for tokens < 24 h).
+  with taker -> adds transaction (base64 v0, taker = fee payer, 1 required signature, no
+    pre-filled signatures), lastValidBlockHeight (STRING, e.g. "424496168"), signatureFeePayer ==
+    taker, signatureFeeLamports 5000, prioritizationFeeLamports/Payer, rentFeeLamports/Payer,
+    mode "manual". Documented error fields when the order cannot be built: errorCode
+    (aggregator: 1 insufficient funds, 2 insufficient SOL for gas, 3 below gasless minimum;
+    jupiterz: 1 insufficient balance, 2 missing ATA, 3 quote could not be built) + errorMessage.
+  Rate-limit headers: x-ratelimit-remaining / x-ratelimit-current / x-ratelimit-reset.
+
+``POST /execute`` (per the v2 reference; never exercised here): body {signedTransaction, requestId,
+  lastValidBlockHeight?(string)} -> {status "Success"|"Failed", signature, slot (string), code
+  (0 ok; -1 missing cached order, -2 invalid signed tx, -3 invalid message bytes; -1000 failed to
+  land, -1001 unknown, -1002 invalid tx, -1003 not fully signed, -1004 invalid block height;
+  -2000 RFQ failed to land, -2001 unknown, -2002 invalid payload, -2003 quote expired, -2004 swap
+  rejected), error, totalInputAmount, totalOutputAmount, inputAmountResult, outputAmountResult,
+  swapEvents [{inputMint, inputAmount, outputMint, outputAmount}]}. The same signedTransaction +
+  requestId is idempotent for ~2 minutes.
+
+``GET /program-id-to-label`` verified keyless at https://api.jup.ag/swap/v2/program-id-to-label
+(107 entries; /swap/v1 and lite-api.jup.ag serve the same map). Cached to
+``data/jupiter_program_labels.json``.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+
+import httpx
+
+from .. import config
+from ..db.apilog import record_api_call
+from ..logging_setup import scrub
+from .jupiter_tokens import TokenBucket
+
+log = logging.getLogger(__name__)
+
+JUPITER_EXECUTE_RPS = config.JUPITER_RPS  # separate bucket, same rate
+JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label"
+JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json"
+JUPITER_LABELS_MAX_AGE_S = 86400.0
+DEFAULT_EXCLUDE_ROUTERS = "jupiterz"
+
+
+class JupiterError(RuntimeError):
+    def __init__(self, endpoint: str, status: int | None, detail: str = ""):
+        self.endpoint = endpoint
+        self.status = status
+        self.detail = detail
+        super().__init__(f"jupiter {endpoint}: HTTP {status}: {detail}")
+
+
+def _read_labels_cache(path: Path) -> dict | None:
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, dict) and isinstance(data.get("labels"), dict):
+            return data
+    except (OSError, ValueError):
+        pass
+    return None
+
+
+def fetch_program_labels(cache_path: Path | None = None, refresh: bool = False,
+                         max_age_s: float = JUPITER_LABELS_MAX_AGE_S) -> dict[str, str]:
+    """Keyless ``GET /program-id-to-label`` with a JSON file cache; stale cache on network failure."""
+    path = Path(cache_path) if cache_path else JUPITER_LABELS_CACHE
+    cached = _read_labels_cache(path)
+    if cached and not refresh and time.time() - float(cached.get("fetched_at", 0)) < max_age_s:
+        return cached["labels"]
+    t0 = time.monotonic()
+    status = None
+    err = None
+    nbytes = None
+    try:
+        r = httpx.get(JUPITER_LABELS_URL, timeout=20)
+        status = r.status_code
+        nbytes = len(r.content)
+        body = r.json() if status < 400 else None
+        if status >= 400 or not isinstance(body, dict):
+            raise JupiterError("/program-id-to-label", status, scrub(r.text[:200]))
+        labels = {str(k): str(v) for k, v in body.items()}
+    except (JupiterError, httpx.HTTPError, ValueError) as e:
+        err = f"{type(e).__name__}: {scrub(str(e))}"
+        if cached:
+            log.warning("program-id-to-label fetch failed (%s); using stale cache", type(e).__name__)
+            return cached["labels"]
+        raise
+    finally:
+        record_api_call("jupiter", "swap/program-id-to-label", "GET", status,
+                        int((time.monotonic() - t0) * 1000), err is None, err, response_bytes=nbytes)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump({"fetched_at": time.time(), "url": JUPITER_LABELS_URL, "labels": labels}, f, indent=0)
+        os.replace(tmp, path)
+    except OSError as e:
+        log.warning("could not write %s: %s", path, type(e).__name__)
+    return labels
+
+
+class JupiterSwap:
+    def __init__(self, api_key: str | None = None, rps: float | None = None, timeout_s: float = 30.0):
+        self.api_key = api_key or config.JUPITER_API_KEY
+        if not self.api_key:
+            raise RuntimeError("JUPITER_API_KEY is not set")
+        self.base = config.JUPITER_SWAP_BASE
+        self.order_bucket = TokenBucket(rps or config.JUPITER_RPS)
+        self.execute_bucket = TokenBucket(rps or JUPITER_EXECUTE_RPS)
+        self.client = httpx.Client(timeout=timeout_s, headers={"x-api-key": self.api_key})
+
+    def _request(self, method: str, path: str, *, bucket: TokenBucket, params: dict | None = None,
+                 json_body: dict | None = None, log_request: dict | None = None, accept_error_body,
+                 retries: int = 3) -> tuple[int, object, int]:
+        """One rate-limited request. Returns ``(http_status, parsed_body, latency_ms)``.
+        Raises ``JupiterError`` on transport errors and on HTTP >= 400 unless ``accept_error_body(body)``
+        says the JSON body is a structured Jupiter result the caller wants to see."""
+        bucket.acquire()
+        t0 = time.monotonic()
+        status: int | None = None
+        err: str | None = None
+        nbytes: int | None = None
+        ok = False
+        try:
+            for attempt in range(retries + 1):
+                try:
+                    r = self.client.request(method, self.base + path, params=params, json=json_body)
+                except httpx.HTTPError as e:
+                    raise JupiterError(path, None, f"{type(e).__name__}: {scrub(str(e))}") from None
+                status = r.status_code
+                nbytes = len(r.content)
+                if status == 429 and attempt < retries:
+                    time.sleep(1.0 + attempt)
+                    bucket.acquire()
+                    continue
+                break
+            try:
+                body = r.json()
+            except ValueError:
+                body = None
+            latency = int((time.monotonic() - t0) * 1000)
+            if status >= 400 and not accept_error_body(body):
+                snippet = json.dumps(body)[:300] if body is not None else r.text[:300]
+                raise JupiterError(path, status, scrub(snippet))
+            if isinstance(body, dict):
+                if body.get("errorCode") is not None:
+                    err = f"errorCode {body.get('errorCode')}: {body.get('errorMessage') or body.get('error') or ''}"
+                elif body.get("code") not in (None, 0):
+                    err = f"code {body.get('code')}: {body.get('error') or body.get('status') or ''}"
+            ok = status < 400 and err is None
+            return status, body, latency
+        except JupiterError as e:
+            err = str(e)
+            raise
+        finally:
+            record_api_call("jupiter", f"swap{path}", method, status, int((time.monotonic() - t0) * 1000),
+                            ok, scrub(err) if err else None, request=log_request, response_bytes=nbytes)
+
+    def order(self, input_mint: str, output_mint: str, amount: int, taker: str | None = None,
+              slippage_bps: int | None = None, exclude_routers: str | None = DEFAULT_EXCLUDE_ROUTERS,
+              exclude_dexes: str | None = None) -> dict:
+        """``GET /order``. Without ``taker`` this is a quote (no transaction is built). Returns the
+        parsed JSON plus ``_latency_ms`` and ``_http_status``; a body carrying ``errorCode`` is returned
+        as-is (the caller decides), other HTTP errors raise ``JupiterError``."""
+        params: dict = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(int(amount))}
+        if taker:
+            params["taker"] = str(taker)
+        if slippage_bps is not None:
+            params["slippageBps"] = int(slippage_bps)
+        if exclude_routers:
+            params["excludeRouters"] = exclude_routers
+        if exclude_dexes:
+            params["excludeDexes"] = exclude_dexes
+        status, body, latency = self._request(
+            "GET", "/order", bucket=self.order_bucket, params=params, log_request={"params": params},
+            accept_error_body=lambda b: isinstance(b, dict) and "errorCode" in b)
+        if not isinstance(body, dict):
+            raise JupiterError("/order", status, "non-JSON response")
+        body["_latency_ms"] = latency
+        body["_http_status"] = status
+        return body
+
+    def execute(self, signed_transaction_b64: str, request_id: str, last_valid_block_height: int | None = None) -> dict:
+        """``POST /execute``. Returns the parsed JSON plus ``_latency_ms``/``_http_status``; a body with
+        ``status``/``code`` is returned even on HTTP errors so the broker can see the code."""
+        payload: dict = {"signedTransaction": signed_transaction_b64, "requestId": request_id}
+        if last_valid_block_height is not None:
+            payload["lastValidBlockHeight"] = str(int(last_valid_block_height))
+        summary = {"requestId": request_id, "lastValidBlockHeight": payload.get("lastValidBlockHeight"),
+                   "signedTransactionBytes": len(signed_transaction_b64)}
+        status, body, latency = self._request(
+            "POST", "/execute", bucket=self.execute_bucket, json_body=payload, log_request=summary,
+            accept_error_body=lambda b: isinstance(b, dict) and ("code" in b or "status" in b))
+        if not isinstance(body, dict):
+            raise JupiterError("/execute", status, "non-JSON response")
+        body["_latency_ms"] = latency
+        body["_http_status"] = status
+        return body
+
+    def program_id_to_label(self, refresh: bool = False) -> dict[str, str]:
+        return fetch_program_labels(refresh=refresh)

fly_trader/chain/jupiter_tokens.py

@@ -0,0 +1,82 @@
+"""Jupiter Tokens API v2 client (https://api.jup.ag/tokens/v2).
+
+Endpoint (verified 2026-09-12 with the paid key): /search?query=<csv of up to 100 mints>. Tokens carry organicScore,
+holderCount, liquidity, mcap, stats5m/1h/6h/24h and audit: mintAuthorityDisabled,
+freezeAuthorityDisabled, topHoldersPercentage, devBalancePercentage, devMints, devMigrations (isSus is
+absent unless flagged). Rate limit headers: x-ratelimit-remaining/current/reset.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+
+import httpx
+
+from .. import config
+from ..db.apilog import record_api_call
+
+log = logging.getLogger(__name__)
+
+
+class TokenBucket:
+    def __init__(self, rate_per_s: float, burst: int | None = None):
+        self.rate = rate_per_s
+        self.capacity = burst or max(1, int(rate_per_s))
+        self.tokens = float(self.capacity)
+        self.ts = time.monotonic()
+        self.lock = threading.Lock()
+
+    def acquire(self) -> None:
+        while True:
+            with self.lock:
+                now = time.monotonic()
+                self.tokens = min(self.capacity, self.tokens + (now - self.ts) * self.rate)
+                self.ts = now
+                if self.tokens >= 1:
+                    self.tokens -= 1
+                    return
+                wait = (1 - self.tokens) / self.rate
+            time.sleep(wait)
+
+
+class JupiterTokens:
+    def __init__(self, api_key: str | None = None, rps: float | None = None):
+        self.api_key = api_key or config.JUPITER_API_KEY
+        if not self.api_key:
+            raise RuntimeError("JUPITER_API_KEY is not set")
+        self.base = config.JUPITER_TOKENS_BASE
+        self.bucket = TokenBucket(rps or config.JUPITER_RPS)
+        self.client = httpx.Client(timeout=30, headers={"x-api-key": self.api_key})
+
+    def _get(self, path: str, params: dict | None = None) -> list[dict]:
+        self.bucket.acquire()
+        t0 = time.monotonic()
+        status = None
+        err = None
+        try:
+            for attempt in range(4):
+                r = self.client.get(self.base + path, params=params)
+                status = r.status_code
+                if r.status_code == 429:
+                    time.sleep(1.0 + attempt)
+                    continue
+                r.raise_for_status()
+                data = r.json()
+                return data if isinstance(data, list) else []
+            raise httpx.HTTPStatusError("rate limited", request=r.request, response=r)
+        except Exception as e:
+            err = f"{type(e).__name__}: {e}"
+            raise
+        finally:
+            record_api_call("jupiter", f"tokens{path}", "GET", status, int((time.monotonic() - t0) * 1000),
+                            err is None, err, request={"params": params} if params else None)
+
+    def search(self, query: str) -> list[dict]:
+        return self._get("/search", {"query": query})
+
+    def search_many(self, mints: list[str]) -> list[dict]:
+        out: list[dict] = []
+        for i in range(0, len(mints), 100):
+            out.extend(self.search(",".join(mints[i:i + 100])))
+        return out

fly_trader/chain/keys.py

@@ -0,0 +1,141 @@
+"""Bot wallet keys (plan §9; pattern lifted from VOC dexlp/keys.py).
+
+The secret lives ONLY in the gitignored ``.env`` as ``BOT_PRIVATE_KEY`` — base58 of the 64-byte
+secret key (what ``create_wallet`` writes) or a JSON int array (what ``solana-keygen`` writes). It
+is never logged, never printed and never placed in an exception message: errors name the source
+and the failure type only. Note that ``str(Keypair)`` IS the base58 secret, so no Keypair object
+is ever formatted, repr'd or passed to a logger in this module.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from pathlib import Path
+
+import base58
+from psycopg.types.json import Jsonb
+from solders.keypair import Keypair
+
+from .. import config
+from ..db.connection import transaction
+
+log = logging.getLogger(__name__)
+
+ENV_NAME = "BOT_PRIVATE_KEY"
+ENV_PATH = config.REPO_ROOT / ".env"
+_ENV_LINE = re.compile(r"^\s*(?:export\s+)?BOT_PRIVATE_KEY\s*=")
+
+
+class WalletKeyError(RuntimeError):
+    """Missing or malformed bot key. The message never includes secret material."""
+
+
+class WalletExists(RuntimeError):
+    """``create_wallet`` refused because a BOT_PRIVATE_KEY already exists."""
+
+
+def _keypair_from(raw: str | None) -> Keypair:
+    raw = (raw or "").strip()
+    if not raw:
+        raise WalletKeyError(f"{ENV_NAME} not set")
+    try:
+        if raw.startswith("["):
+            arr = json.loads(raw)
+            if not isinstance(arr, list):
+                raise ValueError("not a list")
+            data = bytes(int(x) & 0xFF for x in arr)
+        else:
+            data = base58.b58decode(raw)
+    except Exception as e:  # never surface the value
+        raise WalletKeyError(f"could not decode {ENV_NAME}: {type(e).__name__}") from None
+    try:
+        if len(data) == 64:
+            return Keypair.from_bytes(data)
+        if len(data) == 32:  # a bare seed (solana-keygen never writes this, but accept it)
+            return Keypair.from_seed(data)
+    except Exception as e:
+        raise WalletKeyError(f"invalid {ENV_NAME} bytes: {type(e).__name__}") from None
+    raise WalletKeyError(f"{ENV_NAME} must decode to 64 bytes (got {len(data)})")
+
+
+def load_keypair(raw: str | None = None) -> Keypair:
+    """Keypair from ``BOT_PRIVATE_KEY`` (or ``raw``). Accepts base58 or a JSON int array."""
+    return _keypair_from(raw if raw is not None else os.environ.get(ENV_NAME))
+
+
+def bot_pubkey(raw: str | None = None) -> str:
+    return str(load_keypair(raw).pubkey())
+
+
+def key_available() -> bool:
+    try:
+        load_keypair()
+        return True
+    except WalletKeyError:
+        return False
+
+
+def env_file_has_key(path: Path | None = None) -> bool:
+    path = Path(path) if path else ENV_PATH
+    if not path.exists():
+        return False
+    with open(path, encoding="utf-8") as f:
+        return any(_ENV_LINE.match(line) for line in f)
+
+
+def create_wallet(env_path: Path | None = None, url: str | None = None) -> str:
+    """Generate the bot keypair once and append it to ``.env`` (mode 600). Returns the pubkey.
+
+    Refuses if ``BOT_PRIVATE_KEY`` is already present in the process environment or in the file.
+    A preceding newline is written when the file does not end with one. Only the pubkey is
+    logged; a ``wallet_events(kind='created')`` row records the event.
+    """
+    path = Path(env_path) if env_path else ENV_PATH
+    if os.environ.get(ENV_NAME, "").strip():
+        raise WalletExists(f"{ENV_NAME} is already set in the environment; refusing to create another wallet")
+    if env_file_has_key(path):
+        raise WalletExists(f"{path} already contains {ENV_NAME}; refusing to overwrite")
+    kp = Keypair()
+    pubkey = str(kp.pubkey())
+    secret_b58 = base58.b58encode(bytes(kp)).decode()
+    needs_newline = path.exists() and path.stat().st_size > 0 and not path.read_bytes().endswith(b"\n")
+    fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
+    try:
+        with os.fdopen(fd, "a", encoding="utf-8") as f:
+            if needs_newline:
+                f.write("\n")
+            f.write(f"{ENV_NAME}={secret_b58}\n")
+    finally:
+        os.chmod(path, 0o600)
+    del secret_b58, kp
+    try:
+        with transaction(url) as conn:
+            conn.execute(
+                "INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('created', %s, %s)",
+                (pubkey, Jsonb({"env_path": str(path)})),
+            )
+    except Exception as e:  # the key is already safely on disk; do not lose it over a DB hiccup
+        log.warning("wallet_events insert failed after wallet creation: %s", type(e).__name__)
+    log.info("bot wallet created: %s", pubkey)
+    return pubkey
+
+
+def show_wallet(rpc=None, url: str | None = None) -> str:
+    """Pubkey + SOL balance (read-only RPC); records ``wallet_events(kind='balance_snapshot')``."""
+    pubkey = bot_pubkey()
+    if rpc is None:
+        from .rpc import HttpSolanaRpc
+        rpc = HttpSolanaRpc()
+    lamports = int(rpc.get_balance(pubkey))
+    sol = lamports / config.LAMPORTS_PER_SOL
+    try:
+        with transaction(url) as conn:
+            conn.execute(
+                "INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('balance_snapshot', %s, %s)",
+                (pubkey, Jsonb({"lamports": lamports, "sol": sol})),
+            )
+    except Exception as e:
+        log.warning("wallet_events insert failed: %s", type(e).__name__)
+    return f"pubkey={pubkey} sol={sol:.9f} lamports={lamports}"

fly_trader/chain/rpc.py

@@ -0,0 +1,200 @@
+"""Solana JSON-RPC over Helius using httpx + solders only (no solana-py). Pattern: VOC
+dexlp/ingest/helius_swaps.py HttpSolanaRpc.
+
+The RPC URL carries the Helius API key: it is never logged, never stored in ``api_calls`` and never
+placed in an exception message (httpx error text is scrubbed and re-raised as our own types).
+Every call is persisted to ``api_calls`` (service='helius', endpoint=<method>) through
+``fly_trader.db.apilog.record_api_call``, which swallows DB errors so a database hiccup never
+breaks an RPC call.
+
+Response shapes verified against mainnet on 2026-09-12:
+  getBalance            -> {"context": {"slot", "apiVersion"}, "value": <int lamports>}
+  getTokenAccountsByOwner (jsonParsed) -> {"context", "value": [{"pubkey", "account": {"lamports",
+                           "owner": <program>, "data": {"program": "spl-token"|"spl-token-2022",
+                           "parsed": {"type": "account", "info": {"mint", "owner", "state", "isNative",
+                           "tokenAmount": {"amount": "<str>", "decimals", "uiAmount", "uiAmountString"},
+                           "extensions": [...] (token-2022 only)}}}}}]}
+  getSignatureStatuses  -> {"context", "value": [{"slot", "confirmations", "err", "status",
+                           "confirmationStatus": processed|confirmed|finalized} | null]}
+  getLatestBlockhash    -> {"context", "value": {"blockhash", "lastValidBlockHeight": <int>}}
+  getBlockHeight        -> <int>   (same units as lastValidBlockHeight)
+  getTransaction        -> {"transaction", "meta": {"err", "status", "fee", "preBalances",
+                           "postBalances", "preTokenBalances", "postTokenBalances", "innerInstructions",
+                           "logMessages", "computeUnitsConsumed", ...}, "version", "slot", "blockTime"}
+  sendTransaction       -> "<signature>"; malformed input returns error {"code": -32602, ...}
+  JSON-RPC batches are supported (array in, array out, matched by id).
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import Any
+
+import httpx
+
+from .. import config
+from ..db.apilog import record_api_call
+from ..logging_setup import scrub
+
+log = logging.getLogger(__name__)
+
+TOKEN_PROGRAM_ID = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"
+TOKEN_2022_PROGRAM_ID = "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb"
+TOKEN_PROGRAM_IDS = (TOKEN_PROGRAM_ID, TOKEN_2022_PROGRAM_ID)
+RPC_TIMEOUT_S = 15.0
+COMMITMENT = "confirmed"
+
+
+class RpcError(RuntimeError):
+    """A JSON-RPC level error ({"error": {code, message, data}})."""
+
+    def __init__(self, method: str, error: dict | Any):
+        err = error if isinstance(error, dict) else {"message": str(error)}
+        self.method = method
+        self.code = err.get("code")
+        self.message = str(err.get("message", ""))
+        self.data = err.get("data")
+        super().__init__(f"rpc {method} error {self.code}: {scrub(self.message)}")
+
+
+class RpcTransportError(RuntimeError):
+    """HTTP/transport failure. Message never contains the URL."""
+
+
+def _request_summary(method: str, params: list | None) -> dict:
+    if method == "sendTransaction" and params:
+        opts = params[1] if len(params) > 1 and isinstance(params[1], dict) else {}
+        return {"tx_b64_len": len(params[0]) if isinstance(params[0], str) else None, **opts}
+    if method == "getSignatureStatuses" and params:
+        # full signatures are 88 base58 chars and the log scrubber would redact them as secrets
+        return {"n": len(params[0]), "signature_prefixes": [str(s)[:12] for s in params[0]]}
+    if method == "getTransaction" and params:
+        return {"signature_prefix": str(params[0])[:12]}
+    return {"params": params}
+
+
+class HttpSolanaRpc:
+    def __init__(self, url: str | None = None, timeout_s: float = RPC_TIMEOUT_S):
+        self._url = url or config.helius_http_url()
+        self._client = httpx.Client(timeout=timeout_s)
+        self._id = 0
+        self._lock = threading.Lock()
+
+    def _next_id(self) -> int:
+        with self._lock:
+            self._id += 1
+            return self._id
+
+    def _post(self, payload: dict | list, endpoint: str, summary: dict | None):
+        t0 = time.monotonic()
+        status: int | None = None
+        err: str | None = None
+        nbytes: int | None = None
+        try:
+            try:
+                r = self._client.post(self._url, json=payload)
+            except httpx.HTTPError as e:
+                raise RpcTransportError(f"{endpoint}: {type(e).__name__}: {scrub(str(e))}") from None
+            status = r.status_code
+            nbytes = len(r.content)
+            if status >= 400:
+                raise RpcTransportError(f"{endpoint}: HTTP {status}: {scrub(r.text[:200])}")
+            try:
+                body = r.json()
+            except ValueError:
+                raise RpcTransportError(f"{endpoint}: non-JSON response ({nbytes} bytes)") from None
+            if isinstance(payload, dict):
+                if not isinstance(body, dict):
+                    raise RpcTransportError(f"{endpoint}: unexpected response shape {type(body).__name__}")
+                if body.get("error"):
+                    raise RpcError(endpoint, body["error"])
+                return body.get("result")
+            if not isinstance(body, list):
+                raise RpcTransportError(f"{endpoint}: batch response is not a list")
+            by_id = {item.get("id"): item for item in body if isinstance(item, dict)}
+            results = []
+            for req in payload:
+                item = by_id.get(req["id"])
+                if item is None:
+                    raise RpcTransportError(f"{endpoint}: missing batch response for id {req['id']}")
+                if item.get("error"):
+                    raise RpcError(req["method"], item["error"])
+                results.append(item.get("result"))
+            return results
+        except Exception as e:
+            err = str(e)
+            raise
+        finally:
+            record_api_call("helius", endpoint, "POST", status, int((time.monotonic() - t0) * 1000),
+                            err is None, err, request=summary, response_bytes=nbytes)
+
+    # ---- generic ----
+    def call(self, method: str, params: list | None = None):
+        payload = {"jsonrpc": "2.0", "id": self._next_id(), "method": method, "params": params or []}
+        return self._post(payload, method, _request_summary(method, params))
+
+    def batch(self, calls: list[tuple[str, list | None]]) -> list:
+        """Send several calls in one HTTP request; results in call order. Raises on the first error."""
+        if not calls:
+            return []
+        payload = [{"jsonrpc": "2.0", "id": self._next_id(), "method": m, "params": p or []} for m, p in calls]
+        methods = [m for m, _ in calls]
+        return self._post(payload, "batch:" + ",".join(methods), {"methods": methods})
+
+    # ---- typed helpers ----
+    def get_balance(self, pubkey) -> int:
+        res = self.call("getBalance", [str(pubkey), {"commitment": COMMITMENT}])
+        return int(res["value"])
+
+    def get_token_accounts_by_owner(self, pubkey) -> list[dict]:
+        """All token accounts of ``pubkey`` under BOTH token programs, as
+        ``{address, mint, amount:int, decimals, program}``."""
+        out: list[dict] = []
+        for program in TOKEN_PROGRAM_IDS:
+            res = self.call("getTokenAccountsByOwner",
+                            [str(pubkey), {"programId": program}, {"encoding": "jsonParsed", "commitment": COMMITMENT}])
+            for item in (res or {}).get("value") or []:
+                try:
+                    info = item["account"]["data"]["parsed"]["info"]
+                    amt = info["tokenAmount"]
+                    out.append({"address": item["pubkey"], "mint": info["mint"], "amount": int(amt["amount"]),
+                                "decimals": int(amt["decimals"]), "program": program})
+                except (KeyError, TypeError, ValueError):
+                    log.warning("unparsed token account %s", item.get("pubkey") if isinstance(item, dict) else "?")
+        return out
+
+    def get_signature_statuses(self, sigs: list[str], search_history: bool = False) -> list[dict | None]:
+        res = self.call("getSignatureStatuses", [list(sigs), {"searchTransactionHistory": bool(search_history)}])
+        value = (res or {}).get("value")
+        if not isinstance(value, list):
+            return [None] * len(sigs)
+        return value
+
+    def get_transaction(self, sig: str, encoding: str = "jsonParsed") -> dict | None:
+        return self.call("getTransaction", [sig, {"encoding": encoding, "maxSupportedTransactionVersion": 0,
+                                                  "commitment": COMMITMENT}])
+
+    def get_latest_blockhash(self) -> dict:
+        """``{"blockhash": str, "last_valid_block_height": int}``."""
+        v = self.call("getLatestBlockhash", [{"commitment": COMMITMENT}])["value"]
+        return {"blockhash": v["blockhash"], "last_valid_block_height": int(v["lastValidBlockHeight"])}
+
+    def get_block_height(self) -> int:
+        return int(self.call("getBlockHeight", [{"commitment": COMMITMENT}]))
+
+    def get_account_info(self, pubkey) -> dict | None:
+        res = self.call("getAccountInfo", [str(pubkey), {"encoding": "jsonParsed", "commitment": COMMITMENT}])
+        return (res or {}).get("value")
+
+    def get_mint_decimals(self, mint: str) -> int | None:
+        try:
+            info = self.get_account_info(mint)
+            return int(info["data"]["parsed"]["info"]["decimals"])
+        except Exception:
+            return None
+
+    def send_transaction(self, tx_b64: str, skip_preflight: bool = False, max_retries: int = 3) -> str:
+        sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight),
+                                                     "preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}])
+        return str(sig)

fly_trader/chain/signing.py

@@ -0,0 +1,86 @@
+"""Sign only our own signer slot of a Jupiter-built VersionedTransaction (plan §9).
+
+The multi-signer trap (flywheel utils/transaction_signing.py:99-205): ``VersionedTransaction(msg,
+[keypair])`` demands keypairs for EVERY required signer and replaces all signatures, which pushed
+that code into rewriting account keys and the header — Jupiter then rejects the transaction (and a
+rewritten fee payer would make us pay for someone else's message). Here the message, header and fee
+payer are never touched: we locate our pubkey among the static account keys, require its index to
+be below ``num_required_signatures``, sign the versioned message bytes, and drop the signature into
+that slot while keeping any signatures already present (RFQ makers pre-sign their slot).
+
+Verified 2026-09-12: a Swap v2 ``/order`` transaction with a taker is v0 with
+``num_required_signatures=1``, the taker as static key 0 (fee payer), 4 address-table lookups and no
+pre-filled signatures.
+"""
+from __future__ import annotations
+
+import base64
+
+from solders.message import to_bytes_versioned
+from solders.signature import Signature
+from solders.transaction import VersionedTransaction
+
+
+class NotASigner(ValueError):
+    """Our pubkey is absent from the static keys or sits outside the signer range."""
+
+
+def _as_tx(tx) -> VersionedTransaction:
+    if isinstance(tx, VersionedTransaction):
+        return tx
+    if isinstance(tx, str):
+        tx = base64.b64decode(tx)
+    return VersionedTransaction.from_bytes(bytes(tx))
+
+
+def signer_index(tx, pubkey) -> int:
+    """Index of ``pubkey`` among the static account keys, verified to be a required signer."""
+    tx = _as_tx(tx)
+    msg = tx.message
+    keys = list(msg.account_keys)
+    if pubkey not in keys:
+        raise NotASigner(f"{pubkey} is not among the transaction's static account keys")
+    idx = keys.index(pubkey)
+    n_req = msg.header.num_required_signatures
+    if idx >= n_req:
+        raise NotASigner(f"{pubkey} is account {idx} but only the first {n_req} account(s) sign")
+    return idx
+
+
+def sign_transaction_b64(tx_b64: str, keypair) -> tuple[str, int]:
+    """Return ``(base64 signed transaction, our signer index)``. Message/header/fee payer untouched."""
+    tx = _as_tx(tx_b64)
+    msg = tx.message
+    idx = signer_index(tx, keypair.pubkey())
+    n_req = msg.header.num_required_signatures
+    sigs = list(tx.signatures)
+    if len(sigs) < n_req:
+        sigs.extend([Signature.default()] * (n_req - len(sigs)))
+    sigs[idx] = keypair.sign_message(to_bytes_versioned(msg))
+    signed = VersionedTransaction.populate(msg, sigs)
+    return base64.b64encode(bytes(signed)).decode(), idx
+
+
+def transaction_id(tx) -> str:
+    """The transaction signature (first signature) of a signed transaction."""
+    return str(_as_tx(tx).signatures[0])
+
+
+def static_program_ids(tx) -> list[str]:
+    """Program ids of the compiled instructions that resolve to STATIC account keys, unique, in order.
+    Program ids that live in address lookup tables are not resolved in v1 (see ``n_lookup_programs``)."""
+    msg = _as_tx(tx).message
+    keys = [str(k) for k in msg.account_keys]
+    out: list[str] = []
+    for ci in msg.instructions:
+        i = ci.program_id_index
+        if i < len(keys) and keys[i] not in out:
+            out.append(keys[i])
+    return out
+
+
+def n_lookup_programs(tx) -> int:
+    """How many compiled instructions reference a program id from an address lookup table."""
+    msg = _as_tx(tx).message
+    n_static = len(msg.account_keys)
+    return sum(1 for ci in msg.instructions if ci.program_id_index >= n_static)

fly_trader/config.py

@@ -0,0 +1,196 @@
+"""Typed configuration from the environment (.env is loaded once, never overriding real env vars).
+
+Every knob lives here. Secret values are never printed; ``summary()`` returns the non-secret
+configuration for persistence in ``runs.config``. All time handling is UTC.
+"""
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+load_dotenv(REPO_ROOT / ".env", override=False)
+
+SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY")
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def env_str(name: str, default: str | None = None) -> str | None:
+    v = os.environ.get(name)
+    if v is None or v.strip() == "":
+        return default
+    return v.strip()
+
+
+def env_int(name: str, default: int) -> int:
+    v = env_str(name)
+    return int(v) if v is not None else default
+
+
+def env_float(name: str, default: float) -> float:
+    v = env_str(name)
+    return float(v) if v is not None else default
+
+
+def env_bool(name: str, default: bool = False) -> bool:
+    v = env_str(name)
+    if v is None:
+        return default
+    return v.lower() in ("1", "true", "yes", "on")
+
+
+def env_list(name: str, default: list[str]) -> list[str]:
+    v = env_str(name)
+    if v is None:
+        return list(default)
+    return [x.strip() for x in v.split(",") if x.strip()]
+
+# ---- secrets (names only ever appear in logs) ----
+HELIUS_API_KEY = env_str("HELIUS_API_KEY")
+JUPITER_API_KEY = env_str("JUPITER_API_KEY")
+
+# ---- paths ----
+DATA_DIR = (REPO_ROOT / env_str("DATA_DIR", "data")).resolve()
+LOG_DIR = (REPO_ROOT / env_str("LOG_DIR", "logs")).resolve()
+BRAIN_DIR = DATA_DIR / "brain"
+PG_ARCHIVE_DIR = DATA_DIR / "pg_archive"
+
+# ---- database ----
+DATABASE_URL = env_str("DATABASE_URL", "postgresql:///fly_trader")
+
+# ---- chain / execution (default-deny) ----
+SOLANA_CLUSTER = env_str("SOLANA_CLUSTER", "mainnet-beta")
+LIVE_ENABLED = env_bool("LIVE_ENABLED", False)
+CAPITAL_SOL = env_float("CAPITAL_SOL", 0.0)
+MAX_POSITION_SOL = env_float("MAX_POSITION_SOL", 0.0)
+GAS_RESERVE_SOL = env_float("GAS_RESERVE_SOL", 0.30)
+# position sizing (agent/sizing.py): a fraction of the growth-optimal (Kelly) bet measured per score band in the backtest
+KELLY_FRACTION = env_float("KELLY_FRACTION", 0.25)                 # quarter Kelly: the band estimates are noisy
+MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10)   # of the deployable bankroll (wealth minus the gas reserve)
+MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02)                 # of the pool's quote reserve (bounds price impact)
+# The training label charges what an exit really costs at the size the book takes. market/features.py prices
+# exit_cost_0p1 for a fixed 0.1 SOL, but positions are Kelly-sized up to MAX_POSITION_FRACTION of the bankroll, and
+# impact = value/(value + reserves) grows with size. 0 = the sizing rule's own ceiling (train/decisions.py).
+LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.0)
+MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02)             # smaller sizes are skipped
+KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30)
+KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False)   # a tripped kill switch also sells every open live position (agent/fly_live.py)
+CIRCUIT_THRESHOLD = env_int("CIRCUIT_THRESHOLD", 3)
+EXECUTABILITY_MAX_IMPACT = env_float("EXECUTABILITY_MAX_IMPACT", 0.30)
+SLIPPAGE_ENTRY_BPS = env_int("SLIPPAGE_ENTRY_BPS", 150)
+SLIPPAGE_EXIT_BPS = env_int("SLIPPAGE_EXIT_BPS", 300)
+SLIPPAGE_FORCED_BPS = env_int("SLIPPAGE_FORCED_BPS", 500)
+SLIPPAGE_STEP_BPS = env_int("SLIPPAGE_STEP_BPS", 100)
+MAX_SLIPPAGE_ENTRY_BPS = env_int("MAX_SLIPPAGE_ENTRY_BPS", 300)
+
+HELIUS_HTTP = "https://mainnet.helius-rpc.com/"
+JUPITER_SWAP_BASE = "https://api.jup.ag/swap/v2"
+JUPITER_TOKENS_BASE = "https://api.jup.ag/tokens/v2"
+JUPITER_RPS = env_float("JUPITER_RPS", 8.0)
+
+WSOL_MINT = "So11111111111111111111111111111111111111112"
+USDC_MINT = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v"
+LAMPORTS_PER_SOL = 1_000_000_000
+
+# ---- token stats (Jupiter) ----
+STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0)
+
+# ---- connectome (the fly) ----
+DEVICE = env_str("DEVICE", "mps")
+NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu")  # shiu | flybrain
+CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations")  # annotations | flybrain
+
+# ---- costs ----
+LAMBDA_IMPACT_DLMM = env_float("LAMBDA_IMPACT_DLMM", 0.02)
+
+# ---- training lifecycle ----
+RESET_ON_START = env_bool("RESET_ON_START", True)  # wipe the paper book and its stats when the runner starts
+
+# ---- the fly's handover (agent/fly_session.py): when its paper race earns it the live seat ----
+HANDOVER_DAYS = env_int("HANDOVER_DAYS", 14)                        # days of racing first; 0 = as soon as the trades are in
+HANDOVER_MIN_TRADES = env_int("HANDOVER_MIN_TRADES", 30)            # closed paper_fly trades required
+HANDOVER_BEAT_SELECTOR = env_bool("HANDOVER_BEAT_SELECTOR", True)   # and realized P&L at least the paper selector's over the window
+
+
+def helius_http_url() -> str:
+    if not HELIUS_API_KEY:
+        raise RuntimeError("HELIUS_API_KEY is not set")
+    return f"{HELIUS_HTTP}?api-key={HELIUS_API_KEY}"
+
+
+def summary() -> dict:
+    """Non-secret configuration snapshot for ``runs.config``."""
+    out = {}
+    for k, v in globals().items():
+        if k.isupper() and k not in SECRET_ENV_NAMES and not k.startswith("_"):
+            if isinstance(v, (str, int, float, bool, list, tuple)):
+                out[k] = v
+            elif isinstance(v, Path):
+                out[k] = str(v)
+            elif isinstance(v, dict):
+                out[k] = v
+    if "DATABASE_URL" in out:                      # may carry a password (URL or key=value form); the database name is enough
+        try:
+            from psycopg.conninfo import conninfo_to_dict
+            db = conninfo_to_dict(str(out["DATABASE_URL"])).get("dbname") or "?"
+        except Exception:
+            db = "?"
+        out["DATABASE_URL"] = f"postgresql:///{db}"
+    return out
+
+
+def live_prerequisites_missing() -> list[str]:
+    missing = []
+    if SOLANA_CLUSTER != "mainnet-beta":
+        missing.append("SOLANA_CLUSTER must be mainnet-beta for live memecoin trading")
+    if not LIVE_ENABLED:
+        missing.append("LIVE_ENABLED=1")
+    if CAPITAL_SOL <= 0:
+        missing.append("CAPITAL_SOL")
+    if MAX_POSITION_SOL <= 0:
+        missing.append("MAX_POSITION_SOL")
+    if GAS_RESERVE_SOL <= 0:
+        missing.append("GAS_RESERVE_SOL")
+    if not env_str("BOT_PRIVATE_KEY"):
+        missing.append("BOT_PRIVATE_KEY")
+    if not JUPITER_API_KEY:
+        missing.append("JUPITER_API_KEY")
+    if not HELIUS_API_KEY:
+        missing.append("HELIUS_API_KEY")
+    return missing
+
+# ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ----
+CORPUS_DIR = DATA_DIR / "corpus"
+CORPUS_DAYS = env_int("CORPUS_DAYS", 240)                          # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE)
+CORPUS_MIGRATION_AUTHORITY = env_str("CORPUS_MIGRATION_AUTHORITY", "39azUYFWPz3VHgKCf3VChUwbpURdCHRxjWVowf5jUJjg")
+CORPUS_SWAP_API = env_str("CORPUS_SWAP_API", "https://swap-api.pump.fun")
+CORPUS_PACE_S = env_float("CORPUS_PACE_S", 2.2)                    # seconds between swap-api requests (~30/min bucket; 1/s throttles after ~20)
+CORPUS_PACE_MIN_S = env_float("CORPUS_PACE_MIN_S", 2.2)
+CORPUS_CANDLE_1M_H = env_float("CORPUS_CANDLE_1M_H", 12.0)          # 1-minute candles for this long after graduation
+CORPUS_CANDLE_5M_D = env_float("CORPUS_CANDLE_5M_D", 7.0)          # then 5-minute candles until this many days
+CORPUS_MIN_LIFE_H = env_float("CORPUS_MIN_LIFE_H", 1.0)             # per-trade rows only for tokens that traded at least this long
+CORPUS_TRADES_H = env_float("CORPUS_TRADES_H", 3.0)                 # per-trade rows for the first hours after graduation
+CORPUS_TRADES_DAYS = env_int("CORPUS_TRADES_DAYS", 400)             # ... and only for graduations this recent (the 10 % sample bounds the cost)
+CORPUS_TRADES_MAX_PAGES = env_int("CORPUS_TRADES_MAX_PAGES", 200)   # 100 trades per page
+CORPUS_TRADES_SAMPLE = env_float("CORPUS_TRADES_SAMPLE", 0.10)      # share of surviving tokens that get per-trade rows (deterministic by mint)
+CORPUS_MIN_AGE_H = env_float("CORPUS_MIN_AGE_H", 6.0)                # pull a token only once its first hours are complete
+CORPUS_RQ0_SOL = env_float("CORPUS_RQ0_SOL", 79.0)                  # PumpSwap quote reserve at migration (85 SOL curve − 6 SOL fee); R_q(t) ≈ RQ0·sqrt(p/p_grad)
+CORPUS_FEATURES_DIR = CORPUS_DIR / "features"
+
+# ---- PumpAPI historical replay (free hourly archive of decoded pump.fun / PumpSwap events since 2026-04-18) ----
+REPLAY_URL = env_str("REPLAY_URL", "https://replay.pumpapi.io")
+REPLAY_START = env_str("REPLAY_START", "2026-04-18")
+REPLAY_PARALLEL = env_int("REPLAY_PARALLEL", 4)                     # concurrent hour downloads (~2-4 MB/s each)
+REPLAY_ASSEMBLE = env_bool("REPLAY_ASSEMBLE", True)                  # 0: download and parse only (no assembly / corpus_meta / mature rounds)
+REPLAY_DIR = CORPUS_DIR / "replay"
+CORPUS_PULL_BEFORE = env_str("CORPUS_PULL_BEFORE", REPLAY_START)     # swap-api puller only handles graduations before the replay archive begins
+
+# ---- PumpAPI live stream (free firehose, same events as the replay archive; the selector's parity feed) ----
+PUMPSTREAM_URL = env_str("PUMPSTREAM_URL", "wss://stream.pumpapi.io")
+PUMP_MINUTES_KEEP_DAYS = env_int("PUMP_MINUTES_KEEP_DAYS", 7)       # older minute rows are archived to Parquet, then removed from the table

fly_trader/db/apilog.py

@@ -0,0 +1,49 @@
+"""Persist every external API call to ``api_calls``. Never stores keys; never breaks the caller."""
+from __future__ import annotations
+
+import json
+import logging
+
+from .connection import transaction
+from ..logging_setup import scrub
+
+log = logging.getLogger(__name__)
+
+
+def record_api_call(service: str, endpoint: str, method: str = "GET", status: int | None = None,
+                    latency_ms: int | None = None, ok: bool = True, error: str | None = None,
+                    request: dict | None = None, response_bytes: int | None = None) -> None:
+    try:
+        req_json = None
+        if request is not None:
+            req_json = json.loads(scrub(json.dumps(request, default=str)))
+        with transaction() as conn:
+            conn.execute(
+                "INSERT INTO api_calls (service, endpoint, method, status, latency_ms, ok, error, request, response_bytes) "
+                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+                (service, endpoint, method, status, latency_ms, ok, scrub(error) if error else None,
+                 json.dumps(req_json) if req_json is not None else None, response_bytes),
+            )
+    except Exception as e:  # a DB hiccup must never break an API call
+        log.warning("api_calls insert failed: %s", type(e).__name__)
+
+
+def _finite(x):
+    if isinstance(x, float) and (x != x or x in (float("inf"), float("-inf"))):
+        return None
+    if isinstance(x, dict):
+        return {k: _finite(v) for k, v in x.items()}
+    if isinstance(x, (list, tuple)):
+        return [_finite(v) for v in x]
+    return x
+
+
+def record_event(level: str, source: str, message: str, detail: dict | None = None) -> None:
+    try:
+        with transaction() as conn:
+            conn.execute(
+                "INSERT INTO events (level, source, message, detail) VALUES (%s,%s,%s,%s)",
+                (level, source, scrub(message), json.dumps(_finite(detail), default=str) if detail is not None else None),
+            )
+    except Exception as e:
+        log.warning("events insert failed: %s", type(e).__name__)

fly_trader/db/connection.py

@@ -0,0 +1,85 @@
+"""psycopg 3 connections to the fly_trader database.
+
+Pattern from better_bot code/db_connection.py: a production database is refused while pytest is
+running unless its name contains "test" (the autouse fixture in tests/conftest.py points
+DATABASE_URL at fly_trader_test).
+"""
+from __future__ import annotations
+
+import os
+import time
+from contextlib import contextmanager
+
+import psycopg
+from psycopg.conninfo import conninfo_to_dict
+from psycopg.rows import dict_row
+
+from .. import config
+
+
+class ProductionDatabaseUnderPytest(RuntimeError):
+    pass
+
+
+def database_url() -> str:
+    return os.environ.get("DATABASE_URL") or config.DATABASE_URL
+
+
+def database_name(url: str | None = None) -> str:
+    return conninfo_to_dict(url or database_url()).get("dbname") or "fly_trader"   # URL or key=value DSN
+
+
+def _refuse_production_under_pytest(url: str) -> None:
+    if "PYTEST_CURRENT_TEST" in os.environ and "test" not in database_name(url):
+        raise ProductionDatabaseUnderPytest(
+            f"refusing to open non-test database {database_name(url)!r} under pytest"
+        )
+
+
+def connect(url: str | None = None, *, autocommit: bool = False) -> psycopg.Connection:
+    url = url or database_url()
+    _refuse_production_under_pytest(url)
+    last: Exception | None = None
+    for attempt in range(3):          # a busy machine can refuse one connection; a long training run must not die for it,
+        # but an unreachable database must still report in seconds, not half a minute: three short attempts, ~10 s in all
+        try:
+            return psycopg.connect(url, autocommit=autocommit, row_factory=dict_row, options="-c timezone=UTC", connect_timeout=3)
+        except psycopg.OperationalError as e:
+            last = e
+            if attempt < 2:
+                time.sleep(0.25 * 2 ** attempt)
+    raise last
+
+
+@contextmanager
+def transaction(url: str | None = None):
+    """One transaction; commits on success, rolls back on exception."""
+    conn = connect(url)
+    try:
+        yield conn
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        conn.close()
+
+
+def fetch_all(sql: str, params=None, url: str | None = None) -> list[dict]:
+    with connect(url) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            return cur.fetchall()
+
+
+def fetch_one(sql: str, params=None, url: str | None = None) -> dict | None:
+    with connect(url) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            return cur.fetchone()
+
+
+def execute(sql: str, params=None, url: str | None = None) -> None:
+    with transaction(url) as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, params)

fly_trader/db/queries.py

@@ -0,0 +1,37 @@
+"""Read-side helpers for the console and `status`. Everything counts rows; nothing reads status flags."""
+from __future__ import annotations
+
+import json
+
+from .connection import connect
+
+
+def q(sql: str, params=None) -> list[dict]:
+    with connect() as conn:
+        return [dict(r) for r in conn.execute(sql, params).fetchall()]
+
+
+def q1(sql: str, params=None) -> dict | None:
+    rows = q(sql, params)
+    return rows[0] if rows else None
+
+
+def status() -> dict:
+    out: dict = {}
+    out["wealth"] = q("""SELECT DISTINCT ON (book) book, ts, wealth, sol_free, positions_value, exposure, n_open, peak, drawdown
+                         FROM wealth_marks ORDER BY book, ts DESC""")
+    out["positions"] = q("SELECT book, status, count(*) AS n, COALESCE(sum(realized_sol),0) AS realized FROM positions GROUP BY 1,2 ORDER BY 1,2")
+    out["fills"] = q("SELECT book, count(*) AS n, min(ts) AS first, max(ts) AS last FROM fills GROUP BY 1 ORDER BY 1")
+    out["beats_last_hour"] = (q1("SELECT count(*) AS n, max(ts) AS last FROM beats WHERE ts > now() - interval '1 hour'") or {})
+    out["decisions_last_hour"] = q("SELECT kind, COALESCE(rail,'') AS rail, count(*) AS n FROM decisions WHERE ts > now() - interval '1 hour' GROUP BY 1,2 ORDER BY 3 DESC")
+    out["tokens"] = q("SELECT watch_status, count(*) AS n FROM tokens GROUP BY 1 ORDER BY 1")
+    out["circuit"] = q1("SELECT * FROM circuit_state WHERE id = 1")
+    out["brain"] = q1("SELECT * FROM brain_state")
+    out["processes"] = q("SELECT name, pid, started_at, stopped_at, exit_code FROM processes ORDER BY started_at DESC LIMIT 10")
+    out["events"] = q("SELECT ts, level, source, message FROM events ORDER BY ts DESC LIMIT 10")
+    return out
+
+
+def print_status() -> None:
+    s = status()
+    print(json.dumps(s, indent=1, default=str))

fly_trader/db/schema.py

@@ -0,0 +1,363 @@
+"""Versioned DDL ladder for the fly_trader database.
+
+Pattern from better_bot code/db_schema.py: a singleton ``schema_version`` row, a cross-process
+advisory lock around migrations, and an idempotent list of statements. Adding a migration means
+appending a version-gated block AND bumping SCHEMA_VERSION, or it never runs.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+import psycopg
+from psycopg import sql
+
+from .. import config
+from .connection import connect, database_name, database_url
+
+log = logging.getLogger(__name__)
+
+SCHEMA_VERSION = 6          # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6])
+_LOCK_KEY = 0x666C795F6D6967  # "fly_mig"
+
+BASE_DDL: list[str] = [
+    """CREATE TABLE IF NOT EXISTS schema_version (
+        singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
+        version bigint NOT NULL,
+        updated_at timestamptz NOT NULL DEFAULT now())""",
+    """CREATE TABLE IF NOT EXISTS wallet_events (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(),
+        kind text NOT NULL, pubkey text, detail jsonb)""",
+    """CREATE TABLE IF NOT EXISTS tokens (
+        mint text PRIMARY KEY, symbol text, name text, decimals int, token_program text,
+        launchpad text, partner_config text, graduated_pool text, graduated_at timestamptz,
+        first_pool_id text, first_pool_created_at timestamptz,
+        mint_auth_disabled boolean, freeze_auth_disabled boolean,
+        first_seen timestamptz NOT NULL DEFAULT now(), last_seen timestamptz NOT NULL DEFAULT now(),
+        watch_status text NOT NULL DEFAULT 'candidate', raw jsonb)""",
+    "CREATE INDEX IF NOT EXISTS tokens_graduated_at_idx ON tokens (graduated_at DESC)",
+    "CREATE INDEX IF NOT EXISTS tokens_watch_status_idx ON tokens (watch_status)",
+    """CREATE TABLE IF NOT EXISTS token_stats (
+        id bigserial PRIMARY KEY, mint text NOT NULL, ts timestamptz NOT NULL DEFAULT now(),
+        organic_score real, organic_label text, holder_count int, liquidity_usd double precision,
+        usd_price double precision, mcap double precision, top_holders_pct real, dev_balance_pct real,
+        dev_mints int, is_sus boolean, mint_auth_disabled boolean, freeze_auth_disabled boolean,
+        is_verified boolean, stats jsonb)""",
+    "CREATE INDEX IF NOT EXISTS token_stats_mint_ts_idx ON token_stats (mint, ts DESC)",
+    """CREATE TABLE IF NOT EXISTS watch_pools (
+        pool text PRIMARY KEY, mint text NOT NULL, quote_mint text, program_id text, program_label text,
+        base_vault text, quote_vault text, base_decimals int, quote_decimals int, source text,
+        active boolean NOT NULL DEFAULT true, tradable boolean NOT NULL DEFAULT true,
+        added_at timestamptz NOT NULL DEFAULT now(), deactivated_at timestamptz, reason text)""",
+    "CREATE INDEX IF NOT EXISTS watch_pools_active_idx ON watch_pools (active)",
+    "CREATE INDEX IF NOT EXISTS watch_pools_mint_idx ON watch_pools (mint)",
+    """CREATE TABLE IF NOT EXISTS swap_tape (
+        id bigserial, ts timestamptz NOT NULL, slot bigint, sig text, tx_index int,
+        pool text NOT NULL, mint text, side smallint NOT NULL,
+        amount_base numeric(30,0), amount_quote bigint, price_sol double precision,
+        signer text, res_base numeric(30,0), res_quote bigint, program_label text,
+        PRIMARY KEY (id, ts)) PARTITION BY RANGE (ts)""",
+    "CREATE INDEX IF NOT EXISTS swap_tape_pool_ts_idx ON swap_tape (pool, ts)",
+    """CREATE TABLE IF NOT EXISTS capture_status (
+        singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
+        updated_at timestamptz NOT NULL DEFAULT now(), last_swap_ts timestamptz, rows_total bigint DEFAULT 0,
+        pools_subscribed int DEFAULT 0, subscription_id bigint, reconnects int DEFAULT 0,
+        decode_failures bigint DEFAULT 0, last_error text)""",
+    """CREATE TABLE IF NOT EXISTS runs (
+        run_id uuid PRIMARY KEY, kind text NOT NULL, started_at timestamptz NOT NULL DEFAULT now(),
+        ended_at timestamptz, git_sha text, config jsonb, connectome_sha256 text, encoder_sha256 text,
+        brain_snapshot_id bigint, corpus text, split jsonb, status text NOT NULL DEFAULT 'running',
+        metrics jsonb)""",
+    """CREATE TABLE IF NOT EXISTS populations (
+        id smallint PRIMARY KEY, name text NOT NULL UNIQUE, n int NOT NULL, kind text, source text)""",
+    """CREATE TABLE IF NOT EXISTS beats (
+        id bigserial PRIMARY KEY, run_id uuid, ts timestamptz NOT NULL DEFAULT now(), beat_no bigint,
+        sim_ts timestamptz, tape_last_id bigint, n_slots_active int, n_held_live int, n_held_paper int,
+        ticks int, gpu_ms int, total_ms int, feed_age_ms int, notes jsonb)""",
+    "CREATE INDEX IF NOT EXISTS beats_run_ts_idx ON beats (run_id, ts)",
+    "CREATE INDEX IF NOT EXISTS beats_ts_idx ON beats (ts)",
+    """CREATE TABLE IF NOT EXISTS beat_slots (
+        beat_id bigint NOT NULL, slot smallint NOT NULL, ts timestamptz NOT NULL,
+        mint text, pool text, dwell_beats int, features real[], feature_mask bigint, danger real,
+        portfolio real[], glomeruli real[], stim real[], kc_active_frac real, m_hat real,
+        rho_app real, rho_av real, dan_rew real, dan_pun real, delta_in real,
+        decision_kind text, decision_size real, softmax_p real,
+        PRIMARY KEY (beat_id, slot, ts)) PARTITION BY RANGE (ts)""",
+    "CREATE INDEX IF NOT EXISTS beat_slots_mint_ts_idx ON beat_slots (mint, ts)",
+    """CREATE TABLE IF NOT EXISTS brain_activity (
+        beat_id bigint PRIMARY KEY, group_rates real[], kc_sparsity real, mbon_app_rate real,
+        mbon_av_rate real, dan_rew_rate real, dan_pun_rate real, total_spikes bigint,
+        v_mean real, v_max real, nan_flag boolean DEFAULT false)""",
+    """CREATE TABLE IF NOT EXISTS decisions (
+        id bigserial PRIMARY KEY, beat_id bigint, run_id uuid, ts timestamptz NOT NULL DEFAULT now(),
+        slot smallint, mint text, pool text, kind text NOT NULL, m_hat real, size_sol real,
+        forced boolean NOT NULL DEFAULT false, rail text, reason text, softmax_p real,
+        book_targets text[], detail jsonb)""",
+    "CREATE INDEX IF NOT EXISTS decisions_ts_idx ON decisions (ts)",
+    "CREATE INDEX IF NOT EXISTS decisions_mint_ts_idx ON decisions (mint, ts)",
+    "CREATE INDEX IF NOT EXISTS decisions_kind_ts_idx ON decisions (kind, ts)",
+    """CREATE TABLE IF NOT EXISTS orders (
+        id bigserial PRIMARY KEY, decision_id bigint, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(),
+        attempt int NOT NULL DEFAULT 1, side text NOT NULL, input_mint text, output_mint text,
+        amount_in numeric(30,0), slippage_bps int, request jsonb, order_response jsonb,
+        execute_request jsonb, execute_response jsonb, request_id text, signature text,
+        last_valid_block_height bigint, router text, fee_bps int, program_ids text[],
+        status text, error_code int, error text, latency_ms int)""",
+    "CREATE INDEX IF NOT EXISTS orders_decision_idx ON orders (decision_id)",
+    "CREATE INDEX IF NOT EXISTS orders_signature_idx ON orders (signature)",
+    "CREATE INDEX IF NOT EXISTS orders_ts_idx ON orders (ts)",
+    """CREATE TABLE IF NOT EXISTS fills (
+        id bigserial PRIMARY KEY, order_id bigint, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(),
+        signature text, slot bigint, mint text, side text, token_delta numeric(30,0),
+        sol_delta_lamports bigint, price_sol double precision, fee_lamports bigint, platform_fee jsonb,
+        verified_by text, pre_snapshot jsonb, post_snapshot jsonb)""",
+    "CREATE INDEX IF NOT EXISTS fills_book_ts_idx ON fills (book, ts)",
+    "CREATE INDEX IF NOT EXISTS fills_mint_ts_idx ON fills (mint, ts)",
+    """CREATE TABLE IF NOT EXISTS positions (
+        id bigserial PRIMARY KEY, book text NOT NULL, mint text NOT NULL, pool text,
+        opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz,
+        entry_decision_id bigint, exit_decision_id bigint, qty numeric(30,0) NOT NULL DEFAULT 0,
+        cost_sol double precision NOT NULL DEFAULT 0, entry_price double precision, peak_price double precision,
+        last_mark_price double precision, last_mark_ts timestamptz, exit_price double precision,
+        realized_sol double precision, fees_sol double precision NOT NULL DEFAULT 0,
+        status text NOT NULL DEFAULT 'open', forced_exit_kind text, last_swap_ts timestamptz, satiety double precision NOT NULL DEFAULT 0)""",
+    "CREATE UNIQUE INDEX IF NOT EXISTS positions_open_uniq ON positions (book, mint) WHERE status = 'open'",
+    "CREATE INDEX IF NOT EXISTS positions_book_status_idx ON positions (book, status)",
+    """CREATE TABLE IF NOT EXISTS wealth_marks (
+        beat_id bigint NOT NULL, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(),
+        sol_free double precision, positions_value double precision, exit_cost double precision,
+        wealth double precision, peak double precision, drawdown real, exposure double precision, n_open int,
+        PRIMARY KEY (beat_id, book))""",
+    "CREATE INDEX IF NOT EXISTS wealth_marks_book_ts_idx ON wealth_marks (book, ts)",
+    """CREATE TABLE IF NOT EXISTS rewards (
+        beat_id bigint NOT NULL, slot smallint NOT NULL, book text, mint text,
+        r_slot real, r_global real, r_tilde real, m_prev real, m_now real, delta real, source text,
+        PRIMARY KEY (beat_id, slot))""",
+    """CREATE TABLE IF NOT EXISTS synapse_updates (
+        beat_id bigint PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), eta real, gamma real,
+        n_slots int, reward_source text, sum_abs real, max_abs real, frob real, frob_capped boolean,
+        n_pos int, n_neg int, n_clipped int, w_mean real, w_min real, w_max real,
+        delta_path text, delta_sha256 text)""",
+    """CREATE TABLE IF NOT EXISTS brain_snapshots (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), run_id uuid, beat_id bigint,
+        path text NOT NULL, sha256 text, kind text NOT NULL, promoted_at timestamptz, promoted_by text, note text)""",
+    """CREATE TABLE IF NOT EXISTS brain_state (
+        singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
+        live_snapshot_id bigint, pending_snapshot_id bigint, updated_at timestamptz NOT NULL DEFAULT now())""",
+    "INSERT INTO brain_state (singleton) VALUES (true) ON CONFLICT DO NOTHING",
+    """CREATE TABLE IF NOT EXISTS encoder_state (
+        name text PRIMARY KEY, version int NOT NULL DEFAULT 1, seed bigint, path text, sha256 text,
+        running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0,
+        updated_at timestamptz NOT NULL DEFAULT now())""",
+    """CREATE TABLE IF NOT EXISTS circuit_state (
+        id int PRIMARY KEY CHECK (id = 1), fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz,
+        tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text,
+        peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false,
+        updated_at timestamptz NOT NULL DEFAULT now())""",
+    "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING",
+    """CREATE TABLE IF NOT EXISTS circuit_events (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""",
+    """CREATE TABLE IF NOT EXISTS notional_ledger (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), sol double precision NOT NULL, kind text)""",
+    "CREATE INDEX IF NOT EXISTS notional_ledger_ts_idx ON notional_ledger (ts)",
+    """CREATE TABLE IF NOT EXISTS api_calls (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), service text NOT NULL, endpoint text,
+        method text, status int, latency_ms int, ok boolean, error text, request jsonb, response_bytes int)""",
+    "CREATE INDEX IF NOT EXISTS api_calls_ts_idx ON api_calls (ts)",
+    "CREATE INDEX IF NOT EXISTS api_calls_service_ok_ts_idx ON api_calls (service, ok, ts)",
+    """CREATE TABLE IF NOT EXISTS events (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), level text NOT NULL DEFAULT 'info',
+        source text, message text NOT NULL, detail jsonb)""",
+    "CREATE INDEX IF NOT EXISTS events_ts_idx ON events (ts)",
+    """CREATE TABLE IF NOT EXISTS slot_visits (
+        mint text PRIMARY KEY, last_visit_ts timestamptz, visits bigint NOT NULL DEFAULT 0)""",
+    """CREATE TABLE IF NOT EXISTS processes (
+        id bigserial PRIMARY KEY, name text NOT NULL, pid int NOT NULL, cmd text[],
+        started_at timestamptz NOT NULL DEFAULT now(), stopped_at timestamptz, exit_code int,
+        log_path text, started_by text)""",
+    "CREATE UNIQUE INDEX IF NOT EXISTS processes_live_uniq ON processes (name) WHERE stopped_at IS NULL",
+    """CREATE TABLE IF NOT EXISTS corpus_tokens (
+        mint text PRIMARY KEY, graduated_at timestamptz NOT NULL, grad_sig text, grad_slot bigint,
+        source text NOT NULL DEFAULT 'migration_wallet', status text NOT NULL DEFAULT 'pending',
+        life_h double precision, candles_1m int, candles_5m int, trades int, trades_through timestamptz,
+        candle_path text, trade_path text, last_error text,
+        added_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""",
+    "CREATE INDEX IF NOT EXISTS corpus_tokens_status_idx ON corpus_tokens (status, graduated_at DESC)",
+    """CREATE TABLE IF NOT EXISTS corpus_features (
+        mint text PRIMARY KEY, built_at timestamptz NOT NULL DEFAULT now(), rows int, trade_rows int, has_trades boolean, path text)""",
+    """CREATE TABLE IF NOT EXISTS replay_hours (
+        hour timestamptz PRIMARY KEY, status text NOT NULL DEFAULT 'done', trades int, events int, bytes bigint, took_s real,
+        last_error text, done_at timestamptz NOT NULL DEFAULT now())""",
+    """CREATE TABLE IF NOT EXISTS replay_days (
+        day date PRIMARY KEY, graduations int, tokens_written int, took_s real, assembled_at timestamptz NOT NULL DEFAULT now())""",
+    """CREATE TABLE IF NOT EXISTS corpus_meta (
+        mint text PRIMARY KEY, create_ts timestamptz, creator text, dev_sol double precision, dev_tokens double precision, dev_share real,
+        supply double precision, mayhem boolean, uri text, name text, symbol text, ttg_min double precision, rq0 double precision, pool_id text,
+        prior_launches int, prior_grads int, prior_known int, prior_rug_share real, prior_moon_share real,
+        own_dd60 real, own_max60 real, own_alive6h boolean, updated_at timestamptz NOT NULL DEFAULT now())""",
+    "CREATE INDEX IF NOT EXISTS corpus_meta_creator_idx ON corpus_meta (creator)",
+    "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS graduated_at timestamptz",
+    "CREATE INDEX IF NOT EXISTS corpus_meta_creator_g_idx ON corpus_meta (creator, graduated_at)",
+    """CREATE TABLE IF NOT EXISTS mature_days (
+        day date PRIMARY KEY, mints int, rows int, took_s real, built_at timestamptz NOT NULL DEFAULT now())""",
+    """CREATE TABLE IF NOT EXISTS pump_minutes (
+        mint text NOT NULL, ts timestamptz NOT NULL, pool_id text, open double precision, high double precision, low double precision, close double precision,
+        buy_sol double precision, sell_sol double precision, n_buys int, n_sells int, n_traders int, resq_sol double precision, fee_rate double precision,
+        PRIMARY KEY (mint, ts))""",
+    "CREATE INDEX IF NOT EXISTS pump_minutes_ts_idx ON pump_minutes (ts)",
+    """CREATE TABLE IF NOT EXISTS pump_events (
+        sig text PRIMARY KEY, ts timestamptz NOT NULL, action text NOT NULL, pool text, mint text, pool_id text, signer text, dev_sol double precision,
+        dev_tokens double precision, supply double precision, mayhem boolean, quote_in_pool double precision, name text, symbol text, uri text)""",
+    "CREATE INDEX IF NOT EXISTS pump_events_mint_idx ON pump_events (mint)",
+    "CREATE INDEX IF NOT EXISTS pump_events_creates_idx ON pump_events (signer, ts) WHERE action = 'create'",
+    # PumpSwap pools NOT created by a pump.fun migration (createPool by anyone): unburned liquidity, blocked from the universe
+    """CREATE TABLE IF NOT EXISTS pump_pools (
+        pool_id text PRIMARY KEY, mint text, created_by text, ts timestamptz, source text NOT NULL, added_at timestamptz NOT NULL DEFAULT now())""",
+    "CREATE INDEX IF NOT EXISTS pump_pools_mint_idx ON pump_pools (mint)",
+    """CREATE TABLE IF NOT EXISTS ui_settings (
+        key text PRIMARY KEY, value jsonb, updated_at timestamptz NOT NULL DEFAULT now())""",
+    # the plastic fly (agent/fly_session.py): every scored minute (pending rows are its synaptic tags; x is dropped once
+    # the label resolves), daily calibrations per arm (plastic / frozen), hourly learning statistics, rollbacks
+    """CREATE TABLE IF NOT EXISTS fly_scored (
+        ts timestamptz NOT NULL, mint text NOT NULL, x real[], score real, frozen_score real, line real, frozen_line real,
+        label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint,
+        PRIMARY KEY (ts, mint))""",
+    "CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)",
+    """CREATE TABLE IF NOT EXISTS fly_calibrations (
+        day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int,
+        created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""",
+    """CREATE TABLE IF NOT EXISTS fly_updates (
+        hour timestamptz PRIMARY KEY, n int, mean_delta real, mean_abs_delta real, step real, capped int, drift real, ic real, detail jsonb)""",
+    """CREATE TABLE IF NOT EXISTS fly_rollbacks (
+        id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), reason text, checks jsonb, from_snapshot bigint, to_snapshot bigint)""",
+    # per-book entry halts of the paper race books (agent/rails.check_book_drawdown); the live book uses circuit_state
+    """CREATE TABLE IF NOT EXISTS book_state (
+        book text PRIMARY KEY, halted boolean NOT NULL DEFAULT false, reason text, peak double precision,
+        updated_at timestamptz NOT NULL DEFAULT now())""",
+]
+
+# Version-gated migrations: {version: [statements]} applied when stored version < version.
+MIGRATIONS: dict[int, list[str]] = {
+    2: ["ALTER TABLE beat_slots ALTER COLUMN feature_mask TYPE bigint"],
+    3: ["ALTER TABLE positions ADD COLUMN IF NOT EXISTS satiety double precision NOT NULL DEFAULT 0"],
+    4: ["ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS fee_rate double precision"],       # the pool fee the stream reports as charged
+    # 6: the selector's strategy stack — per-minute wallet flow (train/flow.py), graduation-time curve facts and insiders
+    # (train/corpus_meta.py), per-position hold and strategy, the fly's per-strategy scored rows
+    6: ["ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS n_buyers int",
+        "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS wash_sol double precision",
+        "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS wash_buy_sol double precision",
+        "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS top_sell_sol double precision",
+        "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS insider_sell_sol double precision",
+        "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS skill_buy real[]",
+        "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS bundle_share real",
+        "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS dev_hold_share real",
+        "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS dev_sold_frac real",
+        "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS grad_hhi real",
+        "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS curve_known boolean",
+        "CREATE TABLE IF NOT EXISTS token_insiders (mint text NOT NULL, wallet text NOT NULL, kind text NOT NULL, PRIMARY KEY (mint, wallet))",
+        "ALTER TABLE positions ADD COLUMN IF NOT EXISTS hold_s double precision",
+        "ALTER TABLE positions ADD COLUMN IF NOT EXISTS strategy text",
+        "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS strategy text NOT NULL DEFAULT 'ev'",
+        "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int",
+        "ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey",
+        "ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"],
+}
+
+
+def ensure_database(url: str | None = None) -> bool:
+    """Create the database named in DATABASE_URL if it does not exist. Returns True if created."""
+    url = url or database_url()
+    name = database_name(url)
+    admin_url = psycopg.conninfo.make_conninfo(url, dbname="postgres")   # same user/host/password, URL or key=value DSN
+    with psycopg.connect(admin_url, autocommit=True) as conn:
+        row = conn.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,)).fetchone()
+        if row:
+            return False
+        conn.execute(f'CREATE DATABASE "{name}"')
+        log.info("created database %s", name)
+        return True
+
+
+def apply_schema(url: str | None = None) -> int:
+    """Apply the ladder under an advisory lock. Returns the resulting version."""
+    with connect(url) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,))
+            try:
+                for stmt in BASE_DDL:
+                    cur.execute(stmt)
+                cur.execute("SELECT version FROM schema_version WHERE singleton")
+                row = cur.fetchone()
+                current = int(row["version"]) if row else 0
+                for v in sorted(MIGRATIONS):
+                    if current < v:
+                        for stmt in MIGRATIONS[v]:
+                            cur.execute(stmt)
+                        current = v
+                target = max(current, SCHEMA_VERSION)
+                cur.execute(
+                    "INSERT INTO schema_version (singleton, version) VALUES (true, %s) "
+                    "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, updated_at = now()",
+                    (target,),
+                )
+                conn.commit()
+                ensure_partitions(conn)
+                conn.commit()
+                return target
+            except Exception:
+                conn.rollback()
+                raise
+            finally:
+                cur.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,))
+                conn.commit()
+
+
+def _partition_name(base: str, start: datetime, granularity: str) -> str:
+    fmt = "%Y%m%d%H" if granularity == "hour" else "%Y%m%d"
+    return f"{base}_{start.strftime(fmt)}"
+
+
+def ensure_partition(conn: psycopg.Connection, base: str, start: datetime, granularity: str) -> None:
+    start = start.astimezone(timezone.utc)
+    if granularity == "hour":
+        start = start.replace(minute=0, second=0, microsecond=0)
+        end = start + timedelta(hours=1)
+    else:
+        start = start.replace(hour=0, minute=0, second=0, microsecond=0)
+        end = start + timedelta(days=1)
+    name = _partition_name(base, start, granularity)
+    with conn.cursor() as cur:
+        cur.execute("SELECT 1 FROM pg_class WHERE relname = %s", (name,))
+        if cur.fetchone():
+            return
+        cur.execute(
+            sql.SQL("CREATE TABLE IF NOT EXISTS {} PARTITION OF {} FOR VALUES FROM ({}) TO ({})").format(
+                sql.Identifier(name), sql.Identifier(base), sql.Literal(start), sql.Literal(end)
+            )
+        )
+
+
+def ensure_partitions(conn: psycopg.Connection, now: datetime | None = None) -> None:
+    """Partitions for the near past and future so writers never hit a missing partition."""
+    now = now or datetime.now(timezone.utc)
+    for h in range(-2, 4):
+        ensure_partition(conn, "swap_tape", now + timedelta(hours=h), "hour")
+    for d in range(-1, 3):
+        ensure_partition(conn, "beat_slots", now + timedelta(days=d), "day")
+
+
+def ensure_partitions_for(conn: psycopg.Connection, ts: datetime) -> None:
+    """Partitions around an arbitrary (e.g. simulated) timestamp."""
+    ensure_partition(conn, "swap_tape", ts, "hour")
+    ensure_partition(conn, "swap_tape", ts + timedelta(hours=1), "hour")
+    ensure_partition(conn, "beat_slots", ts, "day")
+    ensure_partition(conn, "beat_slots", ts + timedelta(days=1), "day")
+
+
+def table_names(url: str | None = None) -> list[str]:
+    with connect(url) as conn:
+        rows = conn.execute(
+            "SELECT tablename FROM pg_tables WHERE schemaname = 'public' "
+            "AND tablename NOT LIKE 'swap_tape_%%' AND tablename NOT LIKE 'beat_slots_%%' ORDER BY 1"
+        ).fetchall()
+    return [r["tablename"] for r in rows]

fly_trader/execution/__init__.py

@@ -0,0 +1 @@
+

fly_trader/execution/broker_live.py

@@ -0,0 +1,455 @@
+"""Live execution through Jupiter Swap API v2 with balance-verified fills (plan §9).
+
+Per attempt (one in-flight order per process, guarded by a module lock):
+  assert_signing_allowed() -> pre = snapshot_balances() -> GET /order (taker = bot, excludeRouters=
+  jupiterz) -> assert signatureFeePayer == taker -> orders row (status 'ordered') -> sign only our
+  signer slot -> POST /execute -> orders row updated -> poll getSignatureStatuses every 1 s until
+  confirmed or getBlockHeight() > lastValidBlockHeight (no other timeout) -> post = snapshot ->
+  fill iff the input balance decreased AND the output balance increased (a confirmed sell whose token
+  balance fell is a fill even when the SOL delta is <= 0: dust proceeds below fees) -> fills row.
+
+Jupiter's execute status is recorded but never trusted alone: 'Failed' with moved balances is a
+fill (verified_by='balance_delta_despite_failed'); 'Success' without movement is a fills row with
+verified_by='unverified' plus an events row after a getTransaction check. On execute code -1000
+(failed to land) or a slippage error with NO balance movement the order is retried with
+slippage + step_bps up to max_slippage_bps; each attempt is its own orders row.
+
+orders.status words: ordered | order_error | sign_error | executed | execute_error | confirmed |
+failed_on_chain | expired | unsent.   fills.verified_by: balance_delta |
+balance_delta_despite_failed | unverified.   price_sol = |sol delta| / 1e9 / (|token delta| /
+10^decimals) — an all-in effective price (transaction fee, priority fee and any ATA rent are inside
+the SOL delta; fee_lamports carries the estimate so the ledger must not add it again).
+Every write is committed immediately so a crash mid-poll leaves an auditable row for verify-fills.
+"""
+from __future__ import annotations
+
+import base64
+import logging
+import re
+import threading
+import time
+from dataclasses import dataclass
+
+from psycopg.types.json import Jsonb
+from solders.hash import Hash
+from solders.instruction import AccountMeta, Instruction
+from solders.message import MessageV0
+from solders.pubkey import Pubkey
+from solders.transaction import VersionedTransaction
+
+from .. import config
+from ..chain.balances import Snapshot, compute_delta, snapshot_balances
+from ..chain.cluster_guard import assert_signing_allowed
+from ..chain.jupiter_swap import JupiterError, JupiterSwap
+from ..db.connection import transaction
+from ..chain.keys import load_keypair
+from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc
+from ..chain.signing import NotASigner, n_lookup_programs, sign_transaction_b64, static_program_ids, transaction_id
+from ..db.apilog import record_event
+from ..db.connection import connect, transaction
+from ..logging_setup import scrub
+
+log = logging.getLogger(__name__)
+
+_INFLIGHT = threading.Lock()  # one in-flight order at a time (plan §9)
+
+CONFIRM_POLL_S = 1.0
+BLOCKHASH_VALIDITY_BLOCKS = 150  # fallback horizon when /order carries no lastValidBlockHeight
+CLOSE_ATA_BATCH = 10
+SYSTEM_PROGRAM_ID = "11111111111111111111111111111111"
+COMPUTE_BUDGET_PROGRAM_ID = "ComputeBudget111111111111111111111111111111"
+ASSOCIATED_TOKEN_PROGRAM_ID = "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL"
+JUPITER_PROGRAM_IDS = ("JUP6LkbZbjS1jKKwapdHNy74zcZ3tLUZoi5QNyVTaV4",)
+KNOWN_PROGRAM_IDS = frozenset({SYSTEM_PROGRAM_ID, COMPUTE_BUDGET_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
+                               TOKEN_PROGRAM_ID, TOKEN_2022_PROGRAM_ID, *JUPITER_PROGRAM_IDS})
+RETRY_CODES = frozenset({-1000})  # aggregator: failed to land
+_SLIPPAGE_RE = re.compile(r"slippage|0x1771|\b6001\b", re.IGNORECASE)  # Jupiter SlippageToleranceExceeded = 6001
+
+
+@dataclass
+class FillResult:
+    ok: bool
+    order_id: int | None
+    fill_id: int | None
+    signature: str | None
+    status: str | None
+    code: int | None
+    error: str | None
+    token_delta: int
+    lamports_delta: int
+    price_sol: float | None
+    attempts: int
+
+
+def _int_or_none(v) -> int | None:
+    try:
+        return int(v) if v is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def await_confirmation(rpc, signature: str | None, last_valid_block_height: int | None) -> tuple[str, dict | None]:
+    """Poll until the signature is confirmed/finalized, fails on chain, or the chain's block height
+    passes ``last_valid_block_height`` with no status for the signature (it can then never land; a
+    signature already seen as processed keeps being polled until it confirms or its status disappears).
+    Sleeps 1 s between polls; there is deliberately no other timeout."""
+    if not signature:
+        return "unsent", None
+    if last_valid_block_height is None:
+        last_valid_block_height = rpc.get_block_height() + BLOCKHASH_VALIDITY_BLOCKS
+    while True:
+        st = (rpc.get_signature_statuses([signature]) or [None])[0]
+        if st:
+            if st.get("err") is not None:
+                return "failed_on_chain", st
+            if st.get("confirmationStatus") in ("confirmed", "finalized"):
+                return "confirmed", st
+        if not st and rpc.get_block_height() > last_valid_block_height:
+            return "expired", st
+        time.sleep(CONFIRM_POLL_S)
+
+
+class LiveBroker:
+    def __init__(self, rpc=None, jup=None, keypair=None):
+        self._rpc = rpc
+        self._jup = jup
+        self._keypair = keypair
+        self._labels: dict[str, str] | None = None
+
+    @property
+    def rpc(self):
+        if self._rpc is None:
+            self._rpc = HttpSolanaRpc()
+        return self._rpc
+
+    @property
+    def jup(self):
+        if self._jup is None:
+            self._jup = JupiterSwap()
+        return self._jup
+
+    @property
+    def keypair(self):
+        if self._keypair is None:
+            self._keypair = load_keypair()
+        return self._keypair
+
+    @property
+    def pubkey(self) -> str:
+        return str(self.keypair.pubkey())
+
+    @property
+    def labels(self) -> dict[str, str]:
+        if self._labels is None:
+            try:
+                self._labels = dict(self.jup.program_id_to_label())
+            except Exception as e:
+                log.warning("program-id-to-label unavailable: %s", type(e).__name__)
+                self._labels = {}
+        return self._labels
+
+    # ---- public ----
+    def swap(self, conn, *, decision_id: int | None, book: str, side: str, mint: str, amount_in: int,
+             slippage_bps: int, max_slippage_bps: int, step_bps: int) -> FillResult:
+        """Buy (lamports in -> ``mint``) or sell (raw token units in -> SOL) with balance verification."""
+        assert_signing_allowed()
+        if side not in ("buy", "sell"):
+            raise ValueError(f"side must be 'buy' or 'sell', got {side!r}")
+        amount_in = int(amount_in)
+        if amount_in <= 0:
+            raise ValueError("amount_in must be positive")
+        if side == "buy":
+            input_mint, output_mint = config.WSOL_MINT, mint
+        else:
+            input_mint, output_mint = mint, config.WSOL_MINT
+        with _INFLIGHT:
+            slip = int(slippage_bps)
+            attempt = 0
+            while True:
+                attempt += 1
+                result, retryable = self._attempt(
+                    conn, decision_id=decision_id, book=book, side=side, mint=mint, input_mint=input_mint,
+                    output_mint=output_mint, amount_in=amount_in, slippage_bps=slip, attempt=attempt)
+                result.attempts = attempt
+                if result.ok or not retryable or step_bps <= 0 or slip + step_bps > max_slippage_bps:
+                    if retryable and not result.ok:
+                        log.warning("order %s not retried: slippage ladder exhausted at %d bps", result.order_id, slip)
+                    return result
+                slip += int(step_bps)
+                log.warning("retrying %s %s after %s (code %s) with slippage %d bps",
+                            side, mint, result.status, result.code, slip)
+
+    # ---- one attempt ----
+    def _attempt(self, conn, *, decision_id, book, side, mint, input_mint, output_mint, amount_in,
+                 slippage_bps, attempt) -> tuple[FillResult, bool]:
+        taker = self.pubkey
+        pre = snapshot_balances(self.rpc, taker)
+        request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker,
+                   "slippageBps": slippage_bps, "excludeRouters": "jupiterz"}
+        base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint,
+                    output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request)
+
+        def fail(status, error, code=None, order_response=None, order_id=None):
+            if order_id is None:
+                order_id = self._insert_order(conn, **base, order_response=order_response, status=status,
+                                              error_code=code, error=error)
+            else:
+                self._update_order(conn, order_id, status=status, error_code=code, error=error)
+            log.error("%s %s attempt %d: %s: %s", side, mint, attempt, status, error)
+            return FillResult(False, order_id, None, None, status, code, error, 0, 0, None, attempt), False
+
+        # 1. order
+        try:
+            order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps)
+        except JupiterError as e:
+            return fail("order_error", scrub(str(e)))
+        err_code = _int_or_none(order.get("errorCode"))
+        tx_b64 = order.get("transaction")
+        if err_code is not None or not tx_b64:
+            msg = order.get("errorMessage") or order.get("error") or "no transaction in /order response"
+            return fail("order_error", scrub(str(msg)), err_code, order)
+        fee_payer = order.get("signatureFeePayer")
+        if fee_payer and fee_payer != taker:
+            return fail("order_error", f"signatureFeePayer {fee_payer} != taker", None, order)
+        lvbh = _int_or_none(order.get("lastValidBlockHeight"))
+        program_ids = static_program_ids(tx_b64)
+        unknown = [p for p in program_ids if p not in KNOWN_PROGRAM_IDS and p not in self.labels]
+        if unknown:
+            log.warning("order references unknown static program ids: %s", unknown)
+        n_lookup = n_lookup_programs(tx_b64)
+        if n_lookup:
+            log.warning("%d instruction(s) use program ids from lookup tables (unresolved in v1)", n_lookup)
+        order_id = self._insert_order(
+            conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"),
+            fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered")
+
+        # 2. sign (our slot only)
+        try:
+            signed_b64, _idx = sign_transaction_b64(tx_b64, self.keypair)
+        except (NotASigner, ValueError) as e:
+            return fail("sign_error", f"{type(e).__name__}: {scrub(str(e))}", None, order_id=order_id)
+        if _idx != 0 and not order.get("gasless"):
+            return fail("sign_error", f"our key is signer {_idx}, but a non-gasless order must have the taker as fee payer", None, order_id=order_id)
+        tx_sig = transaction_id(signed_b64)
+
+        # 3. execute
+        exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64}
+        ex: dict | None
+        try:
+            ex = self.jup.execute(signed_b64, order.get("requestId"), lvbh)
+        except JupiterError as e:
+            ex = None
+            code = None
+            error = scrub(str(e))
+            signature = tx_sig  # the transaction may still have been broadcast: poll it anyway
+            self._update_order(conn, order_id, execute_request=exec_req, execute_response=None, signature=signature,
+                               status="execute_error", error_code=None, error=error)
+        else:
+            code = _int_or_none(ex.get("code"))
+            error = ex.get("error")
+            signature = ex.get("signature") or tx_sig
+            if ex.get("signature") and ex["signature"] != tx_sig:
+                log.warning("execute signature %s differs from our transaction id %s", ex["signature"], tx_sig)
+            self._update_order(conn, order_id, execute_request=exec_req, execute_response=ex, signature=signature,
+                               status="executed", error_code=code, error=error, latency_ms=ex.get("_latency_ms"))
+
+        # 4. confirm (or expire), 5. verify by balance deltas — retried: a transport error here must not leave a landed swap unrecorded
+        chain_status = st = post = None; last_exc = None
+        for k in range(4):
+            try:
+                if chain_status is None:
+                    chain_status, st = await_confirmation(self.rpc, signature, lvbh)
+                    self._update_order(conn, order_id, status=chain_status)
+                post = snapshot_balances(self.rpc, taker); break
+            except Exception as e:                       # RPC transport / RPC errors
+                last_exc = e; log.warning("confirm/verify attempt %d for %s failed: %s", k + 1, signature, scrub(str(e)))
+                time.sleep(2.0 * (k + 1))
+        if post is None:
+            fill_id = self._insert_fill(conn, order_id, book, signature, None, mint, side, 0, 0, None, None, order.get("platformFee"), "unknown", pre, pre)
+            record_event("error", "broker_live", "fill unknown: could not confirm or verify after retries; entries paused until reconciled (verify-fills)",
+                         {"order_id": order_id, "signature": signature, "side": side, "mint": mint, "error": scrub(str(last_exc))[:200]})
+            try:                                            # a landed but unrecorded swap must not be bought again or leave a stale position
+                with transaction() as c2:
+                    c2.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = 1")
+                    c2.execute("INSERT INTO circuit_events (kind, detail) VALUES ('unknown_fill_pause', %s)", (Jsonb({"order_id": order_id, "signature": signature, "mint": mint, "side": side}),))
+            except Exception:
+                log.exception("could not pause entries after an unknown fill")
+            raise RuntimeError(f"confirm/verify failed for {signature}: {type(last_exc).__name__}") from last_exc
+        delta = compute_delta(pre, post)
+        lam = int(delta["lamports_delta"])
+        tok = int(delta["token_deltas"].get(mint, 0))
+        # a confirmed sell that moved the tokens is a fill even at lam <= 0 (a rugged token's dust proceeds < fees)
+        verified = (lam < 0 and tok > 0) if side == "buy" else (tok < 0 and (lam > 0 or chain_status == "confirmed"))
+        jup_status = (ex or {}).get("status")
+        slot = _int_or_none((ex or {}).get("slot")) or _int_or_none((st or {}).get("slot"))
+        if verified:
+            verified_by = "balance_delta" if jup_status == "Success" else "balance_delta_despite_failed"
+            decimals = self._decimals(conn, mint, pre, post)
+            sol = abs(lam) if side == "buy" else max(lam, 0)
+            price = (sol / config.LAMPORTS_PER_SOL) / (abs(tok) / 10 ** decimals) if decimals is not None else None
+            fee = self._fee_lamports(side, lam, amount_in, ex, order)
+            fill_id = self._insert_fill(conn, order_id, book, signature, slot, mint, side, tok, lam, price, fee,
+                                        order.get("platformFee"), verified_by, pre, post)
+            log.info("fill %s: %s %s token_delta=%d lamports_delta=%d price_sol=%s (%s)",
+                     fill_id, side, mint, tok, lam, price, verified_by)
+            return FillResult(True, order_id, fill_id, signature, chain_status, code, error, tok, lam, price, attempt), False
+        if jup_status == "Success" or chain_status == "confirmed":
+            tx_info = None
+            try:
+                tx_info = self.rpc.get_transaction(signature)
+            except Exception as e:
+                log.warning("getTransaction failed for %s: %s", signature, type(e).__name__)
+            meta = (tx_info or {}).get("meta") or {}
+            record_event("warning", "broker_live",
+                         "fill unverified: execute reported success but balances did not move as expected",
+                         {"order_id": order_id, "signature": signature, "side": side, "mint": mint,
+                          "lamports_delta": lam, "token_delta": tok, "chain_status": chain_status,
+                          "jupiter_status": jup_status, "tx_found": tx_info is not None,
+                          "tx_err": meta.get("err"), "tx_fee": meta.get("fee"), "tx_slot": (tx_info or {}).get("slot")})
+            fill_id = self._insert_fill(conn, order_id, book, signature, slot, mint, side, tok, lam, None, None,
+                                        order.get("platformFee"), "unverified", pre, post)
+            return FillResult(False, order_id, fill_id, signature, chain_status, code, error, tok, lam, None, attempt), False
+        retryable = (code in RETRY_CODES) or bool(error and _SLIPPAGE_RE.search(str(error)))
+        log.error("%s %s attempt %d not filled: chain=%s jupiter=%s code=%s error=%s lamports_delta=%d token_delta=%d",
+                  side, mint, attempt, chain_status, jup_status, code, error, lam, tok)
+        return FillResult(False, order_id, None, signature, chain_status, code, error, tok, lam, None, attempt), retryable
+
+    # ---- helpers ----
+    def _decimals(self, conn, mint: str, pre: Snapshot, post: Snapshot) -> int | None:
+        d = post.decimals.get(mint, pre.decimals.get(mint))
+        if d is not None:
+            return int(d)
+        try:
+            row = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone()
+            if row and row["decimals"] is not None:
+                return int(row["decimals"])
+        except Exception:
+            conn.rollback()
+        try:
+            return self.rpc.get_mint_decimals(mint)
+        except Exception:
+            return None
+
+    @staticmethod
+    def _fee_lamports(side: str, lam: int, amount_in: int, ex: dict | None, order: dict) -> int | None:
+        """Buy: lamports beyond the swap input (tx fee + priority fee + ATA rent). Sell: SOL the route
+        produced that did not reach the wallet. Approximations from execute amounts, else the order's
+        fee fields."""
+        ex = ex or {}
+        if side == "buy":
+            total_in = _int_or_none(ex.get("totalInputAmount")) or amount_in
+            return abs(lam) - total_in
+        total_out = _int_or_none(ex.get("totalOutputAmount"))
+        if total_out is not None:
+            return total_out - lam
+        sig_fee = _int_or_none(order.get("signatureFeeLamports")) or 0
+        prio = _int_or_none(order.get("prioritizationFeeLamports")) or 0
+        return sig_fee + prio
+
+    def _insert_order(self, conn, *, decision_id, book, attempt, side, input_mint, output_mint, amount_in,
+                      slippage_bps, request, order_response=None, request_id=None, router=None, fee_bps=None,
+                      program_ids=None, lvbh=None, status, error_code=None, error=None) -> int:
+        row = conn.execute(
+            "INSERT INTO orders (decision_id, book, attempt, side, input_mint, output_mint, amount_in, slippage_bps, "
+            "request, order_response, request_id, router, fee_bps, program_ids, last_valid_block_height, status, "
+            "error_code, error) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id",
+            (decision_id, book, attempt, side, input_mint, output_mint, amount_in, slippage_bps, Jsonb(request),
+             Jsonb(order_response) if order_response is not None else None, request_id, router, fee_bps,
+             list(program_ids) if program_ids is not None else None, lvbh, status, error_code, error),
+        ).fetchone()
+        conn.commit()
+        return int(row["id"])
+
+    def _update_order(self, conn, order_id: int, **fields) -> None:
+        json_cols = {"execute_request", "execute_response", "order_response"}
+        cols, vals = [], []
+        for k, v in fields.items():
+            cols.append(f"{k} = %s")
+            vals.append(Jsonb(v) if k in json_cols and v is not None else v)
+        conn.execute(f"UPDATE orders SET {', '.join(cols)} WHERE id = %s", (*vals, order_id))
+        conn.commit()
+
+    def _insert_fill(self, conn, order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports,
+                     price_sol, fee_lamports, platform_fee, verified_by, pre: Snapshot, post: Snapshot) -> int:
+        row = conn.execute(
+            "INSERT INTO fills (order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports, price_sol, "
+            "fee_lamports, platform_fee, verified_by, pre_snapshot, post_snapshot) "
+            "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id",
+            (order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports, price_sol, fee_lamports,
+             Jsonb(platform_fee) if platform_fee is not None else None, verified_by,
+             Jsonb(pre.to_json()), Jsonb(post.to_json())),
+        ).fetchone()
+        conn.commit()
+        return int(row["id"])
+
+
+def swap_smoke(sol: float) -> None:
+    """Operator smoke test (plan phase 3.12): SOL -> USDC then USDC -> SOL through the live broker.
+    Requires the full live gate (LIVE_ENABLED=1 etc.). Prints both FillResults and the net lamports."""
+    assert_signing_allowed()
+    lamports = int(round(float(sol) * config.LAMPORTS_PER_SOL))
+    broker = LiveBroker()
+    print(f"swap-smoke: wallet {broker.pubkey}: {sol} SOL -> USDC -> SOL")
+    with connect() as conn:
+        r1 = broker.swap(conn, decision_id=None, book="live", side="buy", mint=config.USDC_MINT, amount_in=lamports,
+                         slippage_bps=config.SLIPPAGE_ENTRY_BPS, max_slippage_bps=config.MAX_SLIPPAGE_ENTRY_BPS,
+                         step_bps=config.SLIPPAGE_STEP_BPS)
+        print(f"leg 1 (buy USDC): {r1}")
+        net = r1.lamports_delta
+        if r1.ok and r1.token_delta > 0:
+            r2 = broker.swap(conn, decision_id=None, book="live", side="sell", mint=config.USDC_MINT,
+                             amount_in=r1.token_delta, slippage_bps=config.SLIPPAGE_EXIT_BPS,
+                             max_slippage_bps=config.SLIPPAGE_FORCED_BPS, step_bps=config.SLIPPAGE_STEP_BPS)
+            print(f"leg 2 (sell USDC): {r2}")
+            net += r2.lamports_delta
+        else:
+            print("leg 2 skipped: leg 1 was not a verified fill")
+    print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)")
+
+
+def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH) -> dict:
+    """Close every zero-balance token account owned by the bot (rent back to the wallet).
+    SPL Token / Token-2022 CloseAccount = instruction index 9, accounts [account, destination, owner].
+    One v0 transaction per ``batch`` accounts; each is recorded as wallet_events(kind='ata_closed')."""
+    assert_signing_allowed()
+    kp = keypair or load_keypair()
+    owner = kp.pubkey()
+    owner_s = str(owner)
+    rpc = rpc or HttpSolanaRpc()
+    empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0]
+    result = {"pubkey": owner_s, "candidates": len(empties), "closed": 0, "signatures": [], "failed": []}
+    if not empties:
+        return result
+    for i in range(0, len(empties), batch):
+        chunk = empties[i:i + batch]
+        accounts = [a["address"] for a in chunk]
+        ixs = [Instruction(Pubkey.from_string(a["program"]), bytes([9]),
+                           [AccountMeta(Pubkey.from_string(a["address"]), False, True),
+                            AccountMeta(owner, False, True),
+                            AccountMeta(owner, True, False)]) for a in chunk]
+        bh = rpc.get_latest_blockhash()
+        msg = MessageV0.try_compile(owner, ixs, [], Hash.from_string(bh["blockhash"]))
+        tx = VersionedTransaction(msg, [kp])
+        tx_b64 = base64.b64encode(bytes(tx)).decode()
+        try:
+            sig = rpc.send_transaction(tx_b64)
+        except Exception as e:
+            err = f"{type(e).__name__}: {scrub(str(e))}"
+            result["failed"].append({"accounts": accounts, "error": err})
+            record_event("error", "close_empty_atas", "close transaction rejected", {"accounts": accounts, "error": err})
+            continue
+        status, _st = await_confirmation(rpc, sig, bh["last_valid_block_height"])
+        detail = {"accounts": accounts, "signature": sig, "status": status}
+        try:
+            with transaction(url) as conn:
+                conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('ata_closed', %s, %s)",
+                             (owner_s, Jsonb(detail)))
+        except Exception as e:
+            log.warning("wallet_events insert failed: %s", type(e).__name__)
+        if status == "confirmed":
+            result["closed"] += len(chunk)
+            result["signatures"].append(sig)
+        else:
+            result["failed"].append(detail)
+    log.info("close_empty_atas: closed %d of %d (%d tx)", result["closed"], len(empties), len(result["signatures"]))
+    return result

fly_trader/execution/broker_paper.py

@@ -0,0 +1,97 @@
+"""PaperBroker: the same decisions executed at model prices.
+
+Fill price = last tape price; cost = the real fees (pool fee by market cap, Jupiter 10 bps, network fee) + constant-product impact
+of the position size against the pool's quote reserve (market/exit_cost.py). Deviation from the plan's
+"first tape price ≥ 2 s after the decision" rule: fills are immediate at the last price; the
+live-vs-mirror gap measures the difference.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from .. import config
+from ..market.exit_cost import exit_cost_fraction, fee_fraction, impact_fraction
+from . import ledger
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class PaperFill:
+    ok: bool
+    position_id: int | None
+    fill_id: int | None
+    price: float
+    sol_delta: float
+    token_delta: int
+    fee_sol: float
+    reason: str = ""
+
+
+class PaperBroker:
+    def __init__(self, book: str):
+        assert book in ("paper_selector", "paper_fly") or book.startswith("replay_"), book   # the race books; replay_* for tests
+        self.book = book
+
+    def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None,
+            res_quote_sol: float | None, mcap_sol: float | None, decimals: int, program_label: str | None, pool_fee: float | None = None,
+            ts: datetime | None = None, hold_s: float | None = None, strategy: str | None = None) -> PaperFill:
+        """Fill at ``price`` pushed up by the constant-product impact, paying the real fees (market/exit_cost.py): the pool
+        fee the stream reports (``pool_fee``) or the schedule's rate for ``mcap_sol``, Jupiter's platform fee, the network fee."""
+        ts = ts or datetime.now(timezone.utc)
+        if not price or price <= 0:
+            return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price")
+        if res_quote_sol is None or res_quote_sol <= 0:
+            return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate")
+        fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee)
+        impact = impact_fraction(size_sol, res_quote_sol, program_label)
+        if impact > config.EXECUTABILITY_MAX_IMPACT:
+            # a router would not fill this either; refuse like the live executability check
+            return PaperFill(False, None, None, price, 0.0, 0, 0.0, f"impact {impact:.2f} > {config.EXECUTABILITY_MAX_IMPACT}")
+        eff_price = price * (1.0 + impact)  # buying pushes the price up against us
+        tokens = size_sol * (1.0 - fee_frac) / eff_price
+        qty_raw = int(tokens * (10 ** decimals))
+        if qty_raw <= 0:
+            return PaperFill(False, None, None, price, 0.0, 0, 0.0, "zero quantity")
+        fee_sol = size_sol * fee_frac
+        pid = ledger.open_position(conn, book=self.book, mint=mint, pool=pool, qty_raw=qty_raw, cost_sol=size_sol,
+                                   entry_price=eff_price, decision_id=decision_id, fees_sol=fee_sol, ts=ts, hold_s=hold_s, strategy=strategy)
+        fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=mint, side="buy", token_delta=qty_raw,
+                                 sol_delta_lamports=-int(size_sol * config.LAMPORTS_PER_SOL), price_sol=eff_price,
+                                 fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts)
+        return PaperFill(True, pid, fid, eff_price, -size_sol, qty_raw, fee_sol)
+
+    def sell(self, conn, *, position: dict, decision_id: int | None, price: float | None, res_quote_sol: float | None,
+             mcap_sol: float | None, program_label: str | None, forced_kind: str | None, pool_fee: float | None = None,
+             ts: datetime | None = None, fraction: float = 1.0) -> PaperFill:
+        ts = ts or datetime.now(timezone.utc)
+        price = price or float(position.get("last_mark_price") or position.get("entry_price") or 0.0)
+        fraction = float(min(1.0, max(0.0, fraction)))
+        qty_raw = int(int(position["qty"]) * fraction) if fraction < 0.999 else int(position["qty"])
+        decimals = int(position.get("decimals") or 6)
+        gross = qty_raw / (10 ** decimals) * price
+        c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee)
+        proceeds = gross * (1.0 - c)
+        fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee)
+        if fraction < 0.999:   # partial: shrink the position, realise the sold part
+            realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw,
+                                              cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds,
+                                              fees_sol=fee_sol, price=price, ts=ts)
+            if realized is None:
+                return PaperFill(False, int(position["id"]), None, price, 0.0, 0, 0.0, "position not open")
+            fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=position["mint"], side="sell", token_delta=-qty_raw,
+                                     sol_delta_lamports=int(proceeds * config.LAMPORTS_PER_SOL), price_sol=price * (1.0 - c),
+                                     fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts)
+            return PaperFill(True, int(position["id"]), fid, price * (1.0 - c), proceeds, -qty_raw, fee_sol, reason=f"partial {fraction:.2f} realized {realized:+.5f} SOL")
+        realized = ledger.close_position(conn, position_id=int(position["id"]), exit_price=price * (1.0 - c),
+                                         proceeds_sol=proceeds, fees_sol=fee_sol, decision_id=decision_id,
+                                         forced_kind=forced_kind, ts=ts)
+        if realized is None:
+            return PaperFill(False, int(position["id"]), None, price, 0.0, 0, 0.0, "position not open")
+        fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=position["mint"], side="sell", token_delta=-qty_raw,
+                                 sol_delta_lamports=int(proceeds * config.LAMPORTS_PER_SOL), price_sol=price * (1.0 - c),
+                                 fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts)
+        return PaperFill(True, int(position["id"]), fid, price * (1.0 - c), proceeds, -qty_raw, fee_sol,
+                         reason=f"realized {realized:+.5f} SOL")

fly_trader/execution/ledger.py

@@ -0,0 +1,147 @@
+"""Positions and fills bookkeeping per book (live, paper_selector).
+
+Paper cash is derived, never stored: cash = CAPITAL_SOL + Σ realized_sol(all) − Σ cost_sol(open).
+realized_sol accumulates per position: each partial sale books proceeds − the cost it retires (cost_sol
+shrinks by that part) and the close adds proceeds − the remaining cost, so an open position may carry
+realized P&L from partial sales.
+Live cash is the on-chain SOL balance. Every open/close writes positions rows; every fill writes a
+fills row (verified_by = 'balance_delta' for live, 'model' for paper).
+"""
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+
+from .. import config
+
+log = logging.getLogger(__name__)
+
+
+def open_positions(conn, book: str) -> list[dict]:
+    rows = conn.execute(
+        """SELECT p.*, t.decimals, t.graduated_at, wp.program_label,
+                  COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy
+           FROM positions p LEFT JOIN tokens t ON t.mint = p.mint
+           LEFT JOIN watch_pools wp ON wp.pool = p.pool
+           LEFT JOIN decisions d ON d.id = p.entry_decision_id
+           WHERE p.book = %s AND p.status = 'open' ORDER BY p.opened_at""",
+        (book,),
+    ).fetchall()
+    return [dict(r) for r in rows]
+
+
+def paper_cash(conn, book: str) -> float:
+    r = conn.execute(
+        """SELECT COALESCE(sum(realized_sol), 0) AS realized,
+                  COALESCE(sum(cost_sol) FILTER (WHERE status = 'open'), 0) AS open_cost
+           FROM positions WHERE book = %s""",
+        (book,),
+    ).fetchone()
+    return config.CAPITAL_SOL + float(r["realized"]) - float(r["open_cost"])
+
+
+def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int,
+                sol_delta_lamports: int, price_sol: float, fee_lamports: int, verified_by: str,
+                signature: str | None = None, slot: int | None = None, ts: datetime | None = None,
+                pre: dict | None = None, post: dict | None = None) -> int:
+    row = conn.execute(
+        """INSERT INTO fills (order_id, book, ts, signature, slot, mint, side, token_delta, sol_delta_lamports,
+             price_sol, fee_lamports, verified_by, pre_snapshot, post_snapshot)
+           VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
+        (order_id, book, ts or datetime.now(timezone.utc), signature, slot, mint, side, token_delta, sol_delta_lamports,
+         price_sol, fee_lamports, verified_by, json.dumps(pre) if pre else None, json.dumps(post) if post else None),
+    ).fetchone()
+    return int(row["id"])
+
+
+def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float,
+                  entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None,
+                  hold_s: float | None = None, strategy: str | None = None) -> int:
+    ts = ts or datetime.now(timezone.utc)
+    existing = conn.execute("SELECT id, qty, cost_sol, entry_price FROM positions WHERE book=%s AND mint=%s AND status='open'",
+                            (book, mint)).fetchone()
+    if existing:  # add to an open position: average the entry
+        q0, c0 = int(existing["qty"]), float(existing["cost_sol"])
+        q1, c1 = q0 + qty_raw, c0 + cost_sol
+        p1 = (float(existing["entry_price"] or entry_price) * q0 + entry_price * qty_raw) / max(q1, 1)
+        conn.execute("UPDATE positions SET qty=%s, cost_sol=%s, entry_price=%s, fees_sol=fees_sol+%s, last_mark_price=%s, last_mark_ts=%s WHERE id=%s",
+                     (q1, c1, p1, fees_sol, entry_price, ts, existing["id"]))
+        return int(existing["id"])
+    row = conn.execute(
+        """INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price,
+             last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy)
+           VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s) RETURNING id""",
+        (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy),
+    ).fetchone()
+    return int(row["id"])
+
+
+def realize_partial(conn, *, position_id: int, qty_raw: int, cost_part: float, proceeds_sol: float, fees_sol: float,
+                    price: float, ts: datetime | None = None) -> float | None:
+    """Shrink an open position by qty_raw / cost_part and book proceeds − cost_part into realized_sol.
+    Returns this sale's realized P&L, or None when the position is no longer open (nothing booked)."""
+    ts = ts or datetime.now(timezone.utc)
+    realized = proceeds_sol - cost_part
+    r = conn.execute(
+        """UPDATE positions SET qty = qty - %s, cost_sol = cost_sol - %s, realized_sol = COALESCE(realized_sol, 0) + %s,
+             fees_sol = fees_sol + %s, last_mark_price=%s, last_mark_ts=%s WHERE id=%s AND status='open' RETURNING id""",
+        (qty_raw, cost_part, realized, fees_sol, price, ts, position_id)).fetchone()
+    return realized if r else None
+
+
+def close_position(conn, *, position_id: int, exit_price: float, proceeds_sol: float, fees_sol: float,
+                   decision_id: int | None, forced_kind: str | None, ts: datetime | None = None) -> float | None:
+    """Close an open position; realized_sol accumulates (earlier partial sales + proceeds − remaining cost).
+    Returns this sale's realized P&L, or None when the position was not open (already closed: nothing booked)."""
+    ts = ts or datetime.now(timezone.utc)
+    r = conn.execute(
+        """UPDATE positions SET status='closed', closed_at=%s, exit_decision_id=%s, exit_price=%s,
+             realized_sol=COALESCE(realized_sol, 0) + %s - cost_sol, fees_sol=fees_sol+%s, forced_exit_kind=%s,
+             last_mark_price=%s, last_mark_ts=%s WHERE id=%s AND status='open' RETURNING %s - cost_sol AS realized""",
+        (ts, decision_id, exit_price, proceeds_sol, fees_sol, forced_kind, exit_price, ts, position_id, proceeds_sol),
+    ).fetchone()
+    if r is None:
+        log.warning("close_position: position %s is not open; nothing booked", position_id)
+        return None
+    return float(r["realized"])
+
+
+def mark_positions(conn, book: str, prices: dict[str, float], last_swaps: dict[str, float], ts: datetime,
+                   dt_s: float = 0.0) -> None:
+    """Mark to the last price; track the peak; accrue satiety = Σ max(unrealized return, 0) × dt while in profit."""
+    rows = conn.execute("SELECT id, mint, peak_price, entry_price, satiety FROM positions WHERE book=%s AND status='open'", (book,)).fetchall()
+    for r in rows:
+        p = prices.get(r["mint"])
+        if p is None:
+            continue
+        peak = max(float(r["peak_price"] or 0.0), p)
+        entry = float(r["entry_price"] or 0.0)
+        u = (p / entry - 1.0) if entry > 0 else 0.0
+        sat = float(r["satiety"] or 0.0) + max(u, 0.0) * dt_s
+        ls = last_swaps.get(r["mint"])
+        conn.execute("UPDATE positions SET last_mark_price=%s, last_mark_ts=%s, peak_price=%s, satiety=%s, "
+                     "last_swap_ts=COALESCE(%s, last_swap_ts) WHERE id=%s",
+                     (p, ts, peak, sat, datetime.fromtimestamp(ls, tz=timezone.utc) if ls else None, r["id"]))
+
+
+def verify_fills() -> None:
+    """Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events."""
+    from ..chain import keys
+    from ..chain.rpc import HttpSolanaRpc
+    from ..db.connection import transaction
+    rpc = HttpSolanaRpc()
+    pub = keys.bot_pubkey()
+    accounts = rpc.get_token_accounts_by_owner(pub)
+    onchain = {a["mint"]: int(a["amount"]) for a in accounts}
+    with transaction() as conn:
+        pos = open_positions(conn, "live")
+        mismatches = []
+        for p in pos:
+            have = onchain.get(p["mint"], 0)
+            if abs(have - int(p["qty"])) > max(1, int(p["qty"]) // 1000):
+                mismatches.append({"mint": p["mint"], "db_qty": int(p["qty"]), "onchain": have})
+        stray = [m for m, a in onchain.items() if a > 0 and m not in {p["mint"] for p in pos} and m != config.WSOL_MINT]
+        conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('verify_fills', %s, %s)",
+                     (pub, json.dumps({"mismatches": mismatches, "stray_mints": stray, "n_positions": len(pos)})))
+    print(json.dumps({"positions": len(pos), "mismatches": mismatches, "stray_mints": stray}, indent=1))

fly_trader/execution/worker.py

@@ -0,0 +1,143 @@
+"""Execution queue for the live book: the caller never blocks on Jupiter.
+
+The caller enqueues ExecRequest objects; one background thread executes them sequentially through
+LiveBroker (one in-flight order at a time, balance-verified); results are drained by the caller, which
+opens/closes positions accordingly. Mints with an in-flight request are skipped.
+"""
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import traceback
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from .. import config
+from ..db.connection import transaction
+from ..agent import rails
+from . import ledger
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class ExecRequest:
+    decision_id: int | None
+    mint: str
+    pool: str | None
+    side: str                     # buy | sell
+    amount_in: int                # lamports (buy) or raw token units (sell)
+    slippage_bps: int
+    max_slippage_bps: int
+    decimals: int
+    forced_kind: str | None = None
+    position_id: int | None = None
+    size_sol: float = 0.0
+    enqueued_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+@dataclass
+class ExecResult:
+    request: ExecRequest
+    ok: bool
+    fill_id: int | None = None
+    signature: str | None = None
+    token_delta: int = 0
+    lamports_delta: int = 0
+    price_sol: float | None = None
+    error: str | None = None
+    position_id: int | None = None
+    realized_sol: float | None = None
+
+
+class ExecutionWorker:
+    def __init__(self, broker):
+        self.broker = broker
+        self.q: queue.Queue[ExecRequest] = queue.Queue()
+        self.results: queue.Queue[ExecResult] = queue.Queue()
+        self.inflight: set[str] = set()
+        self.lock = threading.Lock()
+        self.thread = threading.Thread(target=self._loop, name="runner-exec", daemon=True)   # routed to the runner's log
+        self.stop_flag = False
+        self.thread.start()
+
+    def submit(self, req: ExecRequest) -> bool:
+        with self.lock:
+            if req.mint in self.inflight:
+                return False
+            self.inflight.add(req.mint)
+        self.q.put(req)
+        return True
+
+    def pending(self) -> set[str]:
+        with self.lock:
+            return set(self.inflight)
+
+    def drain(self) -> list[ExecResult]:
+        out = []
+        while True:
+            try:
+                out.append(self.results.get_nowait())
+            except queue.Empty:
+                break
+        return out
+
+    def stop(self) -> None:
+        self.stop_flag = True
+
+    def _loop(self) -> None:
+        while not self.stop_flag:
+            try:
+                req = self.q.get(timeout=1.0)
+            except queue.Empty:
+                continue
+            res = self._execute(req)
+            with self.lock:
+                self.inflight.discard(req.mint)
+            self.results.put(res)
+
+    def _execute(self, req: ExecRequest) -> ExecResult:
+        try:
+            with transaction() as conn:
+                fr = self.broker.swap(conn, decision_id=req.decision_id, book="live", side=req.side, mint=req.mint,
+                                      amount_in=req.amount_in, slippage_bps=req.slippage_bps,
+                                      max_slippage_bps=req.max_slippage_bps, step_bps=config.SLIPPAGE_STEP_BPS)
+                if not fr.ok:
+                    rails.record_failure(conn, fr.error or fr.status or "swap failed")
+                    return ExecResult(req, False, error=fr.error or fr.status)
+                rails.record_success(conn)
+                ts = datetime.now(timezone.utc)
+                if req.side == "buy":
+                    cost = -fr.lamports_delta / config.LAMPORTS_PER_SOL
+                    price = fr.price_sol or (cost / max(fr.token_delta / (10 ** req.decimals), 1e-12))
+                    pid = ledger.open_position(conn, book="live", mint=req.mint, pool=req.pool, qty_raw=fr.token_delta,
+                                               cost_sol=cost, entry_price=price, decision_id=req.decision_id,
+                                               fees_sol=0.0, ts=ts)
+                    rails.record_notional(conn, cost, "entry")
+                    return ExecResult(req, True, fr.fill_id, fr.signature, fr.token_delta, fr.lamports_delta, price, position_id=pid)
+                else:
+                    proceeds = fr.lamports_delta / config.LAMPORTS_PER_SOL
+                    pos = conn.execute("SELECT * FROM positions WHERE id=%s FOR UPDATE", (req.position_id,)).fetchone()
+                    price = fr.price_sol if fr.price_sol is not None else (proceeds / max(abs(fr.token_delta) / (10 ** req.decimals), 1e-12))
+                    remaining = int(pos["qty"]) + fr.token_delta if pos else 0  # token_delta negative
+                    if pos is None or pos["status"] != "open":   # closed elsewhere: the fill row stands, nothing is booked twice
+                        log.warning("sell of %s filled but position %s is not open; not booked", req.mint, req.position_id)
+                        realized = None
+                    elif remaining > max(1, int(pos["qty"]) // 200):  # partial fill: keep the remainder open
+                        realized = ledger.realize_partial(conn, position_id=req.position_id, qty_raw=-fr.token_delta,
+                                                          cost_part=float(pos["cost_sol"]) * (1 - remaining / max(int(pos["qty"]), 1)),
+                                                          proceeds_sol=proceeds, fees_sol=0.0, price=price, ts=ts)
+                    else:
+                        realized = ledger.close_position(conn, position_id=req.position_id, exit_price=price, proceeds_sol=proceeds,
+                                                         fees_sol=0.0, decision_id=req.decision_id, forced_kind=req.forced_kind, ts=ts)
+                    return ExecResult(req, True, fr.fill_id, fr.signature, fr.token_delta, fr.lamports_delta, price,
+                                      position_id=req.position_id, realized_sol=realized)
+        except Exception as e:
+            log.error("execution error for %s: %s\n%s", req.mint, e, traceback.format_exc())
+            try:
+                with transaction() as conn:
+                    rails.record_failure(conn, f"{type(e).__name__}: {e}")
+            except Exception:
+                pass
+            return ExecResult(req, False, error=f"{type(e).__name__}: {e}")

fly_trader/ingest/__init__.py

@@ -0,0 +1 @@
+

fly_trader/ingest/corpus_pull.py

@@ -0,0 +1,439 @@
+"""Historical corpus puller (Supervisor thread ``corpus``).
+
+Enumerates every pump.fun graduation by paging the migration authority's succeeded transactions through
+Helius ``getTransactionsForAddress`` (newest first, back to ``CORPUS_DAYS``), then, per token and newest
+first, pulls from pump.fun's own swap-api:
+
+* candles in SOL: 1-minute for the first ``CORPUS_CANDLE_1M_H`` hours after graduation (plus the last 30
+  minutes of the bonding curve), then 5-minute until ``CORPUS_CANDLE_5M_D`` days (2–4 requests per token);
+* tokens are pulled only once they are ``CORPUS_MIN_AGE_H`` old, so the first hours are complete;
+* per-trade rows (wallet, side, SOL, tokens, price) for the first ``CORPUS_TRADES_H`` hours, for a deterministic
+  ``CORPUS_TRADES_SAMPLE`` share of the tokens that kept trading at least ``CORPUS_MIN_LIFE_H`` and graduated within
+  ``CORPUS_TRADES_DAYS`` (trades cost up to 100 pages per token; candles cost one to three).
+
+Registry: ``corpus_tokens``. Data: ``data/corpus/candles/<mint>.parquet`` and ``data/corpus/trades/<mint>.parquet``
+(append-only: a token already on disk is never rewritten). Progress: ``ui_settings['corpus_status']``.
+The swap-api throttles bursts (Cloudflare 1015 after ~20 requests at 1/s), so requests are paced adaptively.
+Measured 2026-09-13: ~36 graduations/hour; candle volume in SOL matches the tape within 0.5 %.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import httpx
+import psycopg
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from .. import config
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..logging_setup import setup
+
+log = logging.getLogger(__name__)
+WSOL = "So11111111111111111111111111111111111111112"
+PUMP_PROGRAM = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P"
+SLOTS_PER_S = 2.5
+RETRY_ERROR_H = 6.0       # a token that failed transiently (retries exhausted on 5xx/429, network, disk) is pulled again after this
+CANDLE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("interval", pa.string()), ("open", pa.float64()),
+                           ("high", pa.float64()), ("low", pa.float64()), ("close", pa.float64()), ("volume_sol", pa.float64())])
+TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot_index", pa.string()), ("tx", pa.string()), ("wallet", pa.string()),
+                          ("side", pa.int8()), ("program", pa.string()), ("price_sol", pa.float64()), ("sol", pa.float64()), ("tokens", pa.float64())])
+
+
+class Throttled(Exception):
+    pass
+
+
+class Permanent(RuntimeError):
+    """A 4xx from the swap-api: the token's error row is never requeued."""
+
+
+class Status:
+    """ui_settings['corpus_status'] writer (throttled to one write per 3 s unless forced)."""
+
+    def __init__(self):
+        self.s: dict = {"started_at": datetime.now(timezone.utc).isoformat(), "req_ok": 0, "req_429": 0, "candles_rows": 0, "trades_rows": 0,
+                        "tokens_pulled": 0, "with_trades": 0}
+        self.lock = threading.Lock(); self.last = 0.0; self.t0 = time.time()
+
+    def update(self, force: bool = False, **kv) -> None:
+        with self.lock:
+            self.s.update(kv); self.s["updated_at"] = datetime.now(timezone.utc).isoformat()
+            hours = (time.time() - self.t0) / 3600
+            self.s["tokens_per_h"] = self.s["tokens_pulled"] / hours if hours > 0.01 else 0.0
+            if not force and time.time() - self.last < 3.0:
+                return
+            self.last = time.time(); payload = dict(self.s)
+        try:
+            with transaction() as conn:
+                conn.execute("INSERT INTO ui_settings (key, value) VALUES ('corpus_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                             (json.dumps(payload, default=str),))
+        except Exception:
+            log.debug("corpus status write failed", exc_info=True)
+
+    def bump(self, key: str, n: int = 1) -> None:
+        with self.lock:
+            self.s[key] = self.s.get(key, 0) + n
+
+
+class Pacer:
+    """Adaptive spacing between swap-api requests: back off 0.5 s per throttle, creep back after 200 clean calls."""
+
+    def __init__(self, pace_s: float, floor_s: float):
+        self.pace = pace_s; self.floor = floor_s; self.last = 0.0; self.clean = 0
+
+    def wait(self, stop: threading.Event | None) -> None:
+        dt = time.time() - self.last
+        if dt < self.pace:
+            _sleep(self.pace - dt, stop)
+        self.last = time.time()
+
+    def ok(self) -> None:
+        self.clean += 1
+        if self.clean >= 200:
+            self.clean = 0; self.pace = max(self.floor, self.pace - 0.1)
+
+    def throttled(self, stop: threading.Event | None, retry_after: float | None) -> None:
+        self.clean = 0; self.pace = min(10.0, self.pace + 0.5)
+        _sleep(max(60.0, retry_after or 0.0), stop)
+
+
+def _sleep(seconds: float, stop: threading.Event | None) -> None:
+    end = time.time() + seconds
+    while time.time() < end:
+        if stop is not None and stop.is_set():
+            return
+        time.sleep(min(1.0, end - time.time()))
+
+
+def _db_retry(stop: threading.Event | None, what: str, fn, *args):
+    """``fn(*args)``, riding out Postgres outages (log, back off 5 s → 60 s, retry); None once ``stop`` is set."""
+    delay = 5.0
+    while not (stop is not None and stop.is_set()):
+        try:
+            return fn(*args)
+        except psycopg.Error as e:
+            log.warning("%s failed (%s: %s); retrying in %.0fs", what, type(e).__name__, str(e)[:160], delay)
+            _sleep(delay, stop); delay = min(60.0, delay * 2)
+    return None
+
+
+class SwapApi:
+    def __init__(self, pacer: Pacer, status: Status, stop: threading.Event | None):
+        self.c = httpx.Client(base_url=config.CORPUS_SWAP_API, headers={"User-Agent": "Mozilla/5.0", "Accept": "application/json"}, timeout=30)
+        self.pacer, self.status, self.stop = pacer, status, stop
+
+    def get(self, path: str, params: dict):
+        for attempt in range(6):
+            if self.stop is not None and self.stop.is_set():
+                raise InterruptedError
+            self.pacer.wait(self.stop)
+            try:
+                r = self.c.get(path, params=params)
+            except httpx.HTTPError as e:
+                log.warning("swap-api %s: %s", path, e); _sleep(5.0 * (attempt + 1), self.stop); continue
+            if r.status_code == 200:
+                self.pacer.ok(); self.status.bump("req_ok"); return r.json()
+            if r.status_code == 429:
+                self.status.bump("req_429"); ra = r.headers.get("retry-after")
+                log.info("swap-api throttled; pace -> %.1f s", self.pacer.pace + 0.5)
+                self.pacer.throttled(self.stop, float(ra) if ra and ra.isdigit() else None); continue
+            if r.status_code >= 500:
+                _sleep(5.0 * (attempt + 1), self.stop); continue
+            raise (Permanent if 400 <= r.status_code < 500 else RuntimeError)(f"swap-api {path} HTTP {r.status_code}: {r.text[:160]}")
+        raise RuntimeError(f"swap-api {path}: gave up after retries")
+
+    def candles(self, mint: str, interval: str, created_ms: int, before_ms: int | None = None) -> list[dict]:
+        p = {"interval": interval, "limit": 1000, "createdTs": created_ms, "currency": "SOL"}
+        if before_ms is not None:
+            p["beforeTs"] = before_ms
+        j = self.get(f"/v2/coins/{mint}/candles", p)
+        return j if isinstance(j, list) else (j.get("candles") or [])
+
+    def trades(self, mint: str, cursor: str | None) -> tuple[list[dict], str | None]:
+        p = {"limit": 100}
+        if cursor:
+            p["cursor"] = cursor
+        j = self.get(f"/v2/coins/{mint}/trades", p)
+        return j.get("trades") or [], (j.get("pagination") or {}).get("nextCursor")
+
+
+class Helius:
+    def __init__(self, stop: threading.Event | None):
+        if not config.HELIUS_API_KEY:
+            raise RuntimeError("HELIUS_API_KEY missing")
+        self.c = httpx.Client(timeout=120); self.stop = stop; self.last = 0.0
+
+    def gtfa(self, address: str, opts: dict) -> dict:
+        for attempt in range(5):
+            dt = time.time() - self.last
+            if dt < 0.3:
+                time.sleep(0.3 - dt)
+            self.last = time.time()
+            try:
+                r = self.c.post(f"{config.HELIUS_HTTP}?api-key={config.HELIUS_API_KEY}",
+                                json={"jsonrpc": "2.0", "id": 1, "method": "getTransactionsForAddress", "params": [address, opts]})
+                j = r.json()
+            except (httpx.HTTPError, ValueError) as e:
+                log.warning("helius gTFA: %s", e); _sleep(3.0 * (attempt + 1), self.stop); continue
+            if "result" in j and j["result"] is not None:
+                return j["result"]
+            err = (j.get("error") or {}).get("message", "unknown error")
+            if "rate" in err.lower() or r.status_code == 429:
+                _sleep(5.0 * (attempt + 1), self.stop); continue
+            raise RuntimeError(f"helius gTFA: {err}")
+        raise RuntimeError("helius gTFA: gave up after retries")
+
+
+# ---------------------------------------------------------------- enumeration
+def _graduations_from_page(data: list[dict]) -> list[tuple[str, int, str, int]]:
+    """(mint, blockTime, signature, slot) for every succeeded migration transaction on the page."""
+    out = []
+    for tx in data:
+        msg = tx["transaction"]["message"]; keys = msg.get("accountKeys") or []
+        progs = set()
+        for ins in msg.get("instructions") or []:
+            pid = ins.get("programId")
+            if pid is None and "programIdIndex" in ins and ins["programIdIndex"] < len(keys):
+                k = keys[ins["programIdIndex"]]; pid = k["pubkey"] if isinstance(k, dict) else k
+            progs.add(pid)
+        if PUMP_PROGRAM not in progs:
+            continue
+        mints = {b.get("mint") for b in ((tx.get("meta") or {}).get("postTokenBalances") or [])} - {WSOL, None}
+        if len(mints) != 1:
+            continue
+        sig = tx["transaction"]["signatures"][0] if tx["transaction"].get("signatures") else None
+        out.append((mints.pop(), int(tx["blockTime"]), sig, int(tx["slot"])))
+    return out
+
+
+def _upsert_graduations(rows: list[tuple[str, int, str, int]]) -> int:
+    if not rows:
+        return 0
+    with transaction() as conn:
+        before = conn.execute("SELECT count(*) AS n FROM corpus_tokens").fetchone()["n"]
+        conn.cursor().executemany(
+            "INSERT INTO corpus_tokens (mint, graduated_at, grad_sig, grad_slot) VALUES (%s, to_timestamp(%s), %s, %s) "
+            "ON CONFLICT (mint) DO UPDATE SET graduated_at = LEAST(corpus_tokens.graduated_at, EXCLUDED.graduated_at), "
+            "grad_slot = LEAST(corpus_tokens.grad_slot, EXCLUDED.grad_slot), updated_at = now()", rows)
+        after = conn.execute("SELECT count(*) AS n FROM corpus_tokens").fetchone()["n"]
+    return int(after - before)
+
+
+def _enum_state() -> dict:
+    with transaction() as conn:
+        r = conn.execute("SELECT value FROM ui_settings WHERE key = 'corpus_enum'").fetchone()
+    v = r["value"] if r else {}
+    return v if isinstance(v, dict) else json.loads(v or "{}")
+
+
+def _save_enum_state(state: dict) -> None:
+    with transaction() as conn:
+        conn.execute("INSERT INTO ui_settings (key, value) VALUES ('corpus_enum', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                     (json.dumps(state, default=str),))
+
+
+def enumerate_graduations(hel: Helius, status: Status, stop: threading.Event | None, max_pages: int = 20) -> int:
+    """One enumeration round: (1) newest pages until a page adds nothing new, (2) up to ``max_pages`` of backfill
+    from the persisted pagination token until ``CORPUS_DAYS`` is reached. Returns rows added."""
+    addr = config.CORPUS_MIGRATION_AUTHORITY
+    pull_before = datetime.fromisoformat(config.CORPUS_PULL_BEFORE).replace(tzinfo=timezone.utc)
+    horizon = int(min(datetime.now(timezone.utc) - timedelta(days=config.CORPUS_DAYS), pull_before - timedelta(days=30)).timestamp())
+    state = _enum_state(); added = 0
+    # (1) incremental: newest first, stop when a full page adds nothing
+    token = None
+    for _ in range(500):          # keeps walking after an outage until a page adds nothing
+        res = hel.gtfa(addr, {"limit": 100, "transactionDetails": "full", "sortOrder": "desc", "filters": {"status": "succeeded"},
+                              **({"paginationToken": token} if token else {})})
+        rows = _graduations_from_page(res.get("data") or []); n = _upsert_graduations(rows); added += n
+        token = res.get("paginationToken")
+        status.update(stage="enumerating (new graduations)", enum_added=added)
+        if n == 0 or not token or (stop is not None and stop.is_set()):
+            break
+    # (2) backfill toward the horizon from where the last round stopped
+    if not state.get("backfill_done"):
+        token = state.get("token"); oldest = state.get("oldest_bt")
+        if token is None and oldest is None:
+            # first round: start a fresh newest-first walk that will proceed past the incremental pages
+            token = None
+        for _ in range(max_pages):
+            if stop is not None and stop.is_set():
+                break
+            res = hel.gtfa(addr, {"limit": 100, "transactionDetails": "full", "sortOrder": "desc", "filters": {"status": "succeeded"},
+                                  **({"paginationToken": token} if token else {})})
+            data = res.get("data") or []
+            rows = _graduations_from_page(data); added += _upsert_graduations(rows)
+            token = res.get("paginationToken")
+            bts = [int(t["blockTime"]) for t in data if t.get("blockTime")]
+            if bts:
+                oldest = min(bts)
+            state.update({"token": token, "oldest_bt": oldest})
+            status.update(stage="enumerating (backfill)", backfill_through=datetime.fromtimestamp(oldest, timezone.utc).isoformat() if oldest else None,
+                          backfill_target=datetime.fromtimestamp(horizon, timezone.utc).date().isoformat(), enum_added=added)
+            if not token or (oldest is not None and oldest < horizon):
+                state["backfill_done"] = True; break
+        _save_enum_state(state)
+    status.update(backfill_done=bool(state.get("backfill_done")), force=True)
+    return added
+
+
+# ---------------------------------------------------------------- per-token pull
+def _write_atomic(table: pa.Table, path: Path) -> None:
+    tmp = path.with_name(path.name + ".tmp")
+    pq.write_table(table, tmp, compression="zstd"); tmp.replace(path)
+
+
+def _candle_rows(cands: list[dict], interval: str) -> list[dict]:
+    return [{"ts": datetime.fromtimestamp(int(c["timestamp"]) / 1000, timezone.utc), "interval": interval, "open": float(c["open"]),
+             "high": float(c["high"]), "low": float(c["low"]), "close": float(c["close"]), "volume_sol": float(c["volume"])} for c in cands]
+
+
+def _page_back(api: SwapApi, mint: str, interval: str, created_ms: int, before_ms: int, until_ms: int, max_pages: int) -> list[dict]:
+    """Candles with timestamp in [until_ms, before_ms), paging backwards."""
+    out, before = [], before_ms
+    for _ in range(max_pages):
+        page = api.candles(mint, interval, created_ms, before)
+        if not page:
+            break
+        out.extend(c for c in page if until_ms <= int(c["timestamp"]) < before_ms)
+        lo = min(int(c["timestamp"]) for c in page)
+        if lo <= until_ms or len(page) < 1000:
+            break
+        before = lo
+    return out
+
+
+def pull_token(api: SwapApi, row: dict, status: Status) -> dict:
+    """Two to four swap-api requests per token: 1-minute candles for the first CORPUS_CANDLE_1M_H hours (plus the last
+    30 curve minutes), then the newest 5-minute page (which also tells how long the token lived), paged back to the
+    1-minute window and capped at CORPUS_CANDLE_5M_D days. Trades (first CORPUS_TRADES_H hours) for a deterministic
+    CORPUS_TRADES_SAMPLE share of tokens that lived ≥ CORPUS_MIN_LIFE_H and graduated within CORPUS_TRADES_DAYS."""
+    mint: str = row["mint"]; grad: datetime = row["graduated_at"]
+    g_ms = int(grad.timestamp() * 1000); created_ms = g_ms - 86_400_000
+    cdir = config.CORPUS_DIR / "candles"; tdir = config.CORPUS_DIR / "trades"
+    cpath, tpath = cdir / f"{mint}.parquet", tdir / f"{mint}.parquet"
+    upd: dict = {"status": "done", "candles_1m": 0, "candles_5m": 0, "trades": 0, "trades_through": None, "candle_path": None, "trade_path": None}
+    start_ms = g_ms - 30 * 60_000; win1_end = g_ms + int(config.CORPUS_CANDLE_1M_H * 3.6e6); win2_end = g_ms + int(config.CORPUS_CANDLE_5M_D * 86_400_000)
+    one_m = _page_back(api, mint, "1m", created_ms, win1_end, start_ms, max_pages=3)
+    newest5 = api.candles(mint, "5m", created_ms)
+    stamps = [int(c["timestamp"]) for c in one_m] + [int(c["timestamp"]) for c in newest5]
+    if not stamps:
+        upd["status"] = "empty"; return upd
+    last_ms = max(stamps); life_h = (last_ms - g_ms) / 3.6e6; upd["life_h"] = life_h
+    five_m = [c for c in newest5 if win1_end <= int(c["timestamp"]) < win2_end]
+    if newest5 and len(newest5) >= 1000 and min(int(c["timestamp"]) for c in newest5) > win1_end:
+        lo5 = min(int(c["timestamp"]) for c in newest5)
+        pages = min(20, int((min(lo5, win2_end) - win1_end) / (1000 * 300_000)) + 2)     # enough pages to reach the 1-minute window's end
+        five_m += _page_back(api, mint, "5m", created_ms, min(lo5, win2_end), win1_end, max_pages=pages)
+        five_m = [c for c in five_m if int(c["timestamp"]) < win2_end]
+    rows = _candle_rows(one_m, "1m") + _candle_rows(five_m, "5m"); upd["candles_1m"] = len(one_m); upd["candles_5m"] = len(five_m)
+    if rows and not cpath.exists():
+        _write_atomic(pa.Table.from_pylist(rows, schema=CANDLE_SCHEMA), cpath)
+    upd["candle_path"] = str(cpath) if cpath.exists() else None
+    status.bump("candles_rows", len(rows))
+    recent = grad >= datetime.now(timezone.utc) - timedelta(days=config.CORPUS_TRADES_DAYS)
+    sampled = (int.from_bytes(hashlib.sha1(mint.encode()).digest()[:4], "big") / 2**32) < config.CORPUS_TRADES_SAMPLE
+    if life_h >= config.CORPUS_MIN_LIFE_H and recent and sampled and not tpath.exists():
+        end_ms = min(g_ms + int(config.CORPUS_TRADES_H * 3.6e6), last_ms + 60_000)
+        slot_t = int(row["grad_slot"] or 0) + int((end_ms - g_ms) / 1000 * SLOTS_PER_S)
+        cursor: str | None = f"{slot_t:010d}000000000000-{end_ms}"
+        trows: list[dict] = []; pages = 0; oldest_seen = end_ms
+        while cursor and pages < config.CORPUS_TRADES_MAX_PAGES:
+            trades, cursor = api.trades(mint, cursor); pages += 1
+            if not trades:
+                break
+            for t in trades:
+                ts = datetime.fromisoformat(t["timestamp"].replace("Z", "+00:00")); ms = int(ts.timestamp() * 1000)
+                oldest_seen = min(oldest_seen, ms)
+                if ms < start_ms:
+                    continue
+                trows.append({"ts": ts, "slot_index": t.get("slotIndexId"), "tx": t.get("tx"), "wallet": t.get("userAddress"),
+                              "side": 1 if t.get("type") == "buy" else -1, "program": t.get("program"),
+                              "price_sol": float(t.get("priceSol") or 0.0), "sol": float(t.get("amountSol") or 0.0), "tokens": float(t.get("baseAmount") or 0.0)})
+            if oldest_seen < start_ms:
+                break
+        if trows:
+            _write_atomic(pa.Table.from_pylist(trows, schema=TRADE_SCHEMA), tpath)
+            upd.update(trades=len(trows), trades_through=datetime.fromtimestamp(end_ms / 1000, timezone.utc), trade_path=str(tpath))
+            status.bump("trades_rows", len(trows)); status.bump("with_trades")
+    return upd
+
+
+def _next_pending(limit: int = 50) -> list[dict]:
+    """Pending tokens plus transient errors (last_error not 'permanent: …') that have cooled down RETRY_ERROR_H."""
+    with transaction() as conn:
+        return conn.execute("SELECT mint, graduated_at, grad_slot FROM corpus_tokens WHERE (status = 'pending' OR (status = 'error' "
+                            "AND updated_at < now() - %s * interval '1 hour' AND COALESCE(last_error, '') NOT LIKE 'permanent:%%')) "
+                            "AND graduated_at < now() - %s * interval '1 hour' AND graduated_at < %s::timestamptz ORDER BY graduated_at DESC LIMIT %s",
+                            (RETRY_ERROR_H, config.CORPUS_MIN_AGE_H, config.CORPUS_PULL_BEFORE, limit)).fetchall()
+
+
+def _mark(mint: str, upd: dict) -> None:
+    with transaction() as conn:
+        conn.execute("UPDATE corpus_tokens SET status = %s, life_h = %s, candles_1m = %s, candles_5m = %s, trades = %s, trades_through = %s, "
+                     "candle_path = %s, trade_path = %s, last_error = %s, updated_at = now() WHERE mint = %s",
+                     (upd.get("status", "done"), upd.get("life_h"), upd.get("candles_1m"), upd.get("candles_5m"), upd.get("trades"), upd.get("trades_through"),
+                      upd.get("candle_path"), upd.get("trade_path"), upd.get("last_error"), mint))
+
+
+def counts() -> dict:
+    with transaction() as conn:
+        rows = conn.execute("SELECT status, count(*) AS n FROM corpus_tokens GROUP BY status").fetchall()
+    return {r["status"]: int(r["n"]) for r in rows}
+
+
+# ---------------------------------------------------------------- worker entry
+def main(stop_event: threading.Event | None = None) -> None:
+    setup("corpus")
+    status = Status(); stop = stop_event
+    (config.CORPUS_DIR / "candles").mkdir(parents=True, exist_ok=True); (config.CORPUS_DIR / "trades").mkdir(parents=True, exist_ok=True)
+    pacer = Pacer(config.CORPUS_PACE_S, config.CORPUS_PACE_MIN_S); api = SwapApi(pacer, status, stop); hel = Helius(stop)
+    record_event("info", "corpus", "corpus puller started", {"days": config.CORPUS_DAYS, "trades_h": config.CORPUS_TRADES_H, "pace_s": pacer.pace})
+    log.info("corpus puller started (days=%d)", config.CORPUS_DAYS)
+    from ..train import corpus_features
+    builder = threading.Thread(target=corpus_features.build_loop, args=(stop,), name="corpus-features", daemon=True); builder.start()
+    try:
+        while not (stop is not None and stop.is_set()):
+            try:
+                added = enumerate_graduations(hel, status, stop)
+                if added:
+                    log.info("enumerated %d new graduations", added)
+            except InterruptedError:
+                break
+            except Exception as e:
+                log.exception("enumeration failed"); record_event("error", "corpus", f"enumeration failed: {type(e).__name__}: {e}")
+                status.update(stage="enumeration error", last_error=str(e)[:200], force=True); _sleep(30, stop)
+            pending = _db_retry(stop, "corpus pending query", _next_pending)
+            if pending is None:
+                break
+            c = _db_retry(stop, "corpus counts", counts) or {}; status.update(stage="pulling" if pending else "idle", pending=c.get("pending", 0), done=c.get("done", 0), empty=c.get("empty", 0),
+                                        errors=c.get("error", 0), enumerated=sum(c.values()), pace_s=round(pacer.pace, 2), force=True)
+            if not pending:
+                _sleep(60, stop); continue
+            for row in pending:
+                if stop is not None and stop.is_set():
+                    break
+                try:
+                    upd = pull_token(api, row, status)
+                except InterruptedError:
+                    break
+                except Exception as e:      # a 4xx is permanent; anything else is requeued by _next_pending after RETRY_ERROR_H
+                    perm = "permanent: " if isinstance(e, Permanent) else ""
+                    log.warning("pull %s failed: %s%s", row["mint"], perm, e); upd = {"status": "error", "last_error": f"{perm}{type(e).__name__}: {e}"[:300]}
+                _db_retry(stop, "corpus mark", _mark, row["mint"], upd); status.bump("tokens_pulled")
+                c = _db_retry(stop, "corpus counts", counts) or {}
+                status.update(stage="pulling", last_mint=row["mint"], last_graduated=row["graduated_at"].isoformat(), pending=c.get("pending", 0), done=c.get("done", 0),
+                              empty=c.get("empty", 0), errors=c.get("error", 0), enumerated=sum(c.values()), pace_s=round(pacer.pace, 2),
+                              eta_h=(c.get("pending", 0) / status.s["tokens_per_h"]) if status.s.get("tokens_per_h") else None)
+    finally:
+        builder.join(30.0)
+        status.update(stage="stopped", force=True)
+        record_event("info", "corpus", "corpus puller stopped", {k: status.s.get(k) for k in ("tokens_pulled", "candles_rows", "trades_rows", "req_ok", "req_429")})
+        log.info("corpus puller stopped")

fly_trader/ingest/discovery.py

@@ -0,0 +1,80 @@
+"""Jupiter token stats (the `discover` worker).
+
+Every STATS_REFRESH_S, token_stats via /search batches (100 mints per call) for every token of the selector's universe
+(traded on the stream in the last hour, pool above its gate). Jupiter serves only current values, so this is how a
+point-in-time history accrues for future model inputs.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import signal
+import time
+
+from .. import config
+from ..chain.jupiter_tokens import JupiterTokens
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..logging_setup import setup
+
+log = logging.getLogger(__name__)
+
+
+def insert_stats(conn, tok: dict) -> None:
+    audit = tok.get("audit") or {}
+    stats = {k: tok.get(k) for k in ("stats5m", "stats1h", "stats6h", "stats24h")}
+    conn.execute(
+        """INSERT INTO token_stats (mint, organic_score, organic_label, holder_count, liquidity_usd, usd_price, mcap,
+              top_holders_pct, dev_balance_pct, dev_mints, is_sus, mint_auth_disabled, freeze_auth_disabled, is_verified, stats)
+           VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+        (tok["id"], tok.get("organicScore"), tok.get("organicScoreLabel"), tok.get("holderCount"), tok.get("liquidity"),
+         tok.get("usdPrice"), tok.get("mcap"), audit.get("topHoldersPercentage"), audit.get("devBalancePercentage"),
+         audit.get("devMints"), audit.get("isSus"), audit.get("mintAuthorityDisabled"), audit.get("freezeAuthorityDisabled"),
+         tok.get("isVerified"), json.dumps(stats, default=str)),
+    )
+
+
+def refresh_stats(conn, client: JupiterTokens) -> dict:
+    """token_stats for the selector's universe: every pump.fun PumpSwap token the stream saw trade in the last hour with a
+    pool above the selector's gate, so a point-in-time history accrues for the tokens actually traded."""
+    from ..train.decisions import MIN_RESQ_SOL   # lazy: keeps the training stack out of the worker's import
+    uni = conn.execute("SELECT DISTINCT mint FROM pump_minutes WHERE ts > now() - interval '1 hour' AND resq_sol >= %s",
+                       (MIN_RESQ_SOL,)).fetchall()
+    mints = sorted(r["mint"] for r in uni)
+    toks = [t for t in client.search_many(mints) if t.get("id")] if mints else []
+    for tok in toks:
+        insert_stats(conn, tok)
+    return {"universe": len(mints), "refreshed": len(toks)}
+
+
+def run_once() -> None:
+    setup("discover")
+    with transaction() as conn:
+        print(json.dumps(refresh_stats(conn, JupiterTokens()), indent=1))
+
+
+def run_forever(stop_event=None) -> None:
+    """Loop until stop_event is set (console thread) or SIGTERM/SIGINT (CLI, main thread only)."""
+    import threading
+    setup("discover")
+    client = JupiterTokens(); stopped = {"v": False}
+    if stop_event is None and threading.current_thread() is threading.main_thread():
+        def _stop(*_):
+            stopped["v"] = True
+        signal.signal(signal.SIGTERM, _stop)
+        signal.signal(signal.SIGINT, _stop)
+    record_event("info", "discover", "token stats worker started", {"refresh_s": config.STATS_REFRESH_S})
+    while not stopped["v"] and not (stop_event is not None and stop_event.is_set()):
+        t0 = time.monotonic()
+        try:
+            with transaction() as conn:
+                log.info("stats refresh %s", refresh_stats(conn, client))
+        except Exception as e:
+            log.exception("token stats refresh error")
+            record_event("error", "discover", f"stats refresh error: {type(e).__name__}: {e}")
+        # sleep the remainder of the interval, waking early on stop
+        remaining = config.STATS_REFRESH_S - (time.monotonic() - t0)
+        while remaining > 0 and not stopped["v"] and not (stop_event is not None and stop_event.is_set()):
+            time.sleep(min(1.0, remaining))
+            remaining -= 1.0
+    record_event("info", "discover", "token stats worker stopped")

fly_trader/ingest/pumpstream.py

@@ -0,0 +1,488 @@
+"""PumpAPI live stream → per-minute rich candles (Supervisor thread ``pumpstream``).
+
+``wss://stream.pumpapi.io`` is a free, keyless firehose (~400–800 events/s, one connection per IP) whose events are
+identical to the replay archive. This worker keeps only what the selector needs: PumpSwap (``pump-amm``, SOL-quoted)
+buys and sells of pump.fun-origin mints, aggregated per (mint, minute) into the same fields ``train/mature.py``
+derives from the archive, with the same filters: legs in a pool created directly rather than by a pump.fun migration
+(``createPool`` with ``poolCreatedBy`` other than ``pump``; the owner can pull its unburned liquidity) are dropped — such pools
+go to ``pump_pools`` as they are created, and the blocked set is reloaded from that table (archive backfill included) at
+start and every 5 minutes —, the pool's quote reserve must lie in 0.001–100,000 SOL, a leg more
+than 50× away from the median of the mint's previous 200 legs that UTC day is dropped, and each minute keeps the
+pool with the most legs. Minutes are written to ``pump_minutes`` once complete: an event stamped ≥ 2 s after the
+minute's end has arrived (event-time watermark) or, once no trade has arrived for 15 s, by wall clock; a leg for a minute
+already written is dropped (never merged into it). ``pumpstream_status.flushed_through``
+publishes the newest complete minute for the selector.
+
+Lifecycle: ``create`` events go to ``pump_events`` as they arrive; at ``migrate`` a ``corpus_meta`` row is written
+with the creation facts and the creator's point-in-time history (``train/corpus_meta.creator_history``, the training
+definition); 61 minutes after a graduation its own 60-minute outcome is filled from the stream's minutes so later
+graduations by the same creator see it, as in training. Minute rows older than ``PUMP_MINUTES_KEEP_DAYS`` are archived
+to ``data/corpus/stream/<day>.parquet`` before removal. Database failures keep the data in memory for the next attempt.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import math
+import ssl
+import statistics
+import threading
+import time
+from collections import defaultdict, deque
+from datetime import datetime, timedelta, timezone
+
+import certifi
+import orjson
+import pyarrow as pa
+import pyarrow.parquet as pq
+import websockets
+
+from .. import config
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..logging_setup import setup
+from ..train.corpus_meta import CURVE_COLS, blocked_pool_ids, creator_history, curve_facts
+from ..train.flow import minute_wallet_cols
+
+log = logging.getLogger(__name__)
+WSOL = "So11111111111111111111111111111111111111112"
+RESQ_BAND = (0.001, 100000.0)     # train/mature.py: quote_in_pool BETWEEN 0.001 AND 100000
+PRICE_BAND = 50.0                  # train/mature.py: within 50x of the trailing median of the mint's previous 200 legs
+REF_LEGS = 200
+FLUSH_GRACE_S = 2.0
+FLUSH_FALLBACK_S = 15.0
+MAX_PENDING_CREATES = 200_000
+
+
+def _txt(v):
+    """Postgres text cannot hold NUL bytes; one in a token name would otherwise wedge the create queue."""
+    return v.replace("\x00", "") if isinstance(v, str) else v
+
+
+class Minute:
+    __slots__ = ("open", "high", "low", "close", "buy", "sell", "nb", "ns", "traders", "wallets", "resq", "pool_id", "fee_rate")
+
+    def __init__(self, pool_id: str | None = None):
+        self.open = None; self.high = -math.inf; self.low = math.inf; self.close = None; self.buy = 0.0; self.sell = 0.0
+        self.nb = 0; self.ns = 0; self.traders = set(); self.resq = None; self.pool_id = pool_id; self.fee_rate = None   # the pool fee charged (poolFeeRate)
+        self.wallets: dict[str, list[float]] = {}                  # trader → [bought SOL, sold SOL] (train/flow.py)
+
+
+class Aggregator:
+    def __init__(self):
+        self.minutes: dict[int, dict[str, dict[str, Minute]]] = defaultdict(lambda: defaultdict(dict))   # minute → mint → pool → Minute
+        self.ref: dict[str, deque] = {}; self.ref_day: int | None = None                                  # trailing legs per mint, reset each UTC day
+        self.max_event_s = 0.0; self.last_event_wall = 0.0                # newest event time; wall clock of the last trade received
+        self.creates: dict[str, dict] = {}                                 # mint → create facts (24 h)
+        self.curve: dict[str, dict] = {}                                   # mint → bonding-curve wallets since its create (corpus_meta.curve_facts)
+        self.pending_creates: list[tuple[str, dict]] = []                 # creates not yet written to pump_events
+        self.blocked: set[str] = set()                                     # pool ids of directly created (custom) PumpSwap pools
+        self.pending_pools: list[tuple] = []                               # custom pools not yet written to pump_pools
+        self.flushed_through: datetime | None = None                       # start of the newest complete minute written
+        self.insiders: dict[str, frozenset] = {}                           # mint → creator/bundle/early wallets (token_insiders)
+        self.skill: dict | None = None; self.skill_day: str | None = None  # wallet → skill decile for the current UTC day (train/wallet_skill.py)
+        self.stats = {"events": 0, "trades": 0, "dropped_band": 0, "dropped_late": 0, "dropped_custom_pool": 0, "flushed_minutes": 0, "flushed_rows": 0, "creates": 0, "migrates": 0,
+                      "custom_pools": 0, "blocked_pools": 0, "reconnects": 0, "outcomes_filled": 0, "started_at": datetime.now(timezone.utc).isoformat()}
+
+    def row(self, minute: int, mint: str) -> Minute | None:
+        """The minute's candle for a mint: the pool with the most legs (as in training)."""
+        pools = self.minutes.get(minute, {}).get(mint)
+        return max(pools.values(), key=lambda r: (r.nb + r.ns, r.buy + r.sell)) if pools else None
+
+    def ingest(self, e: dict) -> None:
+        self.stats["events"] += 1
+        a = e.get("action"); pool = e.get("pool"); mint = e.get("mint"); ts = e.get("timestamp")
+        if a == "createPool" and pool == "pump-amm" and (e.get("poolCreatedBy") or "custom") != "pump":
+            pid = e.get("poolId")                   # not a pump.fun migration: the owner can pull the liquidity unseen
+            if pid:
+                self.block([pid]); self.stats["custom_pools"] += 1
+                self.pending_pools.append((pid, mint, e.get("poolCreatedBy"), datetime.fromtimestamp(ts / 1000, timezone.utc) if ts is not None else None))
+            return
+        if ts is None or not mint:
+            return
+        if a in ("buy", "sell") and pool == "pump":
+            self._curve(e)
+            return
+        if a in ("buy", "sell"):
+            if pool != "pump-amm" or e.get("quoteMint") != WSOL or not str(mint).endswith("pump"):
+                return
+            if e.get("poolId") in self.blocked:
+                self.stats["dropped_custom_pool"] += 1
+                return
+            price = e.get("price"); q = e.get("quoteInPool")
+            if not price or price <= 0 or q is None or not (RESQ_BAND[0] <= float(q) <= RESQ_BAND[1]):
+                return
+            ts_s = ts / 1000.0; self.max_event_s = max(self.max_event_s, ts_s); self.last_event_wall = time.time()
+            day = int(ts_s // 86400)
+            if self.ref_day is None or day > self.ref_day:      # a late event from the previous day never wipes today's references
+                self.ref = {}; self.ref_day = day
+            p = float(price)
+            legs = e.get("breakdown") or [{"action": a, "trader": e.get("txSigner"), "quoteAmount": e.get("quoteAmount")}]
+            dq = self.ref.get(mint)
+            if dq is None:
+                dq = self.ref[mint] = deque(maxlen=REF_LEGS)
+            ok = True
+            if dq:
+                pref = statistics.median(dq)
+                ok = pref / PRICE_BAND <= p <= pref * PRICE_BAND
+            for _ in legs:
+                dq.append(p)
+            if not ok:
+                self.stats["dropped_band"] += 1
+                return
+            m = int(ts_s // 60) * 60; pid = e.get("poolId") or ""
+            if self.flushed_through is not None and m <= self.flushed_through.timestamp():
+                self.stats["dropped_late"] += 1           # its minute is written and consumed; a partial re-write would corrupt it
+                return
+            row = self.minutes[m][mint].get(pid)
+            if row is None:
+                row = self.minutes[m][mint][pid] = Minute(pid or None)
+            row.open = p if row.open is None else row.open; row.high = max(row.high, p); row.low = min(row.low, p); row.close = p
+            for b in legs:
+                sol = float(b.get("quoteAmount") or 0.0)
+                if b.get("action") == "buy":
+                    row.buy += sol; row.nb += 1
+                else:
+                    row.sell += sol; row.ns += 1
+                trader = b.get("trader") or e.get("txSigner")          # as replay_pull writes the archive's trader column
+                if trader:
+                    row.traders.add(trader)
+                    row.wallets.setdefault(trader, [0.0, 0.0])[0 if b.get("action") == "buy" else 1] += sol
+            row.resq = float(q)
+            if e.get("poolFeeRate") is not None:
+                row.fee_rate = float(e["poolFeeRate"])
+            self.stats["trades"] += 1
+        elif a == "create" and pool == "pump":
+            c = {"ts": ts, "creator": e.get("txSigner"), "dev_sol": e.get("quoteAmount"), "dev_tokens": e.get("initialBuy"), "supply": e.get("supply"),
+                 "mayhem": e.get("mayhemMode"), "name": _txt(e.get("name")), "symbol": _txt(e.get("symbol")), "uri": _txt(e.get("uri")), "sig": e.get("signature")}
+            self.creates[mint] = c; self.pending_creates.append((mint, c))
+            self.curve[mint] = {"slot": int(e.get("block") or 0), "creator": e.get("txSigner"), "supply": e.get("supply"), "initial_buy": e.get("initialBuy"), "ts": ts, "wallets": {}}
+            self.stats["creates"] += 1
+            if len(self.creates) > 200_000:
+                cutoff = ts - 86_400_000
+                self.creates = {k: v for k, v in self.creates.items() if v["ts"] >= cutoff}
+                self.curve = {k: v for k, v in self.curve.items() if v["ts"] >= cutoff}
+        elif a == "migrate" and pool == "pump-amm":
+            self.stats["migrates"] += 1
+            self.write_creates()                    # the creator's own earlier creates must be visible to the history query
+            self._graduation(e)
+
+    def _curve(self, e: dict) -> None:
+        """A bonding-curve buy/sell of a token created while the stream ran: per wallet its first buy slot and tokens bought/sold."""
+        st = self.curve.get(e.get("mint"))
+        if st is None:
+            return                                   # created before this process started: the archive fills its facts (fail closed until then)
+        slot = int(e.get("block") or 0)
+        for b in e.get("breakdown") or [{"action": e.get("action"), "trader": e.get("txSigner"), "tokenAmount": e.get("tokenAmount")}]:
+            w = b.get("trader") or e.get("txSigner")
+            if not w:
+                continue
+            rec = st["wallets"].setdefault(w, [None, 0.0, 0.0]); tok = float(b.get("tokenAmount") or 0.0)
+            if (b.get("action") or e.get("action")) == "buy":
+                rec[1] += tok; rec[0] = slot if rec[0] is None else min(rec[0], slot)
+            else:
+                rec[2] += tok
+
+    def write_creates(self) -> int:
+        """Persist creates as they arrive (keyed by signature); on failure they stay queued for the next attempt."""
+        rows, self.pending_creates = self.pending_creates, []
+        if not rows:
+            return 0
+        try:
+            with transaction() as conn:
+                conn.cursor().executemany("INSERT INTO pump_events (sig, ts, action, pool, mint, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri) "
+                                          "VALUES (%s,%s,'create','pump',%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (sig) DO NOTHING",
+                                          [(c["sig"], datetime.fromtimestamp(c["ts"] / 1000, timezone.utc), m, c["creator"], c["dev_sol"], c["dev_tokens"], c["supply"],
+                                            c["mayhem"], c["name"], c["symbol"], c["uri"]) for m, c in rows if c.get("sig")])
+        except Exception:
+            self.pending_creates = (rows + self.pending_creates)[-MAX_PENDING_CREATES:]
+            log.exception("create rows failed; %d queued for the next attempt", len(self.pending_creates))
+            return 0
+        return len(rows)
+
+    def block(self, pool_ids) -> None:
+        self.blocked.update(pool_ids); self.stats["blocked_pools"] = len(self.blocked)
+
+    def write_pools(self) -> int:
+        """Persist custom pools seen on the stream (training excludes the same pools); on failure they stay queued."""
+        rows, self.pending_pools = self.pending_pools, []
+        if not rows:
+            return 0
+        try:
+            with transaction() as conn:
+                conn.cursor().executemany("INSERT INTO pump_pools (pool_id, mint, created_by, ts, source) VALUES (%s,%s,%s,%s,'stream') ON CONFLICT (pool_id) DO NOTHING", rows)
+        except Exception:
+            self.pending_pools = (rows + self.pending_pools)[-MAX_PENDING_CREATES:]
+            log.exception("custom pool rows failed; %d queued for the next attempt", len(self.pending_pools))
+            return 0
+        return len(rows)
+
+    def _graduation(self, e: dict) -> None:
+        mint = e["mint"]; g = datetime.fromtimestamp(e["timestamp"] / 1000, timezone.utc); c = self.creates.get(mint)
+        try:
+            with transaction() as conn:
+                if c is None:                                       # created before this process started: look it up
+                    r = conn.execute("SELECT sig, ts, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri FROM pump_events WHERE mint = %s AND action = 'create' ORDER BY ts LIMIT 1", (mint,)).fetchone()
+                    if r:
+                        c = {"ts": r["ts"].timestamp() * 1000, "creator": r["signer"], "dev_sol": r["dev_sol"], "dev_tokens": r["dev_tokens"], "supply": r["supply"],
+                             "mayhem": r["mayhem"], "name": r["name"], "symbol": r["symbol"], "uri": r["uri"], "sig": r["sig"]}
+                conn.execute("INSERT INTO pump_events (sig, ts, action, pool, mint, pool_id, signer, quote_in_pool) VALUES (%s,%s,'migrate','pump-amm',%s,%s,%s,%s) ON CONFLICT (sig) DO NOTHING",
+                             (e.get("signature"), g, mint, e.get("poolId"), e.get("txSigner"), e.get("quoteInPool")))
+                ct = datetime.fromtimestamp(c["ts"] / 1000, timezone.utc) if c else None
+                h = creator_history(conn, c.get("creator") if c else None, ct, g)
+                conn.execute("""INSERT INTO corpus_meta (mint, graduated_at, create_ts, creator, dev_sol, dev_tokens, dev_share, supply, mayhem, uri, name, symbol, ttg_min, rq0, pool_id,
+                                                         prior_launches, prior_grads, prior_known, prior_rug_share, prior_moon_share)
+                                VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint) DO UPDATE SET
+                                  graduated_at = COALESCE(corpus_meta.graduated_at, EXCLUDED.graduated_at),
+                                  create_ts = COALESCE(corpus_meta.create_ts, EXCLUDED.create_ts), creator = COALESCE(corpus_meta.creator, EXCLUDED.creator),
+                                  dev_sol = COALESCE(corpus_meta.dev_sol, EXCLUDED.dev_sol), dev_tokens = COALESCE(corpus_meta.dev_tokens, EXCLUDED.dev_tokens),
+                                  dev_share = COALESCE(corpus_meta.dev_share, EXCLUDED.dev_share), supply = COALESCE(corpus_meta.supply, EXCLUDED.supply),
+                                  mayhem = COALESCE(corpus_meta.mayhem, EXCLUDED.mayhem), uri = COALESCE(corpus_meta.uri, EXCLUDED.uri),
+                                  name = COALESCE(corpus_meta.name, EXCLUDED.name), symbol = COALESCE(corpus_meta.symbol, EXCLUDED.symbol),
+                                  ttg_min = COALESCE(corpus_meta.ttg_min, EXCLUDED.ttg_min), rq0 = COALESCE(corpus_meta.rq0, EXCLUDED.rq0), pool_id = COALESCE(corpus_meta.pool_id, EXCLUDED.pool_id),
+                                  prior_launches = COALESCE(corpus_meta.prior_launches, EXCLUDED.prior_launches), prior_grads = COALESCE(corpus_meta.prior_grads, EXCLUDED.prior_grads),
+                                  prior_known = COALESCE(corpus_meta.prior_known, EXCLUDED.prior_known), prior_rug_share = COALESCE(corpus_meta.prior_rug_share, EXCLUDED.prior_rug_share),
+                                  prior_moon_share = COALESCE(corpus_meta.prior_moon_share, EXCLUDED.prior_moon_share), updated_at = now()""",
+                             (mint, g, ct, c.get("creator") if c else None, c.get("dev_sol") if c else None, c.get("dev_tokens") if c else None,
+                              (float(c["dev_tokens"]) / float(c["supply"])) if c and c.get("dev_tokens") and c.get("supply") else None,
+                              c.get("supply") if c else None, c.get("mayhem") if c else None, c.get("uri") if c else None, c.get("name") if c else None, c.get("symbol") if c else None,
+                              ((e["timestamp"] - c["ts"]) / 60000.0) if c else None, e.get("quoteInPool"), e.get("poolId"),
+                              h["prior_launches"], h["prior_grads"], h["prior_known"], h["prior_rug_share"], h["prior_moon_share"]))
+                conn.execute("INSERT INTO corpus_tokens (mint, graduated_at, grad_slot, source, status) VALUES (%s,%s,%s,'stream','pending') ON CONFLICT (mint) DO NOTHING",
+                             (mint, g, int(e.get("block") or 0)))
+                st = self.curve.pop(mint, None)
+                if st is not None:                           # the token's whole curve life was seen: the archive's definition (curve_facts)
+                    facts, insiders = curve_facts(st["slot"], st["creator"], st["supply"], st["initial_buy"], st["wallets"])
+                    conn.execute("UPDATE corpus_meta SET " + ", ".join(f"{c} = COALESCE({c}, %s)" for c in CURVE_COLS) + ", curve_known = COALESCE(curve_known, true) WHERE mint = %s",
+                                 (*[facts[c] for c in CURVE_COLS], mint))
+                    conn.cursor().executemany("INSERT INTO token_insiders (mint, wallet, kind) VALUES (%s,%s,%s) ON CONFLICT (mint, wallet) DO NOTHING",
+                                              [(mint, w, k) for w, k in insiders])
+                    self.insiders[mint] = frozenset(w for w, _ in insiders)
+        except Exception:
+            log.exception("graduation upsert failed for %s", mint)
+
+    def flush(self, before_minute: int) -> int:
+        """Write every minute strictly older than ``before_minute`` (seconds). Rows leave memory only after the commit."""
+        done = sorted(m for m in self.minutes if m < before_minute)
+        if not done:
+            return 0
+        rows = []
+        for m in done:
+            ts = datetime.fromtimestamp(m, timezone.utc)
+            for mint in self.minutes[m]:
+                r = self.row(m, mint)
+                if r is None or r.close is None:
+                    continue
+                wc = minute_wallet_cols(r.wallets, self.insiders.get(mint, frozenset()), self.skill)
+                rows.append((mint, ts, r.pool_id, r.open, r.high, r.low, r.close, r.buy, r.sell, r.nb, r.ns, len(r.traders), r.resq, r.fee_rate,
+                             wc["n_buyers"], wc["wash_sol"], wc["wash_buy_sol"], wc["top_sell_sol"], wc["insider_sell_sol"], wc["skill_buy"]))
+        if rows:
+            with transaction() as conn:
+                conn.cursor().executemany("INSERT INTO pump_minutes (mint, ts, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, "
+                                          "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy) "
+                                          "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint, ts) DO UPDATE SET close = EXCLUDED.close, high = greatest(pump_minutes.high, EXCLUDED.high), "
+                                          "low = least(pump_minutes.low, EXCLUDED.low), buy_sol = pump_minutes.buy_sol + EXCLUDED.buy_sol, sell_sol = pump_minutes.sell_sol + EXCLUDED.sell_sol, "
+                                          "n_buys = pump_minutes.n_buys + EXCLUDED.n_buys, n_sells = pump_minutes.n_sells + EXCLUDED.n_sells, n_traders = greatest(pump_minutes.n_traders, EXCLUDED.n_traders), "
+                                          "resq_sol = EXCLUDED.resq_sol, fee_rate = COALESCE(EXCLUDED.fee_rate, pump_minutes.fee_rate), n_buyers = EXCLUDED.n_buyers, wash_sol = EXCLUDED.wash_sol, "
+                                          "wash_buy_sol = EXCLUDED.wash_buy_sol, top_sell_sol = EXCLUDED.top_sell_sol, insider_sell_sol = EXCLUDED.insider_sell_sol, skill_buy = EXCLUDED.skill_buy", rows)
+        for m in done:
+            self.minutes.pop(m, None)
+        self.stats["flushed_minutes"] += len(done); self.stats["flushed_rows"] += len(rows)
+        newest = datetime.fromtimestamp(done[-1], timezone.utc)
+        if self.flushed_through is None or newest > self.flushed_through:          # a late event for an old minute never rewinds it
+            self.flushed_through = newest
+        return len(rows)
+
+    def flush_bound(self, now: float) -> int:
+        """Minutes strictly before this start are complete: event-time watermark, or the wall-clock fallback once no trade
+        has arrived for ``FLUSH_FALLBACK_S`` (a lagging stream still delivers the minute's events, so it waits for them)."""
+        cur = int(now // 60) * 60
+        wm = int((self.max_event_s - FLUSH_GRACE_S) // 60) * 60
+        if now - self.last_event_wall >= FLUSH_FALLBACK_S:
+            wm = max(wm, int((now - FLUSH_FALLBACK_S) // 60) * 60)
+        return min(cur, wm)
+
+
+def fill_outcomes() -> int:
+    """Own 60-minute outcome for stream graduations 61+ minutes old, from the stream's minutes (candle files replace it later)."""
+    with transaction() as conn:
+        todo = conn.execute("""SELECT mint, graduated_at FROM corpus_meta WHERE own_dd60 IS NULL AND graduated_at IS NOT NULL
+                               AND graduated_at BETWEEN now() - interval '6 hours' AND now() - interval '61 minutes'""").fetchall()
+        n = 0
+        for t in todo:
+            rows = conn.execute("SELECT close FROM pump_minutes WHERE mint = %s AND ts >= %s AND ts < %s ORDER BY ts",
+                                (t["mint"], t["graduated_at"], t["graduated_at"] + timedelta(minutes=60))).fetchall()
+            if not rows:
+                continue
+            closes = [float(r["close"]) for r in rows]; p0 = closes[0]
+            if p0 <= 0:
+                continue
+            conn.execute("UPDATE corpus_meta SET own_dd60 = %s, own_max60 = %s, updated_at = now() WHERE mint = %s AND own_dd60 IS NULL",
+                         (min(closes) / p0 - 1.0, max(closes) / p0 - 1.0, t["mint"]))
+            n += 1
+    return n
+
+
+def archive_old(keep_days: int) -> int:
+    """Archive minute rows older than ``keep_days`` to Parquet, then remove them from the table."""
+    cutoff = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=keep_days)
+    with transaction() as conn:
+        days = [r["d"] for r in conn.execute("SELECT DISTINCT date_trunc('day', ts) AS d FROM pump_minutes WHERE ts < %s", (cutoff,)).fetchall()]
+    n = 0
+    for d in days:
+        with transaction() as conn:
+            rows = conn.execute("SELECT * FROM pump_minutes WHERE ts >= %s AND ts < %s ORDER BY mint, ts", (d, d + timedelta(days=1))).fetchall()
+            if rows:
+                out = config.CORPUS_DIR / "stream"; out.mkdir(parents=True, exist_ok=True)
+                path = out / f"{d:%Y-%m-%d}.parquet"; tmp = path.with_name(path.name + ".tmp")
+                pq.write_table(pa.Table.from_pylist([dict(r) for r in rows]), tmp, compression="zstd"); tmp.replace(path)
+                conn.execute("DELETE FROM pump_minutes WHERE ts >= %s AND ts < %s", (d, d + timedelta(days=1)))
+                n += len(rows)
+    return n
+
+
+def _status(agg: Aggregator, extra: dict) -> None:
+    try:
+        with transaction() as conn:
+            conn.execute("INSERT INTO ui_settings (key, value) VALUES ('pumpstream_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                         (json.dumps({**agg.stats, **extra, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str),))
+    except Exception:
+        log.debug("pumpstream status write failed", exc_info=True)
+
+
+def _logged(name: str, fn, *args):
+    try:
+        n = fn(*args)
+        if n:
+            log.info("%s: %d", name, n)
+        return n
+    except Exception:
+        log.exception("%s failed", name)
+        return 0
+
+
+def _load_insiders(agg: Aggregator) -> None:
+    """Insider wallets (creator, bundle, early buyers: token_insiders) of every mint traded today not loaded yet; a token's
+    set is fixed at its graduation, so loaded sets are kept."""
+    todo = [m for m in agg.ref if m not in agg.insiders]
+    if not todo:
+        return
+    try:
+        with transaction() as conn:
+            rows = conn.execute("SELECT mint, wallet FROM token_insiders WHERE mint = ANY(%s)", (todo,)).fetchall()
+    except Exception:
+        log.exception("insider load failed; retried in 1 min")
+        return
+    got: dict[str, set] = {}
+    for r in rows:
+        got.setdefault(r["mint"], set()).add(r["wallet"])
+    for m in todo:
+        agg.insiders[m] = frozenset(got.get(m, ()))
+
+
+def load_skill(agg: Aggregator, now: float | None = None) -> None:
+    """The wallet-skill table of the current UTC day (the file training joins for that day), once per day."""
+    from ..train.mature import SKILL_DIR
+    day = datetime.fromtimestamp(now or time.time(), timezone.utc).date().isoformat()
+    if agg.skill_day == day and agg.skill is not None:
+        return
+    f = SKILL_DIR / f"{day}.parquet"
+    if not f.exists():
+        have = sorted(SKILL_DIR.glob("????-??-??.parquet")) if SKILL_DIR.exists() else []
+        if not have:
+            agg.skill = None; agg.skill_day = day
+            return
+        f = have[-1]              # a shipped table going stale beats none: without one the selector fails closed and the fly's skill inputs read as zero
+        log.warning("no wallet skill table for %s: using the newest on disk, %s (skill inputs %d days stale)", day, f.stem,
+                    (datetime.fromisoformat(day) - datetime.fromisoformat(f.stem)).days)
+    t = pq.read_table(f, columns=["wallet", "bucket"])
+    agg.skill = dict(zip(t["wallet"].to_pylist(), t["bucket"].to_pylist())); agg.skill_day = day
+    log.info("wallet skill table for %s loaded: %d wallets", day, len(agg.skill))
+
+
+def _load_blocked(agg: Aggregator) -> None:
+    """Union the custom pools in ``pump_pools`` (stream rows and the archive backfill) into the blocked set."""
+    try:
+        with transaction() as conn:
+            agg.block(blocked_pool_ids(conn))
+    except Exception:
+        log.exception("custom pool load failed; retried in 5 min")
+
+
+async def _run(stop: threading.Event | None, agg: Aggregator) -> None:
+    ctx = ssl.create_default_context(cafile=certifi.where()); backoff = 1.0
+    _load_blocked(agg); last_pools = time.time(); last_ins = 0.0
+    try:
+        load_skill(agg)
+    except Exception:
+        log.exception("wallet skill load failed; retried in 5 min")
+    last_status = last_flush = last_outcomes = 0.0; last_archive = time.time() - 3000; last_msg = 0.0; rate_n = 0; rate_t = time.time()
+    bg: dict[str, asyncio.Task | None] = {"archive": None, "outcomes": None}
+    while not (stop is not None and stop.is_set()):
+        try:
+            async with websockets.connect(config.PUMPSTREAM_URL, ssl=ctx, max_size=8_000_000, ping_interval=20, ping_timeout=20) as ws:
+                log.info("pumpstream connected"); backoff = 1.0
+                while not (stop is not None and stop.is_set()):
+                    try:
+                        msg = await asyncio.wait_for(ws.recv(), timeout=30)
+                    except asyncio.TimeoutError:
+                        log.warning("pumpstream: no events for 30 s; reconnecting"); break
+                    last_msg = time.time(); rate_n += 1
+                    try:
+                        e = orjson.loads(msg)
+                    except orjson.JSONDecodeError:
+                        continue
+                    if isinstance(e, dict):
+                        agg.ingest(e)
+                    now = time.time()
+                    if now - last_flush >= 1.0:
+                        try:
+                            agg.flush(agg.flush_bound(now))
+                        except Exception:
+                            log.exception("minute flush failed; rows kept for the next attempt")
+                        last_flush = now
+                    if now - last_pools >= 300:
+                        _load_blocked(agg); last_pools = now
+                        try:
+                            load_skill(agg, now)
+                        except Exception:
+                            log.exception("wallet skill load failed; retried in 5 min")
+                    if now - last_ins >= 60:
+                        _load_insiders(agg); last_ins = now
+                    if now - last_status >= 5.0:
+                        agg.write_creates(); agg.write_pools()
+                        _status(agg, {"events_per_s": rate_n / max(now - rate_t, 1e-9), "pending_minutes": len(agg.minutes), "lag_s": now - last_msg, "connected": True,
+                                      "flushed_through": agg.flushed_through.isoformat() if agg.flushed_through else None,
+                                      "event_watermark": datetime.fromtimestamp(agg.max_event_s, timezone.utc).isoformat() if agg.max_event_s else None})
+                        rate_n = 0; rate_t = now; last_status = now
+                    if now - last_outcomes >= 60 and (bg["outcomes"] is None or bg["outcomes"].done()):
+                        bg["outcomes"] = asyncio.create_task(asyncio.to_thread(_logged, "stream outcomes filled", fill_outcomes)); last_outcomes = now
+                    if now - last_archive >= 3600 and (bg["archive"] is None or bg["archive"].done()):
+                        bg["archive"] = asyncio.create_task(asyncio.to_thread(_logged, "archived old minute rows", archive_old, config.PUMP_MINUTES_KEEP_DAYS)); last_archive = now
+        except Exception as e:
+            agg.stats["reconnects"] += 1; log.warning("pumpstream: %s; reconnecting in %.0fs", e, backoff)
+            _status(agg, {"connected": False, "last_error": str(e)[:200]})
+            await asyncio.sleep(backoff); backoff = min(60.0, backoff * 2)
+    for t in bg.values():
+        if t is not None:
+            try:
+                await t
+            except Exception:
+                pass
+    try:
+        agg.flush(int(time.time() // 60) * 60 + 60)
+    except Exception:
+        log.exception("final minute flush failed")
+    agg.write_creates(); agg.write_pools()
+
+
+def main(stop_event: threading.Event | None = None) -> None:
+    setup("pumpstream"); agg = Aggregator()
+    record_event("info", "pumpstream", "pumpstream started", {"url": config.PUMPSTREAM_URL})
+    try:
+        asyncio.run(_run(stop_event, agg))
+    finally:
+        _status(agg, {"connected": False, "stage": "stopped"})
+        record_event("info", "pumpstream", "pumpstream stopped", {k: agg.stats.get(k) for k in ("events", "trades", "dropped_band", "dropped_custom_pool", "flushed_rows", "creates",
+                                                                                                  "custom_pools", "migrates", "reconnects")})

fly_trader/ingest/replay_pull.py

@@ -0,0 +1,315 @@
+"""PumpAPI historical replay → hourly Parquet (Supervisor thread ``replay``).
+
+pumpapi.io publishes every event its decoded data stream emitted, hour by hour, since 2026-04-18:
+``https://replay.pumpapi.io/YYYY/MM/DD/HH.jsonl.zst`` (~400 MB zstd, ~2 GB JSONL, ~1.7 M events; free, no key).
+Each line is one decoded transaction: buys/sells with the trader, SOL and token amounts, price, and the pool's real
+reserves; ``create`` with the creator wallet and metadata URI; ``migrate`` (graduation); ``createPool``.
+
+This worker downloads ``REPLAY_PARALLEL`` hours at a time (measured ~2–4 MB/s per stream), parses each in one
+thread and writes, per hour, ``data/corpus/replay/<date>/<HH>_trades.parquet`` (pump.fun bonding-curve and
+PumpSwap buys/sells, one row per trader leg; signatures and per-mint constants are left out to keep it ~70 MB/hour, the
+lifecycle table carries creator, pool id, metadata URI and the reserves at migration) and ``<HH>_events.parquet`` (every non-trade lifecycle event across
+all pools, small). Hours are processed newest first so the current regime lands first, and hours that close while the backfill runs
+are fetched ahead of it, so the archive stays within ~2 hours of live; the registry is ``replay_hours``. The compressed download is a temporary file and is removed after a successful parse — the
+public archive remains the source of truth. Downstream: ``train/replay_assemble.py`` turns days into per-token
+candle/trade files for the feature builder.
+"""
+from __future__ import annotations
+
+import io
+import json
+import logging
+import queue
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import httpx
+import orjson
+import pyarrow as pa
+import pyarrow.parquet as pq
+import zstandard
+
+from .. import config
+from ..db.apilog import record_event
+from ..db.connection import transaction
+from ..logging_setup import setup
+from .corpus_pull import _db_retry, _sleep
+
+log = logging.getLogger(__name__)
+HOUR_SQL = ("INSERT INTO replay_hours (hour, status, last_error) VALUES (%s, %s, %s) "
+            "ON CONFLICT (hour) DO UPDATE SET status = EXCLUDED.status, last_error = EXCLUDED.last_error, done_at = now()")
+TRADE_POOLS = {"pump", "pump-amm"}
+SKIP_ACTIONS = {"buy", "sell", "transfer", "add", "remove", "claimCreatorFees"}
+TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot", pa.int64()), ("pool", pa.dictionary(pa.int8(), pa.string())), ("mint", pa.string()),
+                          ("trader", pa.string()), ("side", pa.int8()), ("sol", pa.float64()), ("tokens", pa.float64()), ("price", pa.float64()),
+                          ("quote_in_pool", pa.float64()), ("tokens_in_pool", pa.float64()), ("vquote", pa.float32()), ("n_legs", pa.int8()),
+                          ("quote_mint", pa.dictionary(pa.int16(), pa.string())), ("pool_id", pa.dictionary(pa.int32(), pa.string()))])
+EVENT_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot", pa.int64()), ("sig", pa.string()), ("action", pa.string()), ("pool", pa.string()),
+                          ("mint", pa.string()), ("pool_id", pa.string()), ("signer", pa.string()), ("creator_fee_addr", pa.string()), ("name", pa.string()),
+                          ("symbol", pa.string()), ("uri", pa.string()), ("supply", pa.float64()), ("decimals", pa.int16()), ("token_program", pa.string()),
+                          ("initial_buy", pa.float64()), ("quote_amount", pa.float64()), ("quote_in_pool", pa.float64()), ("tokens_in_pool", pa.float64()),
+                          ("mayhem", pa.bool_()), ("pool_created_by", pa.string()), ("mint_authority", pa.string()), ("freeze_authority", pa.string())])
+
+
+def _f(x) -> float | None:
+    try:
+        return float(x) if x is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _q(d: dict, new: str, old: str):
+    """A quote-side field under its current name, else under the archive's pre-2026-05-21 SOL-only name."""
+    v = d.get(new)
+    return v if v is not None else d.get(old)
+
+
+def parse_hour(path: Path) -> tuple[pa.Table, pa.Table, int]:
+    """Stream-decompress one hour file into (trades, events, n_events)."""
+    T = {name: [] for name in TRADE_SCHEMA.names}; E = {name: [] for name in EVENT_SCHEMA.names}; n = 0
+    dctx = zstandard.ZstdDecompressor()
+    with open(path, "rb") as fh, dctx.stream_reader(fh) as reader:
+        for line in io.BufferedReader(reader, buffer_size=8 << 20):
+            if not line.strip():
+                continue
+            try:
+                e = orjson.loads(line)
+            except orjson.JSONDecodeError:
+                continue
+            n += 1
+            # the archive before 2026-05-21 names fields txType / solAmount / solInPool / vSolInBondingCurve; later: action / quote*
+            a = e.get("action") or e.get("txType"); pool = e.get("pool")
+            ts = e.get("timestamp")
+            if ts is None:
+                continue
+            tsdt = datetime.fromtimestamp(int(ts) / 1000, timezone.utc)
+            if a in ("buy", "sell"):
+                if pool not in TRADE_POOLS:
+                    continue
+                legs = e.get("breakdown") or [{"action": a, "trader": e.get("txSigner"), "tokenAmount": e.get("tokenAmount"), "quoteAmount": _q(e, "quoteAmount", "solAmount")}]
+                vq = _q(e, "vQuoteInBondingCurve", "vSolInBondingCurve") if pool == "pump" else _q(e, "virtualQuoteInPool", "virtualSolInPool")
+                for b in legs:
+                    T["ts"].append(tsdt); T["slot"].append(int(e.get("block") or 0)); T["pool"].append(pool); T["mint"].append(e.get("mint"))
+                    T["trader"].append(b.get("trader") or e.get("txSigner")); T["side"].append(1 if (b.get("action") or b.get("txType")) == "buy" else -1)
+                    T["sol"].append(_f(_q(b, "quoteAmount", "solAmount")) or 0.0); T["tokens"].append(_f(b.get("tokenAmount")) or 0.0); T["price"].append(_f(e.get("price")))
+                    T["quote_in_pool"].append(_f(_q(e, "quoteInPool", "solInPool"))); T["tokens_in_pool"].append(_f(e.get("tokensInPool"))); T["vquote"].append(_f(vq))
+                    T["n_legs"].append(len(legs)); T["quote_mint"].append(e.get("quoteMint")); T["pool_id"].append(e.get("poolId"))
+            elif a not in SKIP_ACTIONS:
+                E["ts"].append(tsdt); E["slot"].append(int(e.get("block") or 0)); E["sig"].append(e.get("signature")); E["action"].append(a); E["pool"].append(pool)
+                E["mint"].append(e.get("mint")); E["pool_id"].append(e.get("poolId")); E["signer"].append(e.get("txSigner")); E["creator_fee_addr"].append(e.get("creatorFeeAddress"))
+                E["name"].append(e.get("name")); E["symbol"].append(e.get("symbol")); E["uri"].append(e.get("uri")); E["supply"].append(_f(e.get("supply")))
+                E["decimals"].append(int(e["decimals"]) if e.get("decimals") is not None else None); E["token_program"].append(e.get("tokenProgram"))
+                E["initial_buy"].append(_f(e.get("initialBuy"))); E["quote_amount"].append(_f(_q(e, "quoteAmount", "solAmount"))); E["quote_in_pool"].append(_f(_q(e, "quoteInPool", "solInPool")))
+                E["tokens_in_pool"].append(_f(e.get("tokensInPool"))); E["mayhem"].append(bool(e.get("mayhemMode")) if e.get("mayhemMode") is not None else None)
+                E["pool_created_by"].append(e.get("poolCreatedBy")); E["mint_authority"].append(e.get("mintAuthority")); E["freeze_authority"].append(e.get("freezeAuthority"))
+    return pa.table(T, schema=TRADE_SCHEMA), pa.table(E, schema=EVENT_SCHEMA), n
+
+
+def refetch_missing_pool_id() -> int:
+    """Hours whose trades file predates the ``pool_id`` column (written 2026-08-20..09-13 by an older parser) are marked
+    ``refetch``: ``plan_hours`` downloads them again and ``parse_hour`` writes the column, so the archive applies the live
+    stream's blocked-pool filter and dominant-pool choice (train/mature.py). Returns the number of hours marked."""
+    hours = []
+    for f in sorted(config.REPLAY_DIR.glob("*/*_trades.parquet")):
+        if "pool_id" not in pq.read_schema(f).names:
+            d = datetime.fromisoformat(f.parent.name)
+            hours.append(datetime(d.year, d.month, d.day, int(f.name[:2]), tzinfo=timezone.utc))
+    if hours:
+        with transaction() as conn:
+            conn.cursor().executemany(HOUR_SQL, [(h, "refetch", "trades file lacks pool_id") for h in hours])
+    return len(hours)
+
+
+def plan_hours(now: datetime | None = None) -> list[datetime]:
+    now = now or datetime.now(timezone.utc)   # callers pass the same ``now`` they seed the re-plan bound with
+    start = datetime.fromisoformat(config.REPLAY_START).replace(tzinfo=timezone.utc)
+    last = (now - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0)
+    with transaction() as conn:
+        done = {r["hour"] for r in conn.execute("SELECT hour FROM replay_hours WHERE status = 'done'").fetchall()}
+    hours = []; h = last
+    while h >= start:
+        if h not in done:
+            hours.append(h)
+        h -= timedelta(hours=1)
+    return hours     # newest first
+
+
+def _put_hour(sql: str, params: tuple) -> None:
+    with transaction() as conn:
+        conn.execute(sql, params)
+
+
+def _kept_downloads(tmpdir: Path, planned: set[datetime]) -> list[tuple[datetime, Path, int, float]]:
+    """Complete hour files left by a stopped worker, for hours still planned: parsed instead of downloaded again.
+    Partial files are removed (the archive ignores Range requests, so a download cannot resume), as are complete
+    files for hours no longer planned."""
+    out = []
+    for f in sorted(tmpdir.glob("*")):
+        if f.suffix == ".zst":
+            try:
+                h = datetime.strptime(f.stem, "%Y%m%d%H").replace(tzinfo=timezone.utc)
+            except ValueError:
+                h = None
+            if h in planned:
+                out.append((h, f, f.stat().st_size, 0.0)); continue
+        f.unlink()
+    return out
+
+
+def _download(client: httpx.Client, hour: datetime, tmp: Path, stop: threading.Event | None) -> int:
+    url = f"{config.REPLAY_URL}/{hour:%Y/%m/%d/%H}.jsonl.zst"
+    part = tmp.with_suffix(".part"); size = 0
+    with client.stream("GET", url) as r:
+        if r.status_code == 404:
+            return -1
+        r.raise_for_status()
+        with open(part, "wb") as f:
+            for chunk in r.iter_bytes(1 << 20):
+                if stop is not None and stop.is_set():
+                    raise InterruptedError
+                f.write(chunk); size += len(chunk)
+    part.rename(tmp)
+    return size
+
+
+class _Status:
+    def __init__(self, total: int):
+        self.s = {"hours_total": total, "hours_done": 0, "trades": 0, "events": 0, "bytes": 0, "started_at": datetime.now(timezone.utc).isoformat(), "errors": 0}
+        self.lock = threading.Lock(); self.last = 0.0; self.t0 = time.time()
+
+    def update(self, force=False, **kv):
+        with self.lock:
+            self.s.update(kv); el = time.time() - self.t0
+            self.s["mb_s"] = self.s["bytes"] / 1e6 / el if el > 1 else 0.0
+            self.s["hours_per_h"] = self.s["hours_done"] / el * 3600 if el > 1 else 0.0
+            remaining = self.s["hours_total"] - self.s["hours_done"]
+            self.s["eta_h"] = remaining / self.s["hours_per_h"] if self.s["hours_per_h"] else None
+            self.s["updated_at"] = datetime.now(timezone.utc).isoformat()
+            if not force and time.time() - self.last < 3.0:
+                return
+            self.last = time.time(); payload = dict(self.s)
+        try:
+            with transaction() as conn:
+                conn.execute("INSERT INTO ui_settings (key, value) VALUES ('replay_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
+                             (json.dumps(payload, default=str),))
+        except Exception:
+            log.debug("replay status write failed", exc_info=True)
+
+
+def main(stop_event: threading.Event | None = None) -> None:
+    setup("replay"); stop = stop_event
+    tmpdir = config.REPLAY_DIR / "_tmp"; tmpdir.mkdir(parents=True, exist_ok=True)
+    now0 = datetime.now(timezone.utc); hours = plan_hours(now0); status = _Status(len(hours))
+    preloaded = _kept_downloads(tmpdir, set(hours))                 # finished before a restart: parse, don't download again
+    if preloaded:
+        kept = {p[0] for p in preloaded}; hours = [h for h in hours if h not in kept]
+        log.info("replay ingest: %d complete hour file(s) kept from before the restart", len(preloaded))
+    record_event("info", "replay", "replay ingest started", {"hours_pending": len(hours), "parallel": config.REPLAY_PARALLEL})
+    log.info("replay ingest: %d hours pending (newest first)", len(hours))
+    todo: queue.Queue = queue.Queue(); ready: queue.Queue = queue.Queue(maxsize=config.REPLAY_PARALLEL + 2)
+    for h in hours:
+        todo.put(h)
+    client = httpx.Client(headers={"User-Agent": "fly-trader replay ingest"}, timeout=httpx.Timeout(60.0, read=600.0), follow_redirects=True)
+
+    newest_planned = [(now0 - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0)]   # the plan's upper bound (same clock)
+    fresh: queue.Queue = queue.Queue()      # hours that closed after the plan was made; served before the backfill
+    retried: dict[datetime, float] = {}
+
+    def replan():
+        last = (datetime.now(timezone.utc) - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0)
+        h = newest_planned[0] + timedelta(hours=1)
+        while h <= last:
+            fresh.put(h); h += timedelta(hours=1)
+        if last > newest_planned[0]:
+            newest_planned[0] = last
+        # an hour fetched before the archive published it (404) or that failed is retried every 30 min for 12 h
+        try:
+            with transaction() as conn:
+                again = [r["hour"] for r in conn.execute("SELECT hour FROM replay_hours WHERE status IN ('missing','error') AND hour > now() - interval '12 hours'").fetchall()]
+        except Exception:
+            again = []
+        for h in again:
+            if time.time() - retried.get(h, 0.0) >= 1800:
+                retried[h] = time.time(); fresh.put(h)
+
+    def downloader():
+        while not (stop is not None and stop.is_set()):
+            try:
+                h = fresh.get_nowait()
+            except queue.Empty:
+                try:
+                    h = todo.get_nowait()
+                except queue.Empty:
+                    return
+            tmp = tmpdir / f"{h:%Y%m%d%H}.zst"
+            for attempt in range(4):
+                try:
+                    t0 = time.time(); size = _download(client, h, tmp, stop)
+                    while not (stop is not None and stop.is_set()):
+                        try:
+                            ready.put((h, tmp, size, time.time() - t0), timeout=5.0); break
+                        except queue.Full:
+                            continue
+                    break
+                except InterruptedError:
+                    return
+                except Exception as e:
+                    log.warning("download %s failed (%d): %s", h, attempt, e); time.sleep(10 * (attempt + 1))
+            else:
+                while not (stop is not None and stop.is_set()):
+                    try:
+                        ready.put((h, None, 0, 0.0), timeout=5.0); break
+                    except queue.Full:
+                        continue
+
+    if config.REPLAY_ASSEMBLE:                   # REPLAY_ASSEMBLE=0: a download-only run (e.g. refetch-pool-ids) leaves the builds alone
+        from ..train import replay_assemble
+        assembler = threading.Thread(target=replay_assemble.assemble_loop, args=(stop,), name="replay-assemble", daemon=True); assembler.start()
+    threads = [threading.Thread(target=downloader, name=f"replay-dl-{i}", daemon=True) for i in range(config.REPLAY_PARALLEL)]
+    for t in threads:
+        t.start()
+    try:
+        while not (stop is not None and stop.is_set()):
+            try:
+                h, tmp, size, dl_s = preloaded.pop(0) if preloaded else ready.get(timeout=5.0)
+            except queue.Empty:
+                replan()
+                if not any(t.is_alive() for t in threads) and ready.empty():
+                    if fresh.empty():
+                        _sleep(300, stop); replan()          # caught up: poll for the next closed hour
+                        if fresh.empty():
+                            continue
+                    threads = [threading.Thread(target=downloader, name=f"replay-dl-{i}", daemon=True) for i in range(config.REPLAY_PARALLEL)]
+                    for t in threads:
+                        t.start()
+                continue
+            replan()
+            if tmp is None or size < 0:
+                _db_retry(stop, "replay_hours write", _put_hour, HOUR_SQL, (h, "missing" if size < 0 else "error", None if size < 0 else "download failed"))
+                status.update(errors=status.s["errors"] + (0 if size < 0 else 1)); continue
+            t0 = time.time()
+            try:
+                trades, events, n = parse_hour(tmp)
+                day = config.REPLAY_DIR / f"{h:%Y-%m-%d}"; day.mkdir(parents=True, exist_ok=True)
+                pq.write_table(trades, day / f"{h:%H}_trades.parquet", compression="zstd"); pq.write_table(events, day / f"{h:%H}_events.parquet", compression="zstd")
+                tmp.unlink()
+                _db_retry(stop, "replay_hours write", _put_hour,
+                          "INSERT INTO replay_hours (hour, status, trades, events, bytes, took_s) VALUES (%s,'done',%s,%s,%s,%s) "
+                          "ON CONFLICT (hour) DO UPDATE SET status = 'done', trades = EXCLUDED.trades, events = EXCLUDED.events, bytes = EXCLUDED.bytes, took_s = EXCLUDED.took_s, last_error = NULL, done_at = now()",
+                          (h, trades.num_rows, events.num_rows, size, dl_s + time.time() - t0))
+                status.update(hours_done=status.s["hours_done"] + 1, trades=status.s["trades"] + trades.num_rows, events=status.s["events"] + events.num_rows,
+                              bytes=status.s["bytes"] + size, last_hour=h.isoformat(), last_parse_s=round(time.time() - t0, 1), last_dl_s=round(dl_s, 1))
+                log.info("hour %s: %d events -> %d trade rows, %d lifecycle rows (%.0f MB, dl %.0fs, parse %.1fs)", h.isoformat(), n, trades.num_rows, events.num_rows, size / 1e6, dl_s, time.time() - t0)
+            except Exception as e:
+                log.exception("parse %s failed", h)
+                tmp.unlink(missing_ok=True)      # the ~400 MB download; replan() refetches the hour
+                _db_retry(stop, "replay_hours write", _put_hour, HOUR_SQL, (h, "error", str(e)[:300]))
+                status.update(errors=status.s["errors"] + 1)
+    finally:
+        if stop is not None:
+            stop.set()
+        status.update(stage="stopped", force=True)
+        record_event("info", "replay", "replay ingest stopped", {k: status.s.get(k) for k in ("hours_done", "hours_total", "trades", "events", "bytes", "errors")})
+        log.info("replay ingest stopped")