diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..6ca94c388d72ef32f5dcf1d9c37d5a5f4ee54e2d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +.venv +.env +data +logs +dist +web +media +tests +.pytest_cache +.ruff_cache +__pycache__ +*.pyc +.agents +.claude diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..264617d6ca5ba414bce24741281aeeae00a55de4 --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# fly-trader distribution -- copy to .env and fill in the two API keys. Everything else is set for a 0.5 SOL wallet. +# Keep values bare: Docker Compose keeps a trailing "# comment" as part of an empty value, so comments live on their own lines. + +# --- keys (never logged) --- +# Helius: https://dashboard.helius.dev (RPC for balances and transactions) +HELIUS_API_KEY= +# Jupiter: https://portal.jup.ag (swaps and token stats) +JUPITER_API_KEY= +# Leave empty. `docker compose exec fly fly-trader wallet new` writes it here. +BOT_PRIVATE_KEY= + +# --- database (docker-compose.yml sets the container's URL; this one is for running outside Docker) --- +DATABASE_URL=postgresql:///fly_trader + +# --- money --- +# Live money. Until the wallet is funded the engine trades paper and says so on the console. +LIVE_ENABLED=1 +# What you fund the wallet with; also the paper warm-up bankroll. +CAPITAL_SOL=0.5 +# Hard cap per position. +MAX_POSITION_SOL=0.05 +# Per position, of the bankroll minus the gas reserve. +MAX_POSITION_FRACTION=0.10 +# Per position, of the pool's SOL reserve (bounds price impact). +MAX_POOL_SHARE=0.02 +# Smaller sizes are skipped. +MIN_POSITION_SOL=0.02 +# Never spent (fees and token-account rent). +GAS_RESERVE_SOL=0.05 +KELLY_FRACTION=0.25 + +# --- rails --- +# 20 % below the wallet's peak: stop entering ... +KILL_SWITCH_DRAWDOWN=0.20 +# ... and sell everything open. Clear it on the console's "Safety & wallet" page or with `fly-trader reset-circuit --kill`. +KILL_SWITCH_LIQUIDATE=1 + +# --- paper warm-up, then live --- +# No minimum racing time; after 20 closed paper trades the fly trades the wallet; no race against the paper selector. +HANDOVER_DAYS=0 +HANDOVER_MIN_TRADES=20 +HANDOVER_BEAT_SELECTOR=0 +# A restart keeps the paper book, so the warm-up count survives it. +RESET_ON_START=0 + +# --- runtime --- +SOLANA_CLUSTER=mainnet-beta +DEVICE=cpu +DATA_DIR=data +LOG_DIR=logs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f5c945c57c476e4913219da39783fbc9ef14da78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +.env +.venv/ +data/ +logs/ +dist/ +models/wallet_skill/*.parquet # ~300 MB: on Hugging Face, fetched by docker/entrypoint.sh +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ +.streamlit/secrets.toml +.DS_Store +.agents/ +.claude/ + +# large media assets - deployed straight to Gandi, not tracked in GitHub source repo +web/assets/*.mp4 +media/*.mp4 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..49e98cfe5442d169d01e07b2c0c911234c80fa01 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# fly-trader: the trading engine and its console, on the CPU (a Linux container has no Metal GPU; the fly infers fine on CPU). +FROM python:3.13-slim + +ENV PYTHONUNBUFFERED=1 \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + PATH="/app/.venv/bin:${PATH}" + +RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +WORKDIR /app +COPY . . +# CPU-only torch first (the default Linux wheel pulls CUDA, ~3 GB); the project's own pin is then already satisfied. +RUN uv venv /app/.venv \ + && uv pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.12" \ + && uv pip install . \ + && chmod +x /app/docker/entrypoint.sh + +EXPOSE 8501 +ENTRYPOINT ["/app/docker/entrypoint.sh"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..44a3e898031d647a079fa3e1e823c1b26ca20696 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 fly-trader contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c81a401ed3b64166193c51cc24a20a39db326aa0 --- /dev/null +++ b/README.md @@ -0,0 +1,150 @@ +--- +license: mit +pipeline_tag: reinforcement-learning +library_name: pytorch +tags: + - solana + - memecoin + - pump.fun + - trading + - connectome + - drosophila + - flywire +--- + +# fly-trader — distribution + +**A fruit-fly connectome that trades Solana memecoins with your money.** + +> ## ⚠️ Read this first +> +> This software signs real transactions from a wallet you fund. It buys tokens that graduated from pump.fun, holds them +> for about two hours, and sells. **It can lose everything in that wallet, quickly.** The model was trained on a few +> months of data; the market it trades is adversarial and changes. Nothing here is a claim of profit, and nothing here is +> financial advice. Run it only with money you can lose entirely, on a wallet used for nothing else. MIT licence — no +> warranty of any kind. See [LICENSE](LICENSE). + +This is the ready-to-run branch: the trading engine, its console, a trained fly, the selector that taught it, and the +connectome they run on. The training pipeline lives on [`master`](https://github.com/bryceweiner/fly-trader). + +## What it does + +- Every minute it scores every PumpSwap token that graduated from pump.fun at least six hours ago, from live 1-minute + candles (a token needs that much history before its features mean anything). +- The fly — the FlyWire *Drosophila* central brain (41,756 neurons) as a rate network on its real signed synapses — + predicts each token's net return over the next 120 minutes. It was taught by a gradient-boosted selector; it then keeps + learning from what its own trades earn, through plasticity at its mushroom body's KC→MBON synapses. +- It buys when the predicted return clears its own buy line, sizes the position from measured certainty (a quarter + of the growth-optimal bet, capped), holds 120 minutes, sells. +- **It starts on paper.** After 20 closed paper trades it switches itself to the wallet and stays there. Before that, + and whenever the wallet is unfunded or a key is missing, it trades paper and says so on the console. +- **Rails:** 20 % below the wallet's peak it stops entering and sells every open position (the kill switch); three + failed transactions in a row trip a circuit breaker; a stale market feed halts entries; if live fills trail the paper + mirror by more than 2 points per trade over 50 trades, entries pause and it tells you. + +## Requirements + +- Docker (Desktop on macOS/Windows, or Engine + Compose v2 on Linux). ~2 GB of disk for the image and database. +- A [Helius](https://dashboard.helius.dev) API key (free tier is enough) and a [Jupiter](https://portal.jup.ag) API key. +- 0.5 SOL you are prepared to lose. Not more: the caps below are sized for 0.5. +- Any CPU. No GPU is used. + +## Start trading + +1. Download this branch and open a terminal in it: + ```bash + git clone -b distribution https://github.com/bryceweiner/fly-trader.git && cd fly-trader + ``` +2. Create your `.env` and paste the two API keys into it: + ```bash + cp .env.example .env + ``` +3. Start it: + ```bash + docker compose up -d --build + ``` + The first start builds the image (several minutes), creates the database, installs the models and starts the market + feed and the trading engine. +4. Create the bot wallet — it prints the public key and writes the secret into your `.env`: + ```bash + docker compose exec fly fly-trader wallet new + ``` + **Back up `.env` now.** It is the only copy of that key. +5. Send **0.5 SOL** to the printed address, then restart the engine so it picks up the key: + ```bash + docker compose restart fly + ``` +6. Open the console: . Watch the **Overview** page. + +That is all. From here it runs on its own. The first ~24 hours are a warm-up (it needs a day of candles before it +scores anything); then it trades paper until 20 trades have closed; then it trades the wallet. + +## Day to day + +| Want to | Do | +|---|---| +| See what it is doing | — **Overview** (books, open positions, wealth), **Trades**, **Safety & wallet** (rails, wallet), **Processes** | +| Pause new entries | Safety & wallet → **Pause entries**, or `docker compose exec fly fly-trader pause-entries` | +| Clear a tripped kill switch | Safety & wallet → **Clear kill switch**, or `docker compose exec fly fly-trader reset-circuit --kill` (re-bases the peak) | +| Stop everything | `docker compose down` — **open positions stay open in the wallet**; start again and it sells them on schedule, or sell them yourself | +| See the logs | `docker compose logs -f fly` | +| Start again from nothing | `docker compose down -v` (deletes the database) and delete `data/` | + +The console listens on `127.0.0.1` only. Do not expose it: it can start and stop the engine and clear the rails. + +## What is in the box + +| Path | What | +|---|---| +| `models/policies/fly_*.pt` | the fly (`brain_snapshots` #62): taught by selector #59, two strategies (`ev`, `capitulation`), 120-minute hold, its own buy lines | +| `models/selectors/selector_*.joblib` | the selector that taught it (#59), kept so the console can show the paper selector alongside | +| `models/connectome/` | the FlyWire FAFB v783 central brain as the fly runs on it (14 MB), with its calibrated scale | +| `models/wallet_skill/` | the fitted wallet-skill inputs: `config.json` (the horizon and lookback the model was trained with) and the name of the newest per-day table. The table itself (~300 MB) is on Hugging Face; the container fetches it once at first start | +| `seed/seed.sql` | the database rows that admit the fly to trade: the model records, the passed replay verdict, autostart | +| `.env.example` | every knob, set for 0.5 SOL | +| `docker-compose.yml`, `Dockerfile`, `docker/entrypoint.sh` | one Postgres, one engine + console | + +The fly's replay verdict shipped here: over the 28 evaluation days it made +1.52 % per trade after real costs (profit +factor 1.13) against −5.27 % for random picks of the same tokens. That is a backtest on one corpus, not a forecast. + +## Tuning + +Edit `.env`, then `docker compose restart fly`. The ones that matter: + +| Variable | Shipped | Meaning | +|---|---|---| +| `CAPITAL_SOL` | 0.5 | what you funded; also the paper warm-up bankroll | +| `MAX_POSITION_SOL` / `MAX_POSITION_FRACTION` | 0.05 / 0.10 | per-position caps (the smaller wins) | +| `GAS_RESERVE_SOL` | 0.05 | never spent | +| `KILL_SWITCH_DRAWDOWN` / `KILL_SWITCH_LIQUIDATE` | 0.20 / 1 | drawdown from peak that halts and sells everything | +| `HANDOVER_MIN_TRADES` | 20 | closed paper trades before it goes live | +| `LIVE_ENABLED` | 1 | set 0 to run paper only | + +Scaling the wallet up scales the position caps with it (`MAX_POSITION_FRACTION`), but `MAX_POSITION_SOL` is a hard +ceiling: raise it deliberately or not at all. + +## Refreshing the model + +The model is a snapshot of one training run. To ship a newer one, on a `master` checkout with its database, after a +passed replay: + +```bash +.venv/bin/python tools/make_seed.py --out dist +cp -R dist/models/* models/ && cp -R dist/seed/* seed/ +``` + +Commit `models/` and `seed/` on this branch. The seed is idempotent: an install that already has these rows is not +changed by it, so pull the branch, `docker compose up -d --build`, and the new model is in place. + +## Also on Hugging Face + +- Model and this code: [`bryceweiner/fly-trader`](https://huggingface.co/bryceweiner/fly-trader) +- The training database (the full corpus, for retraining on `master`): [`bryceweiner/fly-trader-seed`](https://huggingface.co/datasets/bryceweiner/fly-trader-seed) + +## Credits + +FlyWire consortium, Dorkenwald et al., *Neuronal wiring diagram of an adult brain*, Nature 2024; annotations Schlegel +et al. 2024; synapse signs Shiu et al. 2024. Connectome export: [snedea/flybrain](https://github.com/snedea/flybrain) +(MIT). Market data: [PumpAPI](https://pumpapi.io); routing: [Jupiter](https://jup.ag). + +[MIT](LICENSE). diff --git a/config/mbon_valence.json b/config/mbon_valence.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9b5e0d2cd5db6b1eb070482aac2c21efd9a46e --- /dev/null +++ b/config/mbon_valence.json @@ -0,0 +1,27 @@ +{ + "_citation": "Approach = MBON-11 (gamma1pedc>alpha/beta, GABA), 12, 13, 14, 15, 16, 18 (cholinergic) and avoidance = MBON-01..07 (all glutamatergic): Aso et al. 2014 eLife 3:e04580 (activation screen; 'all MBONs eliciting aversion were glutamatergic and all the MBONs eliciting attraction were either GABAergic or cholinergic'); NT per type from Aso et al. 2014 eLife 3:e04577 Table 1. MBON17 (alpha'3m) was tested jointly with MBON16 (alpha'3ap) in that screen. MBON19 and MBON23 are NOT in these lists: MBON19's NT is N.D. in Aso 2014 and MBON23 is a later hemibrain type (Li et al. 2020). MBON21, MBON29 (cholinergic, aversive) and MBON33 (attractive) per Rubin & Aso 2024 eLife 90523 (activation results read from partial text; verify). '-like' annotations are left unassigned (no valence evidence). PAM = reward, PPL1 = punishment: Aso et al. 2014 eLife 3:e04577 compartment map; Li et al. 2020 ('PPL1 DANs generally convey punishment, PAM DANs generally convey reward'). Learning = depression of opposite-valence KC->MBON synapses: Hige et al. 2015 Neuron, Owald et al. 2015 Neuron.", + "approach": [ + "MBON11", + "MBON12", + "MBON13", + "MBON14", + "MBON15", + "MBON16", + "MBON17", + "MBON18", + "MBON33" + ], + "avoid": [ + "MBON01", + "MBON02", + "MBON03", + "MBON04", + "MBON05", + "MBON06", + "MBON07", + "MBON21", + "MBON29" + ], + "dan_reward_prefix": "PAM", + "dan_punishment_prefix": "PPL1" +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..992f1c82de6332239d4dd3f0dde340c207569ad5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +# fly-trader, fire-and-forget: one Postgres, one trading container. The console is published on 127.0.0.1 only -- +# it can start and stop the workers and clear the kill switch, so it must never face the internet. +services: + db: + image: postgres:17 + environment: + POSTGRES_USER: fly + POSTGRES_PASSWORD: fly + POSTGRES_DB: fly_trader + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U fly -d fly_trader"] + interval: 5s + timeout: 3s + retries: 30 + restart: unless-stopped + + fly: + build: . + env_file: .env + environment: + DATABASE_URL: postgresql://fly:fly@db:5432/fly_trader + DATA_DIR: /app/data + LOG_DIR: /app/logs + DEVICE: cpu + volumes: + - ./data:/app/data # models, the fly's plastic state, its hourly snapshots: survives restarts and rebuilds + - ./logs:/app/logs + - ./.env:/app/.env # `fly-trader wallet new` writes the bot key here + ports: + - "127.0.0.1:8501:8501" + depends_on: + db: + condition: service_healthy + restart: unless-stopped + +volumes: + pgdata: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..b8c002d7b2c3b38a870e2728b3b3c781be708018 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# First boot: schema, the shipped models into the data volume, the seed rows that admit them. Every boot: the console, +# which autostarts the two trading workers (market feed + trading engine). Nothing here overwrites a running install. +set -e +cd /app + +until pg_isready -d "$DATABASE_URL" >/dev/null 2>&1; do + echo "fly-trader: waiting for postgres"; sleep 2 +done + +fly-trader init-db + +for sub in policies selectors connectome; do + mkdir -p "data/brain/$sub" + for f in models/$sub/*; do + [ -e "$f" ] || continue + [ -e "data/brain/$sub/$(basename "$f")" ] || cp "$f" "data/brain/$sub/" + done +done + +# The wallet-skill inputs: the fitted config (in git) and the newest table (~300 MB: kept on Hugging Face, fetched once). +mkdir -p data/corpus/wallet_skill +for f in models/wallet_skill/*; do + [ -e "$f" ] || continue + [ -e "data/corpus/wallet_skill/$(basename "$f")" ] || cp "$f" data/corpus/wallet_skill/ +done +T=$(cat models/wallet_skill/TABLE 2>/dev/null || true) +if [ -n "$T" ] && [ ! -e "data/corpus/wallet_skill/$T" ]; then + echo "fly-trader: fetching the wallet-skill table $T (~300 MB, once)" + curl -fsSL --retry 5 -o "data/corpus/wallet_skill/$T.part" "https://huggingface.co/bryceweiner/fly-trader/resolve/main/models/wallet_skill/$T" \ + && mv "data/corpus/wallet_skill/$T.part" "data/corpus/wallet_skill/$T" \ + || echo "fly-trader: WARNING could not fetch the wallet-skill table; the paper selector will not trade and the fly's skill inputs read as zero until it is present" +fi + +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f seed/seed.sql + +echo "fly-trader: console at http://localhost:8501" +exec fly-trader ui --port "${PORT:-8501}" diff --git a/fly_trader/__init__.py b/fly_trader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..21f1ce38fd7c6ba7ca244e19dbcd08c2ecf7b3c3 --- /dev/null +++ b/fly_trader/__init__.py @@ -0,0 +1,3 @@ +"""fly-trader: a FlyWire-connectome memecoin trader on Solana.""" + +__version__ = "0.1.0" diff --git a/fly_trader/__main__.py b/fly_trader/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..e751e949cf2c7e004db4c1933c8260a8e8890de8 --- /dev/null +++ b/fly_trader/__main__.py @@ -0,0 +1,229 @@ +"""fly-trader command line. Every command is a thin wrapper over library functions that the +Streamlit console also calls. Always run as ``.venv/bin/python -m fly_trader ``.""" +from __future__ import annotations + +import argparse +import json +import sys + + +def _cmd_init_db(args): + from .db import schema + from .db.connection import database_name + created = schema.ensure_database() + version = schema.apply_schema() + tables = schema.table_names() + print(f"database={database_name()} created={created} schema_version={version} tables={len(tables)}") + for t in tables: + print(f" {t}") + + +def _cmd_wallet(args): + from .chain import keys + if args.action == "new": + print(keys.create_wallet()) + else: + print(keys.show_wallet()) + + +def _cmd_discover(args): + from .ingest import discovery + if args.once: + discovery.run_once() + else: + discovery.run_forever() + + +def _cmd_build_connectome(args): + from .brain import connectome_build + connectome_build.main(annotations=args.annotations) + + +def _cmd_run(args): + from .agent import runner + runner.main() + + +def _cmd_swap_smoke(args): + from .execution import broker_live + broker_live.swap_smoke(args.sol) + + +def _cmd_status(args): + from .db import queries + queries.print_status() + + +def _cmd_reset_circuit(args): + from .agent import rails + rails.reset_circuit(kill=args.kill) + + +def _cmd_pause(args): + from .agent import rails + rails.set_entries_paused(True) + + +def _cmd_resume(args): + from .agent import rails + rails.set_entries_paused(False) + + +def _cmd_reset_training(args): + from .ops.reset import reset_training_state + import json + print(json.dumps(reset_training_state(reason="cli", archive=not args.no_archive), indent=1)) + + +def _cmd_ui(args): + from .ops.supervisor import run_console + run_console(port=args.port) + + +def _cmd_verify_fills(args): + from .execution import ledger + ledger.verify_fills() + + +def _cmd_close_atas(args): + from .execution import broker_live + broker_live.close_empty_atas() + + +def _cmd_replay_pull(args): + from .ingest import replay_pull + replay_pull.main() + + +def _cmd_pumpstream(args): + from .ingest import pumpstream + pumpstream.main() + + +def _cmd_build_corpus_features(args): + from .train import corpus_features + corpus_features.main() + + +def _cmd_refetch_pool_ids(args): + from .ingest import replay_pull + print(f"{replay_pull.refetch_missing_pool_id()} archive hours marked for re-download (run the replay worker or fly-trader replay-pull)") + + +def _cmd_fit_wallet_skill(args): + from .logging_setup import setup + from .train import wallet_skill + setup("train") + out = wallet_skill.fit(days=args.days) + print(json.dumps({k: out.get(k) for k in ("h", "L", "selection", "evaluation", "baseline", "beats_baseline", "candidates", "stopped")}, indent=1, default=str)) + + +def _cmd_train_selector(args): + from .logging_setup import setup + from .train import selector + setup("train") + selector.main(days=args.days) + + +def _cmd_train_fly_selector(args): + from .logging_setup import setup + from .train import fly_selector + setup("train") + fly_selector.main(days=args.days, epochs=args.epochs) + + +def _cmd_fly_replay(args): + from .logging_setup import setup + from .train import fly_replay + setup("train") + out = fly_replay.main(days=args.days, start_day=args.start_day if args.start_day is not None else fly_replay.DEFAULT_START) + print(json.dumps({k: out.get(k) for k in ("S", "passed", "reason", "alpha", "half_life_days", "evaluation", "random", "frozen", "plastic_beats_frozen")}, indent=1, default=str)) + + +def _cmd_selector_eval(args): + import runpy + runpy.run_module("fly_trader.train.selector_eval", run_name="__main__") + + +def _cmd_build_mature(args): + from .train import mature + mature.main() + + +def _cmd_build_corpus_meta(args): + from .train import corpus_meta + corpus_meta.main() + + +def _cmd_assemble_replay(args): + from .train import replay_assemble + replay_assemble.main() + + +def _cmd_backtest_corpus(args): + from .train import corpus_backtest + argv = ["--fees", args.fees] + if args.max_tokens: argv += ["--max-tokens", str(args.max_tokens)] + if args.days: argv += ["--days", str(args.days)] + if args.only: argv += ["--only", args.only] + argv += ["--universe", args.universe] + corpus_backtest.main(argv) + + +def _cmd_worker(args): + from .ops import procs + procs.worker_cli(args.action, args.name) + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="fly-trader") + sub = p.add_subparsers(dest="command", required=True) + sub.add_parser("init-db").set_defaults(fn=_cmd_init_db) + w = sub.add_parser("wallet"); w.add_argument("action", choices=["new", "show"]); w.set_defaults(fn=_cmd_wallet) + d = sub.add_parser("discover"); d.add_argument("--once", action="store_true"); d.set_defaults(fn=_cmd_discover) + sub.add_parser("replay-pull").set_defaults(fn=_cmd_replay_pull) + sub.add_parser("refetch-pool-ids", help="mark archive hours without pool_id for re-download").set_defaults(fn=_cmd_refetch_pool_ids) + fw = sub.add_parser("fit-wallet-skill", help="fit the wallet-skill tables' hold and lookback by the selector's objective (one-off, many hours, resumable)") + fw.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)"); fw.set_defaults(fn=_cmd_fit_wallet_skill) + sub.add_parser("pumpstream").set_defaults(fn=_cmd_pumpstream) + sub.add_parser("build-corpus-features").set_defaults(fn=_cmd_build_corpus_features) + sub.add_parser("assemble-replay").set_defaults(fn=_cmd_assemble_replay) + sub.add_parser("build-corpus-meta").set_defaults(fn=_cmd_build_corpus_meta) + sub.add_parser("build-mature").set_defaults(fn=_cmd_build_mature) + sub.add_parser("selector-eval").set_defaults(fn=_cmd_selector_eval) + ts_ = sub.add_parser("train-selector"); ts_.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)") + ts_.set_defaults(fn=_cmd_train_selector) + tf_ = sub.add_parser("train-fly-selector"); tf_.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)") + tf_.add_argument("--epochs", type=int, default=None, help="cap on full passes over the distillation rows (default: until the fit stops improving)") + tf_.set_defaults(fn=_cmd_train_fly_selector) + fr = sub.add_parser("fly-replay", help="bootstrap the fly once, let it learn over the corpus, judge it (the gate before it trades)") + fr.add_argument("--days", type=int, default=None, help="most recent days only (default: the whole corpus)") + fr.add_argument("--start-day", type=int, default=None, + help="day index of the bootstrap (the selector teaches on the days before it); default: train/fly_replay.DEFAULT_START, " + "the earliest day that leaves the teacher's walk-forward enough days in each half") + fr.set_defaults(fn=_cmd_fly_replay) + bc = sub.add_parser("backtest-corpus"); bc.add_argument("--max-tokens", type=int, default=None); bc.add_argument("--days", type=int, default=None) + bc.add_argument("--fees", default="0.003,0.0055"); bc.add_argument("--only", default=None); bc.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); bc.set_defaults(fn=_cmd_backtest_corpus) + b = sub.add_parser("build-connectome"); b.add_argument("--annotations", default=None); b.set_defaults(fn=_cmd_build_connectome) + sub.add_parser("run").set_defaults(fn=_cmd_run) + s = sub.add_parser("swap-smoke"); s.add_argument("--sol", type=float, default=0.01); s.set_defaults(fn=_cmd_swap_smoke) + sub.add_parser("status").set_defaults(fn=_cmd_status) + r = sub.add_parser("reset-circuit"); r.add_argument("--kill", action="store_true"); r.set_defaults(fn=_cmd_reset_circuit) + sub.add_parser("pause-entries").set_defaults(fn=_cmd_pause) + sub.add_parser("resume-entries").set_defaults(fn=_cmd_resume) + u = sub.add_parser("ui"); u.add_argument("--port", type=int, default=8501); u.set_defaults(fn=_cmd_ui) + rt = sub.add_parser("reset-training"); rt.add_argument("--no-archive", action="store_true"); rt.set_defaults(fn=_cmd_reset_training) + sub.add_parser("verify-fills").set_defaults(fn=_cmd_verify_fills) + sub.add_parser("close-empty-atas").set_defaults(fn=_cmd_close_atas) + wk = sub.add_parser("worker"); wk.add_argument("action", choices=["start", "stop", "status"]); wk.add_argument("name", nargs="?"); wk.set_defaults(fn=_cmd_worker) + return p + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + rc = args.fn(args) + return rc if isinstance(rc, int) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fly_trader/agent/__init__.py b/fly_trader/agent/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/agent/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/agent/fly_live.py b/fly_trader/agent/fly_live.py new file mode 100644 index 0000000000000000000000000000000000000000..42085295665a737db31e4ad2ef5ed7b0e6eb08a1 --- /dev/null +++ b/fly_trader/agent/fly_live.py @@ -0,0 +1,159 @@ +"""Live execution for the fly once it holds the selector's seat: its decisions also trade the bot wallet (book 'live'). + +The paper book ``paper_fly`` keeps running as the mirror. Each trade minute — only after the handover and only while +signing is allowed (LIVE_ENABLED=1 and the live prerequisites, chain/cluster_guard.py) — ``LiveMirror.minute``: +- drains the execution worker (execution/worker.py books live positions from balance-verified fills); +- exits: live positions held the hold are sold (exit slippage, the forced ladder once ``OVERDUE_S`` late), retried every + minute; ``DEAD_BAG_S`` past the hold a position that could not be sold is written off (dead-bag: closed at 0, so it + leaves wealth) and its tokens are swept every ``SWEEP_S``; +- entries: every paper entry of this minute is mirrored with the same score, line and certainty bands, sized from the + wallet (SOL balance + open live positions at cost as the bankroll, the SOL balance as cash; the gas reserve is never + spent — agent/sizing.py), one position per token, unless the kill switch, the circuit breaker or paused entries block; +- wealth: wallet SOL + open positions net of exit cost → ``wealth_marks`` (book 'live'); the kill switch on its own peak, + and with ``KILL_SWITCH_LIQUIDATE`` every open position is sold at the forced slippage while it stays on; +- gap: over the last ``GAP_TRADES`` closed live trades, the mean return per trade trailing the paper mirror's same + decisions by more than ``GAP_MAX`` pauses entries and raises an alert. Learning is unaffected: it learns from market + labels, not fills. +""" +from __future__ import annotations + +import json +import logging +from datetime import datetime, timedelta, timezone + +import numpy as np + +from .. import config +from ..chain.balances import snapshot_balances +from ..db.apilog import record_event +from ..execution import ledger +from ..execution.worker import ExecRequest +from ..market.exit_cost import exit_cost_fraction +from . import rails, sizing + +log = logging.getLogger(__name__) +BOOK, MIRROR = "live", "paper_fly" +DEAD_BAG_S, OVERDUE_S, SWEEP_S = 3 * 3600.0, 600.0, 3600.0 +GAP_TRADES, GAP_MAX = 50, 0.02 + + +class LiveMirror: + def __init__(self, horizon_s: float, broker=None, worker=None): + if broker is None: + from ..execution.broker_live import LiveBroker + broker = LiveBroker() + if worker is None: + from ..execution.worker import ExecutionWorker + worker = ExecutionWorker(broker) + self.broker, self.worker, self.H = broker, worker, float(horizon_s) + self.last_sweep = 0.0 + + def _decision(self, conn, ctx, run_id, beat_id, p, reason: str, forced: bool) -> int: + return int(conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, size_sol, forced, reason, detail) VALUES (%s,%s,%s,%s,%s,'fly_exit',%s,%s,%s,%s) RETURNING id", + (beat_id, run_id, ctx.m1, p["mint"], p["pool"], float(p["cost_sol"]), forced, reason, json.dumps({"book": BOOK}))).fetchone()["id"]) + + def minute(self, ctx, *, run_id: str, beat_id: int, entries: list, line: float, table: list | None) -> dict: + conn, m1 = ctx.conn, ctx.m1 + drained = self.worker.drain() + for r in drained: + if not r.ok: + log.warning("live %s %s failed: %s", r.request.side, r.request.mint, r.error) + inflight = self.worker.pending() + snap = snapshot_balances(self.broker.rpc, self.broker.pubkey); sol_free = snap.lamports / config.LAMPORTS_PER_SOL + n_exit = n_dead = 0 + for p in ledger.open_positions(conn, BOOK): + held = (m1 - p["opened_at"]).total_seconds(); H = float(p.get("hold_s") or self.H) # each position's own hold + if held < H or p["mint"] in inflight: + continue + if held >= H + DEAD_BAG_S: + did = self._decision(conn, ctx, run_id, beat_id, p, f"dead-bag: unsold {held / 3600:.1f} h after entry", True) + ledger.close_position(conn, position_id=int(p["id"]), exit_price=0.0, proceeds_sol=0.0, fees_sol=0.0, decision_id=did, forced_kind="dead_bag", ts=m1) + record_event("warning", "fly_live", f"dead-bag: {p['mint']} could not be sold; written off, its tokens are swept hourly", {"position": int(p["id"]), "cost_sol": float(p["cost_sol"])}) + n_dead += 1 + continue + overdue = held >= H + OVERDUE_S + did = self._decision(conn, ctx, run_id, beat_id, p, f"held {held / 60:.0f} min" + (" (retrying)" if overdue else ""), False) + n_exit += bool(self.worker.submit(ExecRequest(decision_id=did, mint=p["mint"], pool=p["pool"], side="sell", amount_in=int(p["qty"]), + slippage_bps=config.SLIPPAGE_FORCED_BPS if overdue else config.SLIPPAGE_EXIT_BPS, + max_slippage_bps=config.SLIPPAGE_FORCED_BPS, decimals=int(p.get("decimals") or 6), position_id=int(p["id"])))) + opens = ledger.open_positions(conn, BOOK); held_mints = {p["mint"] for p in opens} | set(inflight) + c = rails.load_circuit(conn) + blocked = "kill switch" if c.kill_switch else "circuit tripped" if c.tripped else "paused" if c.entries_paused else None + bankroll = sol_free + sum(float(p["cost_sol"]) for p in opens); cash = sol_free; n_enter = 0; skipped = [] + for e in entries: + if e["mint"] in held_mints: + skipped.append((e["mint"], "held")); continue + size, why = sizing.size_position(e["score"], e.get("threshold", line), e.get("table") or table or [], bankroll, cash, e["info"]["resq"]) + if blocked or size <= 0: + skipped.append((e["mint"], blocked or why)); continue + if self.worker.submit(ExecRequest(decision_id=e["decision_id"], mint=e["mint"], pool=e["info"]["pool"], side="buy", amount_in=int(size * config.LAMPORTS_PER_SOL), + slippage_bps=config.SLIPPAGE_ENTRY_BPS, max_slippage_bps=config.MAX_SLIPPAGE_ENTRY_BPS, decimals=int(e["info"]["decimals"] or 6), size_sol=size)): + n_enter += 1; cash -= size; held_mints.add(e["mint"]) + conn.execute("UPDATE decisions SET book_targets = array_append(COALESCE(book_targets, '{}'), 'live') WHERE id = %s", (e["decision_id"],)) + ledger.mark_positions(conn, BOOK, ctx.prices, {}, m1) + gross_v = exit_cost = exposure = 0.0 + for p in ledger.open_positions(conn, BOOK): + px = float(p.get("last_mark_price") or p["entry_price"] or 0.0) + gross = int(p["qty"]) / 10 ** int(p.get("decimals") or 6) * px + gross_v += gross; exposure += float(p["cost_sol"]) + exit_cost += gross * exit_cost_fraction(gross, ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"]), ctx.mcap(p["mint"], px), p.get("program_label"), ctx.fees.get(p["mint"])) + wealth = sol_free + gross_v - exit_cost + pk = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (BOOK,)).fetchone(); peak = max(float(pk["pk"] or 0.0), wealth) + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (beat_id, book) DO NOTHING", + (beat_id, BOOK, m1, sol_free, gross_v - exit_cost, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens))) + killed = rails.check_drawdown(conn, wealth, peak) + n_liq = self.liquidate(conn, ctx, run_id, beat_id) if killed and config.KILL_SWITCH_LIQUIDATE else 0 + gap = self.gap(conn) + if ctx.m1_epoch - self.last_sweep >= SWEEP_S: + self.last_sweep = ctx.m1_epoch; self.sweep(conn, snap, inflight, m1) + return {"stage": "trading live", "wallet_sol": sol_free, "wealth": wealth, "entered": n_enter, "exited": n_exit, "dead_bags": n_dead, "in_flight": len(inflight), + "blocked": blocked, "skipped": skipped[:10], "gap": gap, "failed_orders": sum(1 for r in drained if not r.ok), "liquidated": n_liq} + + def liquidate(self, conn, ctx, run_id: str, beat_id: int) -> int: + """Sell every open live position at the forced slippage while the kill switch is on (``KILL_SWITCH_LIQUIDATE``): + a book that has lost ``KILL_SWITCH_DRAWDOWN`` of its peak stops holding, not only entering. Called every trade + minute the switch stays on, so a sell that fails is retried until the book is flat; positions already in flight + (this minute's scheduled exits included) are left to the worker.""" + n = 0; inflight = self.worker.pending() + for p in ledger.open_positions(conn, BOOK): + if p["mint"] in inflight: + continue + did = self._decision(conn, ctx, run_id, beat_id, p, "kill switch: liquidating", True) + n += bool(self.worker.submit(ExecRequest(decision_id=did, mint=p["mint"], pool=p["pool"], side="sell", amount_in=int(p["qty"]), + slippage_bps=config.SLIPPAGE_FORCED_BPS, max_slippage_bps=config.SLIPPAGE_FORCED_BPS, + decimals=int(p.get("decimals") or 6), position_id=int(p["id"])))) + if n: + record_event("error", "fly_live", f"kill switch: liquidating {n} open live position(s)", {"positions": n}) + return n + + def gap(self, conn) -> dict: + """Live vs its paper mirror on the same decisions; pauses entries when live trails by more than ``GAP_MAX`` per trade.""" + rows = conn.execute(f"""SELECT l.realized_sol / NULLIF(l.cost_sol, 0) AS rl, p.realized_sol / NULLIF(p.cost_sol, 0) AS rp + FROM positions l JOIN positions p ON p.entry_decision_id = l.entry_decision_id AND p.book = '{MIRROR}' AND p.status = 'closed' + WHERE l.book = '{BOOK}' AND l.status = 'closed' AND l.entry_decision_id IS NOT NULL ORDER BY l.closed_at DESC LIMIT %s""", (GAP_TRADES,)).fetchall() + rl = np.array([r["rl"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64) + rp = np.array([r["rp"] for r in rows if r["rl"] is not None and r["rp"] is not None], dtype=np.float64) + out = {"trades": int(len(rl)), "live_mean": float(rl.mean()) if len(rl) else None, "mirror_mean": float(rp.mean()) if len(rp) else None} + if len(rl) >= GAP_TRADES and rl.mean() - rp.mean() < -GAP_MAX: + c = rails.load_circuit(conn) + if not c.entries_paused: + conn.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = 1") + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('gap_pause', %s)", (json.dumps(out),)) + record_event("error", "fly_live", f"entries paused: live trails its paper mirror by {(rp.mean() - rl.mean()) * 100:.1f} points per trade over {len(rl)} trades", out) + out["paused"] = True + return out + + def sweep(self, conn, snap, inflight: set, m1: datetime) -> int: + """Sell whatever is left of the last week's dead-bag tokens (forced slippage; proceeds reach the wallet).""" + n = 0 + for r in conn.execute("SELECT DISTINCT mint FROM positions WHERE book = %s AND forced_exit_kind = 'dead_bag' AND closed_at > %s", + (BOOK, m1 - timedelta(days=7))).fetchall(): + amt = snap.token(r["mint"]) + if amt > 0 and r["mint"] not in inflight: + n += bool(self.worker.submit(ExecRequest(decision_id=None, mint=r["mint"], pool=None, side="sell", amount_in=amt, slippage_bps=config.SLIPPAGE_FORCED_BPS, + max_slippage_bps=config.SLIPPAGE_FORCED_BPS, decimals=int(snap.decimals.get(r["mint"], 6)), forced_kind="dead_bag_sweep"))) + return n + + def stop(self) -> None: + self.worker.stop() diff --git a/fly_trader/agent/fly_session.py b/fly_trader/agent/fly_session.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7a93ca6e14900e383d5ecd5ae560c12b957f24 --- /dev/null +++ b/fly_trader/agent/fly_session.py @@ -0,0 +1,510 @@ +"""The plastic fly: trading its own book (``paper_fly``) on the minute engine and learning from the market every minute. + +It starts once the fly design passed its replay (``train/fly_replay.verdict``, current definitions) and a deployable +bootstrap exists (``train/fly_selector.latest_deployable``); the replay's chosen configuration (α, half-life) is the +learning rate and forgetting of its mushroom body (``brain/plastic.py``). Each minute (agent/minute_engine.py): + +1. bars: minute start, open, close and exit cost of every traded mint with a pending tag feed the label resolver; +2. labels: tags whose exit minute has now been seen are resolved exactly as ``train/decisions.build`` labels a row + (``resolve_label``) and teach the mushroom body; a label that cannot be resolved is dropped; +3. calibration: at the first minute of each UTC day, the line and sizing of the plastic fly and of its frozen shadow + (the bootstrap, D = 0) from their scores of the minutes resolved in the last seven days (``train/fly_calibrate.py``); +4. scoring: eligible minutes are scored by the plastic fly and its shadow, stored (``fly_scored``: pending rows keep the + feature vector, so tags survive restarts) and tagged; only minutes at or above the line teach it (brain/plastic.py); +5. trading (fresh stream, trade minutes): the race rules (``agent/paper_trading.py``) at the plastic line and sizing. + +Hourly: learning statistics (``fly_updates``), a snapshot of the plastic state (kind 'fly_plastic'; hourly for 7 days, +then one per day), the rollback checks (``train/fly_governance.py``; a rollback restores the newest good snapshot at +least a day old, or D = 0, pauses the checks a day, and the third within 7 days freezes learning and requests a +re-bootstrap), and the handover (``HANDOVER_DAYS`` of racing, at least ``HANDOVER_MIN_TRADES`` trades, realized P&L +at least the selector's). The state file ``data/brain/plastic/state.pt`` is written atomically every minute, before +tags are marked resolved: on restart, pending tags at or before its watermark count as applied. +""" +from __future__ import annotations + +import json +import logging +import math +import os +import time +import uuid +from collections import deque +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import torch + +from .. import config +from ..brain import plastic +from ..db.apilog import record_event +from ..db.connection import transaction +from ..execution import ledger +from ..execution.broker_paper import PaperBroker +from ..ops.reset import fly_state_dir +from ..train import fly_calibrate, fly_governance as gov, fly_selector +from ..train.decisions import X_COLS +from . import handover, paper_trading + +log = logging.getLogger(__name__) +BOOK, KIND = "paper_fly", "fly" +BAR_KEEP = 270 # minutes of bars kept per watched mint (the longest strategy hold, 240, + next-open window + margin) +NEXT_OPEN_S = 120.0 # entry at the next traded minute's open when it comes within two minutes (decisions.build) +JUMP = 50.0 +LABEL_LAG_S = 60.0 +ROLLBACK_PAUSE_S, ROLLBACK_LIMIT, ROLLBACK_WINDOW_S, ROLLBACK_MIN_AGE_S = 86400.0, 3, 7 * 86400.0, 86400.0 +SNAP_HOURLY_DAYS, SCORED_KEEP_DAYS = 7, 35 +COMMAND_KEY = "fly_command" + + +def send_command(cmd: str) -> None: + """Queue an operator command for the running fly (applied at its next minute): 'pause', 'resume' or 'rollback'.""" + assert cmd in ("pause", "resume", "rollback"), cmd + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (COMMAND_KEY, json.dumps({"cmd": cmd, "at": datetime.now(timezone.utc).isoformat()}))) + + +def resolve_label(bars: list, t: float, horizon_s: float) -> float | None: + """Net return of a row scored at minute start ``t`` from the mint's bars [(minute start, open, close, exit cost)] + in time order — ``decisions.build``'s ``fwd_pess``: entry at the next traded minute's open when it comes within + ``NEXT_OPEN_S`` (else the close), exit at the last close at or before ``t + horizon_s``, the one-side exit cost at + both ends. None when the row's own bar is missing, a price is not positive, or the exit minute jumped 50× from the + one before (the archive drops such exits).""" + rows = [b for b in bars if b[0] >= t] + if not rows or rows[0][0] != t: + return None + j = 0 + for i, b in enumerate(rows): + if b[0] <= t + horizon_s: + j = i + else: + break + entry = rows[1][1] if len(rows) > 1 and rows[1][0] - t <= NEXT_OPEN_S and rows[1][1] else rows[0][2] + exit_px = rows[j][2] + if not entry or entry <= 0 or not exit_px or exit_px <= 0: + return None + if j > 0 and rows[j - 1][2] and not (1 / JUMP <= exit_px / rows[j - 1][2] <= JUMP): + return None + return float(exit_px / entry * (1 - rows[0][3]) * (1 - rows[j][3]) - 1) + + +def try_start(live: bool = False) -> tuple["FlyBook | None", str]: + """A FlyBook when the fly may trade, else (None, why).""" + from ..train import fly_replay + v = fly_replay.verdict() + if not v: + return None, "the fly's replay has not run on the current definitions (fly-trader fly-replay)" + if not v.get("passed"): + return None, f"the fly's replay did not pass: {v.get('reason')}" + with transaction() as conn: + boot = fly_selector.latest_deployable(conn) + if boot is None: + return None, "no deployable fly bootstrap (the training pipeline bootstraps one)" + try: + return FlyBook(boot, v, live=live), "ok" + except Exception as e: + log.exception("fly book failed to start") + return None, f"fly book failed to start: {type(e).__name__}: {e}" + + +class FlyBook: + name = "fly" + + def __init__(self, boot: dict, verdict: dict, live: bool = False): + self.live = live; self.done = False; self.mirror = None + self.verdict = verdict + self.state_path = fly_state_dir() / "state.pt" + self._load_bootstrap(boot) + self.run_id = str(uuid.uuid4()); self.beat_no = 0; self.broker = PaperBroker(BOOK) + with transaction() as conn: + conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')", + (self.run_id, KIND, json.dumps({"channels": self.cfg, "book": BOOK}, default=str), self.boot_id)) + self._restore_tags(conn) + record_event("info", "fly", "fly session started", {"run_id": self.run_id, "bootstrap": self.boot_id, "channels": self.cfg, "lines": self.lines["plastic"], + "pending": len(self.pending), "learning_frozen": self.learning_frozen}) + log.info("fly session: bootstrap %d, channels %s, lines %s, %d pending tags", self.boot_id, self.cfg, self.lines["plastic"], len(self.pending)) + + # ---- model and state ---- + def _load_bootstrap(self, boot: dict) -> None: + self.boot_id = int(boot["id"]) + self.fly = fly_selector.load(boot["path"]) + self.idx = np.asarray([X_COLS.index(c) for c in self.fly.cols], dtype=int) + self.names = list(self.fly.strategies); self.holds = {k: float(self.fly.rules[k]["hold_min"]) * 60.0 for k in self.names} + self.H = max(self.holds.values()) + per = self.verdict.get("per_strategy") or {self.names[0]: {"alpha": self.verdict.get("alpha", 0.0), "half_life_days": self.verdict.get("half_life_days")}} + self.cfg = {k: (float((per.get(k) or {}).get("alpha") or 0.0), (per.get(k) or {}).get("half_life_days")) for k in self.names} + self.alpha = self.cfg[self.names[0]][0]; self.half_life = float(self.cfg[self.names[0]][1]) if self.cfg[self.names[0]][1] is not None else math.inf + net = self.fly.net + self.bank = plastic.PlasticBank(net, [(self.alpha, self.half_life)], fly_selector.SCALE, learn=net.learn.cpu().numpy(), read=net.read.cpu().numpy()) + for j, k in enumerate(self.names): # each strategy's channel learns and forgets at its own replay-chosen rate + a, h = self.cfg[k]; cols = self.bank.learn[j] + self.bank.alpha[0, cols] = a; self.bank.half_life_s[0, cols] = float(h) * 86400.0 if h is not None else float("inf") + self.pending = plastic.PendingTags(net.n_kc, net.k_active) + self.bars: dict[str, deque] = {}; self.watch: dict[str, int] = {} + self.lines = {"plastic": dict(self.fly.lines), "frozen": dict(self.fly.lines)} + self.sizing = {"plastic": {k: list(v) for k, v in self.fly.sizings.items()}, "frozen": {k: list(v) for k, v in self.fly.sizings.items()}} + self.applied_through = -math.inf; self.learning_frozen = False; self.checks_paused_until = {k: 0.0 for k in self.names} + self.race_started_at: float | None = None; self.last_day: int | None = None; self.last_hour: int | None = None + self.rollbacks: dict = {k: [] for k in self.names}; self.nu_set = False; self.hour_stats = self._empty_stats(); self.last_checks: dict = {} + s = self._read_state() + if s and s.get("bootstrap_id") == self.boot_id and s.get("cfg") == self.cfg: + self.bank.load_state(s["bank"]); self.nu_set = True + self.lines, self.sizing = s["lines"], s["sizing"] + self.applied_through, self.learning_frozen, self.checks_paused_until = s["applied_through"], s["learning_frozen"], s["checks_paused_until"] + self.race_started_at, self.last_day, self.last_hour, self.rollbacks = s["race_started_at"], s["last_day"], s["last_hour"], s["rollbacks"] + elif s: + self.race_started_at = s.get("race_started_at") # a new bootstrap keeps racing the same book + with transaction() as conn: + conn.execute("UPDATE fly_scored SET state = 'dropped', x = NULL WHERE state = 'pending' AND bootstrap_id IS DISTINCT FROM %s", (self.boot_id,)) + + def _read_state(self) -> dict | None: + if not self.state_path.exists(): + return None + try: + return torch.load(self.state_path, map_location="cpu", weights_only=False) + except Exception: + log.exception("fly state unreadable; starting from the bootstrap") + return None + + def _state(self) -> dict: + return {"bootstrap_id": self.boot_id, "cfg": self.cfg, "bank": self.bank.state(), "lines": self.lines, "sizing": self.sizing, + "applied_through": self.applied_through, "learning_frozen": self.learning_frozen, "checks_paused_until": self.checks_paused_until, + "race_started_at": self.race_started_at, "last_day": self.last_day, "last_hour": self.last_hour, "rollbacks": self.rollbacks, + "saved_at": time.time()} + + def _write_state(self) -> None: + self.state_path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.state_path.with_suffix(".tmp") + torch.save(self._state(), tmp); os.replace(tmp, self.state_path) + + def _restore_tags(self, conn) -> None: + rows = conn.execute("SELECT ts, mint, strategy, x, score, line FROM fly_scored WHERE state = 'pending' AND bootstrap_id = %s ORDER BY ts", (self.boot_id,)).fetchall() + rows = [r for r in rows if r["strategy"] in self.holds] + applied = [(r["ts"], r["mint"], r["strategy"]) for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S <= self.applied_through] + if applied: + with conn.cursor() as cur: + cur.executemany("UPDATE fly_scored SET state = 'resolved', resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", applied) + todo = [r for r in rows if r["ts"].timestamp() + self.holds[r["strategy"]] + LABEL_LAG_S > self.applied_through and r["x"] is not None] + groups: dict[tuple, list] = {} + for r in todo: + groups.setdefault((r["ts"].timestamp(), r["strategy"]), []).append(r) + for (t, name), rs in sorted(groups.items()): + j = self.names.index(name) + Y, u0, k = self.fly.parts_all(np.asarray([r["x"] for r in rs], dtype=np.float32)) + w = plastic.row_weights(torch.tensor([[float(r["score"]) for r in rs]]), torch.tensor([float(rs[0]["line"])])) + self.pending.push([(t, r["mint"], name) for r in rs], np.full(len(rs), t), t + self.holds[name] + LABEL_LAG_S, Y[:, j], u0, k, w, s=j) + for r in rs: + self.watch[r["mint"]] = self.watch.get(r["mint"], 0) + 1 + + def maybe_reload(self) -> bool: + """A newer deployable bootstrap (a re-bootstrap) replaces this one at once: D = 0, pending tags dropped, the book carries on.""" + from ..train import fly_replay + with transaction() as conn: + boot = fly_selector.latest_deployable(conn) + v = fly_replay.verdict() + if boot is None or v is None or not v.get("passed") or int(boot["id"]) == self.boot_id: + return False + old = self.boot_id; self.verdict = v + self._write_state() + self._load_bootstrap(boot) + with transaction() as conn: + conn.execute("UPDATE runs SET brain_snapshot_id = %s WHERE run_id = %s", (self.boot_id, self.run_id)) + self._write_state() + record_event("info", "fly", f"switched to fly bootstrap #{self.boot_id}", {"from": old, "to": self.boot_id, "lines": self.lines["plastic"]}) + return True + + # ---- engine interface ---- + def on_bars(self, t_start: float, bars: dict) -> None: + for m in self.watch: + b = bars.get(m) + if b is not None: + q = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)) + if not q or q[-1][0] < b[0]: + q.append(b) + + def open_mints(self, conn) -> set[str]: + return {p["mint"] for p in ledger.open_positions(conn, BOOK)} + + def on_minute(self, ctx) -> dict: + conn, t_now = ctx.conn, ctx.t_start + self._commands(conn, ctx.m1_epoch) + learned = self._learn(conn, ctx.m1_epoch) + day = int(t_now // 86400) + if self.last_day is None: + self.last_day = day + elif day > self.last_day: + self.last_day = day; self._calibrate(conn, ctx.m1_epoch) + scored = self._score(conn, ctx) + drift = {k: float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) for j, k in enumerate(self.names)} + out = {"minute": ctx.m1.isoformat(), "eligible": len(ctx.mints), "picks": scored["picks"], "lines": self.lines["plastic"], "frozen_lines": self.lines["frozen"], + "line": self.lines["plastic"][self.names[0]], "frozen_line": self.lines["frozen"][self.names[0]], + "drift": max(drift.values()) if drift else 0.0, "drift_by_strategy": drift, "pending": len(self.pending), "learned": learned, + "learning_frozen": self.learning_frozen, "channels": self.cfg, "holds_min": {k: v / 60.0 for k, v in self.holds.items()}, "bootstrap": self.boot_id} + if ctx.trade and ctx.fresh: + if self.race_started_at is None: + self.race_started_at = ctx.m1_epoch + self.beat_no += 1 + d = scored["decision"] + st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos, + scores=d["score"], threshold=d["threshold"], table=None, horizon_s=self.holds[self.names[0]], holds=d["hold_s"], + strategies=d["strategy"], tables=d["tables"]) + out.update({k: v for k, v in st.items() if k != "entries"}); out["stage"] = "trading" + if self.live and handover.state(conn) is not None: + out["live"] = self._live(ctx, st) + else: + out["stage"] = "learning (not trading this minute)" if ctx.trade else "catching up" + hour = int(ctx.m1_epoch // 3600) + if self.last_hour is None: + self.last_hour = hour + elif hour > self.last_hour: + self.last_hour = hour; self._hourly(conn, ctx.m1_epoch) + self._write_state() + if learned.get("keys"): + with conn.cursor() as cur: + cur.executemany("UPDATE fly_scored SET label = %s, state = %s, resolved_at = now(), x = NULL WHERE ts = %s AND mint = %s AND strategy = %s", learned.pop("keys")) + learned.pop("keys", None) + out["checks"] = self.last_checks; out["updated_at"] = datetime.now(timezone.utc).isoformat() + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('fly_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(out, default=str),)) + return out + + def _live(self, ctx, st: dict) -> dict: + """The fly holds the seat: mirror this minute on the bot wallet (agent/fly_live.py) while signing is allowed.""" + from ..chain.cluster_guard import signing_allowed + if not signing_allowed(): + return {"stage": "live gated: signing is not allowed (LIVE_ENABLED=1 and the live prerequisites)"} + if self.mirror is None: + from .fly_live import LiveMirror + self.mirror = LiveMirror(self.H) + record_event("info", "fly_live", "the fly trades the bot wallet", {"run_id": self.run_id}) + return self.mirror.minute(ctx, run_id=self.run_id, beat_id=st["beat_id"], entries=st["entries"], line=self.lines["plastic"][self.names[0]], + table=self.sizing["plastic"][self.names[0]]) + + def finish(self) -> None: + if self.mirror is not None: + self.mirror.stop() + try: + self._write_state() + except Exception: + log.exception("fly state write failed at finish") + with transaction() as conn: + conn.execute("UPDATE runs SET ended_at = now(), status = 'finished' WHERE run_id = %s", (self.run_id,)) + record_event("info", "fly", "fly session stopped", {"run_id": self.run_id, "minutes": self.beat_no}) + + # ---- learning ---- + @staticmethod + def _empty_stats() -> dict: + return {"n": 0, "sum_delta": 0.0, "sum_abs": 0.0, "step": 0.0, "capped": 0, "unknown": 0} + + def _learn(self, conn, now: float) -> dict: + tags = self.pending.pop_due(now, device=self.bank.dev) + if tags is None: + return {"n": 0} + labels = np.array([np.nan if (lb := resolve_label(list(self.bars.get(m, ())), t, self.holds[name])) is None else lb for t, m, name in tags.keys], dtype=np.float64) + ok = np.isfinite(labels) + for t, m, _ in tags.keys: + c = self.watch.get(m, 0) - 1 + if c > 0: + self.watch[m] = c + else: + self.watch.pop(m, None); self.bars.pop(m, None) + stats = {"n": int(ok.sum()), "unknown": int((~ok).sum())} + if ok.any(): + tv = tags.subset(ok) + if self.learning_frozen: + self.bank.decay_to(now) + else: + st = self.bank.update(tv, torch.tensor(labels[ok], dtype=torch.float32), now) + stats.update(mean_delta=st["mean_delta"][0], mean_abs_delta=st["mean_abs_delta"][0], step=st["step"][0], capped=bool(st["capped"][0])) + h = self.hour_stats; h["n"] += stats["n"]; h["sum_delta"] += st["mean_delta"][0] * stats["n"]; h["sum_abs"] += st["mean_abs_delta"][0] * stats["n"] + h["step"] += st["step"][0]; h["capped"] += int(st["capped"][0]) + self.hour_stats["unknown"] += stats["unknown"] + self.applied_through = now + stats["keys"] = [(float(lb) if np.isfinite(lb) else None, "resolved" if np.isfinite(lb) else "unknown", datetime.fromtimestamp(t, timezone.utc), m, name) + for (t, m, name), lb in zip(tags.keys, labels)] + return stats + + def _score(self, conn, ctx) -> dict: + n = len(ctx.mints); empty = {"strategy": np.full(n, None, dtype=object), "score": np.full(n, -1.0), "hold_s": np.zeros(n), "threshold": np.full(n, np.inf), + "tables": [[] for _ in range(n)]} + if not n: + return {"decision": empty, "picks": 0} + X = ctx.X[:, self.idx] + Y, u0, k = self.fly.parts_all(X); trig = self.fly.triggers(ctx.X, X_COLS) + if not self.nu_set: + self.bank.estimate_nu(Y[:, 0], u0, k); self.nu_set = True + t = ctx.t_start; ts = datetime.fromtimestamp(t, timezone.utc); rows = [] + V = np.full((n, len(self.names)), -np.inf) + for j, name in enumerate(self.names): + loc = np.flatnonzero(trig[:, j]) + if not len(loc): + continue + li = torch.as_tensor(loc, device=Y.device) + with torch.no_grad(): + sc, _ = self.bank.predict(Y[li, j], u0[li], k[li], s=np.full(len(loc), j)); sc = sc[0] + fr = self.bank.frozen(Y[li, j], u0[li], s=np.full(len(loc), j)) + v, f = sc.float().cpu().numpy(), fr.float().cpu().numpy(); V[loc, j] = v + line, fline = self.lines["plastic"][name], self.lines["frozen"][name] + rows += [(ts, ctx.mints[i], name, int(self.holds[name] // 60), ctx.X[i].tolist(), float(v[q]), float(f[q]), line, fline, self.boot_id) for q, i in enumerate(loc)] + w = plastic.row_weights(sc[None], torch.tensor([line], dtype=sc.dtype, device=sc.device)) + self.pending.push([(t, ctx.mints[i], name) for i in loc], np.full(len(loc), t), t + self.holds[name] + LABEL_LAG_S, Y[li, j], u0[li], k[li], w, s=j) + for i in loc: + m = ctx.mints[i]; self.watch[m] = self.watch.get(m, 0) + 1 + q_ = self.bars.setdefault(m, deque(maxlen=BAR_KEEP)); b = ctx.bars.get(m) + if b is not None and (not q_ or q_[-1][0] < b[0]): + q_.append(b) + if rows: + with conn.cursor() as cur: + cur.executemany("INSERT INTO fly_scored (ts, mint, strategy, hold_min, x, score, frozen_score, line, frozen_line, bootstrap_id) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (ts, mint, strategy) DO NOTHING", rows) + self.fly.lines = dict(self.lines["plastic"]); self.fly.sizings = {k: list(v) for k, v in self.sizing["plastic"].items()} + d = fly_selector.fly_decide(self.fly, ctx.X, X_COLS, np.where(np.isfinite(V), V, -np.inf)) + return {"decision": d, "picks": int(sum(1 for x in d["strategy"] if x is not None))} + + # ---- daily and hourly ---- + def _resolved(self, conn, since_resolved: float, cols: str, name: str) -> list[dict]: + """A strategy's resolved rows whose labels became known after ``since_resolved`` (epoch s).""" + return conn.execute(f"SELECT {cols} FROM fly_scored WHERE state = 'resolved' AND label IS NOT NULL AND bootstrap_id = %s AND strategy = %s AND ts >= %s ORDER BY ts", + (self.boot_id, name, datetime.fromtimestamp(since_resolved - self.holds[name] - LABEL_LAG_S, timezone.utc))).fetchall() + + def _calibrate(self, conn, now: float) -> None: + day = datetime.fromtimestamp(now, timezone.utc).date() + for name in self.names: + rows = self._resolved(conn, now - fly_calibrate.WINDOW_DAYS * 86400.0, "ts, mint, score, frozen_score, label", name) + if not rows: + continue + ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64) + for arm, col in (("plastic", "score"), ("frozen", "frozen_score")): + cal = fly_calibrate.calibrate(ts, mint, self.holds[name], np.array([r[col] for r in rows], dtype=np.float64), lab, + prev_line=self.lines[arm][name], prev_sizing=self.sizing[arm][name]) + if cal.changed: + record_event("info", "fly", f"{name} {arm} line {self.lines[arm][name] * 100:+.2f}% → {cal.line * 100:+.2f}%", {"trades": cal.trades, "mean": cal.mean, "total": cal.total}) + self.lines[arm][name], self.sizing[arm][name] = cal.line, cal.sizing + conn.execute("INSERT INTO fly_calibrations (day, arm, line, sizing, trades, total, mean, window_days) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) " + "ON CONFLICT (day, arm) DO UPDATE SET line = EXCLUDED.line, sizing = EXCLUDED.sizing, trades = EXCLUDED.trades, total = EXCLUDED.total, mean = EXCLUDED.mean", + (day, f"{arm}:{name}", cal.line, json.dumps(cal.sizing), cal.trades, cal.total, cal.mean, fly_calibrate.WINDOW_DAYS)) + + def _drift(self, j: int) -> float: + return float(self.bank.drift(j)[0]) if self.bank.learn.shape[0] > 1 else float(self.bank.drift()[0]) + + def _checks(self, conn, now: float) -> dict: + """The rollback checks per strategy (its own trades, its own channel's drift).""" + out = {} + for j, name in enumerate(self.names): + rows = self._resolved(conn, now - gov.SHADOW_DAYS * 86400.0, "ts, mint, score, line, frozen_score, frozen_line, label", name) + if rows: + ts = np.array([r["ts"].timestamp() for r in rows]); mint = np.array([r["mint"] for r in rows]); lab = np.array([r["label"] for r in rows], dtype=np.float64) + g = lambda c: np.array([r[c] for r in rows], dtype=np.float64) + H = self.holds[name] + rp = gov.picks_returns(ts, mint, H, g("score"), g("line"), lab); rf = gov.picks_returns(ts, mint, H, g("frozen_score"), g("frozen_line"), lab) + recent = ts + H + LABEL_LAG_S > now - gov.IC_HOURS * 3600.0 + out[name] = gov.check((rp, rf), (g("score")[recent], lab[recent]), self._drift(j)) + else: + out[name] = gov.check(None, None, self._drift(j)) + return out + + def _hourly(self, conn, now: float) -> None: + hour = datetime.fromtimestamp(math.floor(now / 3600) * 3600 - 3600, timezone.utc) + checks = self._checks(conn, now); self.last_checks = checks + h = self.hour_stats; first = checks.get(self.names[0]) or {} + conn.execute("INSERT INTO fly_updates (hour, n, mean_delta, mean_abs_delta, step, capped, drift, ic, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (hour) DO NOTHING", + (hour, h["n"], h["sum_delta"] / h["n"] if h["n"] else None, h["sum_abs"] / h["n"] if h["n"] else None, h["step"], h["capped"], + max((c["drift"] for c in checks.values()), default=0.0), first.get("ic"), json.dumps({"unknown": h["unknown"], "checks": checks}, default=str))) + self.hour_stats = self._empty_stats() + good = not any(c["triggers"] for c in checks.values()) + sid = self._snapshot(conn, now, good, checks) + for name, c in checks.items(): + if c["triggers"] and now >= self.checks_paused_until.get(name, 0.0) and not self.learning_frozen: + self._rollback(conn, now, c, sid, name=name) + self._prune(conn, now) + self._handover_check(conn, now) + + def _snapshot(self, conn, now: float, good: bool, checks: dict) -> int: + d = fly_state_dir(); d.mkdir(parents=True, exist_ok=True) + path = d / f"snap_{datetime.fromtimestamp(now, timezone.utc).strftime('%Y%m%dT%H%M')}.pt" + torch.save({"bank": self.bank.state(), "bootstrap_id": self.boot_id, "lines": self.lines, "sizing": self.sizing}, path) + note = {"bootstrap_id": self.boot_id, "data": fly_selector.FLY_VERSION, "good": good, "checks": checks, + "drift": max((c["drift"] for c in checks.values()), default=0.0), "ic24": {k: c.get("ic") for k, c in checks.items()}} + return int(conn.execute("INSERT INTO brain_snapshots (run_id, path, kind, note) VALUES (%s,%s,'fly_plastic',%s) RETURNING id", + (self.run_id, str(path), json.dumps(note, default=str))).fetchone()["id"]) + + def _commands(self, conn, now: float) -> None: + """Operator commands from the console (``send_command``): pause / resume learning, roll back now.""" + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (COMMAND_KEY,)).fetchone() + if not r or not r["value"]: + return + v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"]) + conn.execute("DELETE FROM ui_settings WHERE key = %s", (COMMAND_KEY,)) + cmd = v.get("cmd") + if cmd == "pause": + self.learning_frozen = True; record_event("info", "fly", "learning paused by the operator") + elif cmd == "resume": + self.learning_frozen = False; self.rollbacks = {}; record_event("info", "fly", "learning resumed by the operator") + elif cmd == "rollback": + for j, name in enumerate(self.names): + self._rollback(conn, now, {"triggers": ["operator"], "drift": self._drift(j)}, None, count=False, name=name) + + def _rollback(self, conn, now: float, checks: dict, from_sid: int | None, count: bool = True, name: str | None = None) -> None: + """Restore one strategy's channel (its KC→MBON columns) from the newest good snapshot at least a day old, or the bootstrap.""" + name = name or self.names[0]; j = self.names.index(name); cols = self.bank.learn[j] + target = None + for r in conn.execute("SELECT id, path, note, ts FROM brain_snapshots WHERE kind = 'fly_plastic' AND ts <= %s ORDER BY id DESC", + (datetime.fromtimestamp(now - ROLLBACK_MIN_AGE_S, timezone.utc),)).fetchall(): + n = json.loads(r["note"] or "{}") + ok = n.get("good") or not ((n.get("checks") or {}).get(name) or {}).get("triggers") + if ok and n.get("bootstrap_id") == self.boot_id and Path(r["path"]).exists(): + target = r; break + if target is not None: + snap = plastic.PlasticBank(self.fly.net, [(0.0, math.inf)], fly_selector.SCALE, learn=self.bank.learn.cpu().numpy(), read=self.bank.read.cpu().numpy()) + st = torch.load(target["path"], map_location="cpu", weights_only=False)["bank"] + snap.D = torch.zeros_like(snap.D); snap.D[:, snap.mask.bool()] = st["D"].to(snap.dev).float() + self.bank.D[0][:, cols] = snap.D[0][:, cols] + else: + self.bank.D[0][:, cols] = 0.0 + self.checks_paused_until[name] = now + ROLLBACK_PAUSE_S + self.rollbacks[name] = [t for t in self.rollbacks.get(name, []) if t > now - ROLLBACK_WINDOW_S] + ([now] if count else []) + conn.execute("INSERT INTO fly_rollbacks (reason, checks, from_snapshot, to_snapshot) VALUES (%s,%s,%s,%s)", + (f"{name}: " + ", ".join(checks["triggers"]), json.dumps(checks, default=str), from_sid, target["id"] if target is not None else None)) + record_event("warning", "fly", f"{name} channel rolled back ({', '.join(checks['triggers'])})", {"to_snapshot": target["id"] if target is not None else "bootstrap (D = 0)", "checks": checks}) + if count and len(self.rollbacks[name]) >= ROLLBACK_LIMIT: + self.learning_frozen = True + from ..train import pipeline + pipeline.request_run(fly=True, reason=f"{name}: {len(self.rollbacks[name])} rollbacks in 7 days") + record_event("error", "fly", f"learning frozen after {len(self.rollbacks[name])} {name} rollbacks in 7 days; re-bootstrap requested", {"rollbacks": self.rollbacks}) + + def _prune(self, conn, now: float) -> None: + rows = conn.execute("SELECT id, path, ts FROM brain_snapshots WHERE kind = 'fly_plastic' AND ts < %s ORDER BY ts DESC", + (datetime.fromtimestamp(now - SNAP_HOURLY_DAYS * 86400.0, timezone.utc),)).fetchall() + kept_days: set = set(); drop = [] + for r in rows: + d = r["ts"].date() + if d in kept_days: + drop.append(r) + else: + kept_days.add(d) + for r in drop: + conn.execute("DELETE FROM brain_snapshots WHERE id = %s", (r["id"],)) + try: + Path(r["path"]).unlink(missing_ok=True) + except OSError: + pass + conn.execute("DELETE FROM fly_scored WHERE state <> 'pending' AND ts < %s", (datetime.fromtimestamp(now - SCORED_KEEP_DAYS * 86400.0, timezone.utc),)) + + def _handover_check(self, conn, now: float) -> None: + """The fly takes the selector's seat once it has raced ``config.HANDOVER_DAYS`` (0: no minimum, the window is the + whole race), closed ``config.HANDOVER_MIN_TRADES`` paper trades and, when ``config.HANDOVER_BEAT_SELECTOR``, + realized at least the paper selector's P&L over that window.""" + days = config.HANDOVER_DAYS + if handover.state(conn) is not None or self.race_started_at is None or now - self.race_started_at < days * 86400.0: + return + since = datetime.fromtimestamp(self.race_started_at if days <= 0 else now - days * 86400.0, timezone.utc) + pnl = {b: conn.execute("SELECT COALESCE(sum(realized_sol), 0) AS s, count(*) AS n FROM positions WHERE book = %s AND status = 'closed' AND closed_at >= %s", + (b, since)).fetchone() for b in (BOOK, "paper_selector")} + fly_pnl, sel_pnl, n = float(pnl[BOOK]["s"]), float(pnl["paper_selector"]["s"]), int(pnl[BOOK]["n"]) + if n >= config.HANDOVER_MIN_TRADES and (fly_pnl >= sel_pnl or not config.HANDOVER_BEAT_SELECTOR): + detail = {"at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "fly_pnl_sol": fly_pnl, "selector_pnl_sol": sel_pnl, "fly_trades": n, + "selector_trades": int(pnl["paper_selector"]["n"]), "days": days, "beat_selector": config.HANDOVER_BEAT_SELECTOR, "bootstrap": self.boot_id} + handover.record(conn, detail) + record_event("info", "handover", f"the fly takes the selector's seat: {fly_pnl:+.3f} SOL vs {sel_pnl:+.3f} SOL over {n} trades" + + (f" in {days} days" if days > 0 else ""), detail) diff --git a/fly_trader/agent/handover.py b/fly_trader/agent/handover.py new file mode 100644 index 0000000000000000000000000000000000000000..f7532d135ce25c444db76c80e39ce36e934ef55d --- /dev/null +++ b/fly_trader/agent/handover.py @@ -0,0 +1,24 @@ +"""The handover: the plastic fly takes the selector's seat (agent/fly_session.py decides; ui_settings['handover']). + +After it the selector stops entering, exits to flat and is dropped while a fly trades; the weekly training schedule +stops (train/pipeline.due runs only on request). If no fly can trade (replaced definitions, a failed re-bootstrap), the +selector trades again until one can. +""" +from __future__ import annotations + +import json + +KEY = "handover" + + +def state(conn) -> dict | None: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() + if not r or r["value"] is None: + return None + v = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"]) + return v or None + + +def record(conn, detail: dict) -> None: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (KEY, json.dumps(detail, default=str))) diff --git a/fly_trader/agent/minute_engine.py b/fly_trader/agent/minute_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..4332037c16a461a4efe6dae06c763fabb54e148d --- /dev/null +++ b/fly_trader/agent/minute_engine.py @@ -0,0 +1,342 @@ +"""The trading engine on the live stream: every UTC minute, features once per mint, then every book trades on them. + +Every UTC minute, once the stream has written it (``pumpstream_status.flushed_through``), the minute's rich candle per +mint (``pump_minutes``: open/high/low/close, SOL volume by side, buy/sell counts, distinct traders, quote reserve; the +same fields and filters ``train/mature.py`` builds from the archive) is fed to the live feature engine as two +synthetic trades, exactly as in training. Graduation time and creation/creator facts come from ``corpus_meta``, the +same source training uses. The full feature vector (``decisions.X_COLS``) of the rows in the models' universe +(``selector.in_universe``) that pass the eligibility gate (pool ≥ ``MIN_RESQ_SOL``, 15-minute volume ≥ +``MIN_VOL_15M_SOL``, no scale break — a close 50× from the previous one or a reserve above 100,000 SOL — since the +mint's state began) goes to every book; each picks its model's columns by name. + +Books — ``SelectorBook`` (agent/selector_session.py) and ``FlyBook`` (agent/fly_session.py) — implement ``name``, +``on_bars(t_start, bars)`` (every processed minute, warm-up included: open, close and exit cost of every traded mint), +``on_minute(ctx)``, ``open_mints(conn)``, ``maybe_reload()``, ``done`` and ``finish()``. Each book trades inside its own +savepoint, so one book's failure never undoes the other's minute. Books join when they qualify (checked every +``MODEL_CHECK_S``): the selector while it has a deployable model and has not handed its seat to a fly that is trading; +the fly once its design passed the replay and a deployable bootstrap exists. No entries or exits while the stream is +stale; missed minutes are replayed through the features without trading. +""" +from __future__ import annotations + +import json +import logging +import math +import threading +import time +from collections import deque +from dataclasses import dataclass +from datetime import datetime, timezone + +import numpy as np + +from ..db.apilog import record_event +from ..db.connection import transaction +from ..market.exit_cost import PUMP_SUPPLY +from ..market.exit_cost import cost_at_size +from ..market.features import FEATURES, FIDX, TokenMeta, TokenState +from ..train.corpus_meta import FEATURE_COLS as META_COLS +from ..train.decisions import MIN_RESQ_SOL, MIN_VOL_15M_SOL, X_COLS +from ..train.flow import FlowWindow, MarketWindow +from ..train.mature import MASKED +from ..train.selector import in_universe + +log = logging.getLogger(__name__) +WARMUP_MIN = 1440 # training warms each day on the previous day's candles +IDLE_EVICT_S = 86400 # training has no history for a mint idle since before the previous day +STREAM_STALE_S = 180 # no entries or exits when the stream's newest complete minute is older than this +META_TTL_S = 600 +MODEL_CHECK_S = 600 # how often books reload newer models and new books are admitted + + +class MintState: + __slots__ = ("st", "meta", "hist", "decimals", "pool", "program_label", "graduated_at", "meta_row", "prev_close", "broken", "flow") + + def __init__(self, mint: str, decimals: int, pool: str | None, program_label: str | None, graduated_at: float | None, meta_row: dict | None): + self.st = TokenState(mint); self.meta = TokenMeta(mint=mint, program_label=program_label or "Pump.fun Amm", graduated_at=graduated_at) + self.hist: deque = deque(maxlen=200) # (t_end, n_trades, n_traders) for the trailing hour + self.decimals, self.pool, self.program_label, self.graduated_at, self.meta_row = decimals, pool, program_label, graduated_at, meta_row + self.prev_close: float | None = None; self.broken = False + self.flow = FlowWindow() # wallet flow inputs (train/flow.py, as mature.build_day) + + +@dataclass +class Minute: + """What every book sees of one minute.""" + conn: object + m0: datetime + m1: datetime + m1_epoch: float + agg: dict + X: np.ndarray # [n, len(X_COLS)] eligible rows in the universe + infos: list + mints: list + bars: dict # mint -> (minute start, open, close, exit cost) for every traded mint + prices: dict + resqs: dict + fees: dict + fresh: bool + trade: bool + engine: "MinuteEngine" + + @property + def t_start(self) -> float: + return self.m1_epoch - 60.0 + + def last_resq(self, mint: str) -> float | None: + return self.engine._last_resq(self.conn, mint) + + def mcap(self, mint: str, price: float | None) -> float | None: + return self.engine._mcap(mint, price) + + +class MinuteEngine: + def __init__(self, books: list | None = None, live: bool = False): + self.books = list(books or []); self.live = live + self.states: dict[str, MintState] = {} + self.meta_cache: dict[str, dict] = {}; self.last_sweep = time.time(); self.last_minute: float | None = None + self.market = MarketWindow() # mkt_vol_1h over every traded mint (as mature._market_windows) + + def has_book(self, name: str) -> bool: + return any(b.name == name for b in self.books) + + # ---- lookups ---- + def _meta(self, conn, mint: str) -> dict | None: + """``corpus_meta`` row (graduation time + creation/creator features), cached for 10 minutes: rows appear at + graduation and the archive rebuild later replaces stream values, so the cache is refreshed.""" + hit = self.meta_cache.get(mint) + if hit is None or time.time() - hit["at"] > META_TTL_S: + r = conn.execute("SELECT graduated_at, supply, " + ", ".join(META_COLS) + " FROM corpus_meta WHERE mint = %s", (mint,)).fetchone() + hit = self.meta_cache[mint] = {"row": dict(r) if r else None, "at": time.time()} + return hit["row"] + + def _state(self, conn, mint: str, pool: str | None, program_label: str | None) -> MintState: + s = self.states.get(mint) + if s is None: + t = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone() + s = MintState(mint, int((t or {}).get("decimals") or 6), pool, program_label, None, None) + self.states[mint] = s + row = self._meta(conn, mint) + s.meta_row = row + g = row["graduated_at"].timestamp() if row and row.get("graduated_at") else None + if g != s.graduated_at: # same source as training: corpus_meta.graduated_at + s.graduated_at = g; s.meta.graduated_at = g + s.meta.supply = float(row["supply"]) if row and row.get("supply") else PUMP_SUPPLY # as training (train/mature.build_day) + return s + + def _last_resq(self, conn, mint: str) -> float | None: + s = self.states.get(mint) + if s is not None and s.st.last_res_quote_sol: + return s.st.last_res_quote_sol + r = conn.execute("SELECT resq_sol FROM pump_minutes WHERE mint = %s AND resq_sol IS NOT NULL ORDER BY ts DESC LIMIT 1", (mint,)).fetchone() + return float(r["resq_sol"]) if r else None + + def _mcap(self, mint: str, price: float | None) -> float | None: + """Market cap in SOL (price × supply), which sets the pool fee tier; None without a price.""" + s = self.states.get(mint) + return float(price) * (s.meta.supply if s is not None else PUMP_SUPPLY) if price else None + + # ---- one minute ---- + def _aggregate(self, conn, m0: datetime, m1: datetime) -> dict[str, dict]: + """PumpAPI minutes: every SOL-quoted PumpSwap pump.fun token, the same fields as the archive.""" + rows = conn.execute("SELECT mint, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, " + "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy FROM pump_minutes WHERE ts = %s", (m0,)).fetchall() + return {r["mint"]: {"open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "buy": r["buy_sol"] or 0.0, "sell": r["sell_sol"] or 0.0, + "nb": r["n_buys"] or 0, "ns": r["n_sells"] or 0, "n_traders": int(r["n_traders"] or 0), "resq": r["resq_sol"], + "pool": r["pool_id"], "program_label": "Pump.fun Amm", "fee_rate": r["fee_rate"], + "n_buyers": r["n_buyers"], "wash_sol": r["wash_sol"], "wash_buy_sol": r["wash_buy_sol"], "top_sell_sol": r["top_sell_sol"], + "insider_sell_sol": r["insider_sell_sol"], "skill_buy": r["skill_buy"]} for r in rows if r["close"]} + + def _features(self, conn, mint: str, a: dict, t_end: float) -> tuple[np.ndarray, dict]: + """The full feature vector (``X_COLS``) of a mint for the minute ending ``t_end``, and what trading needs.""" + s = self._state(conn, mint, a["pool"], a["program_label"]) + st, price, resq = s.st, float(a["close"]), a["resq"] + if (s.prev_close and not (1 / 50 <= price / s.prev_close <= 50)) or (resq is not None and resq > 1e5): + s.broken = True # train/decisions.py: a scale break makes the mint ineligible from that minute on + s.prev_close = price + if a["buy"] > 0: + st.append(t_end - 2e-3, price, a["buy"], True, None, resq) + if a["sell"] > 0 or a["buy"] <= 0: + st.append(t_end - 1e-3, price, a["sell"], False, None, resq) + s.hist.append((t_end, float(a["nb"] + a["ns"]), float(a["n_traders"]))) + s.flow.append(t_end, a["buy"], a["sell"], a.get("n_buyers"), a.get("wash_sol"), a.get("wash_buy_sol"), a.get("top_sell_sol"), a.get("insider_sell_sol"), a.get("skill_buy")) + f, _mask = st.features(t_end, s.meta) + ht = np.array([h[0] for h in s.hist]); hn = np.array([h[1] for h in s.hist]); htr = np.array([h[2] for h in s.hist]) + for k, w in (("1m", 60), ("5m", 300), ("15m", 900), ("1h", 3600)): + f[FIDX[f"logn_{k}"]] = math.log1p(float(hn[ht > t_end - w].sum())) + for name in MASKED: + f[FIDX[name]] = 0.0 + if s.graduated_at is None: + f[FIDX["log_age_h"]] = 0.0 + hod = ((t_end - 60.0) % 86400) / 3600.0 # minute start, as in train/decisions.py + meta = s.meta_row or {} + extra = {"traders_15m": float(htr[ht > t_end - 900].sum()), "traders_1h": float(htr[ht > t_end - 3600].sum()), "n_trades_1m": float(a["nb"] + a["ns"]), + "hod_s": math.sin(2 * math.pi * hod / 24), "hod_c": math.cos(2 * math.pi * hod / 24), "age_known": 1.0 if s.graduated_at is not None else 0.0, + "meta_known": 1.0 if meta.get("ttg_min") is not None else 0.0, **s.flow.features(t_end), "mkt_vol_1h": self.market.value(t_end)} + by_name = {**{n: float(f[i]) for i, n in enumerate(FEATURES)}, **extra, + **{c: float(meta.get(c)) if meta.get(c) is not None else 0.0 for c in META_COLS}} + x = np.nan_to_num(np.asarray([by_name.get(c, 0.0) for c in X_COLS], dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0) + info = {"price": price, "open": float(a["open"]) if a.get("open") else price, "resq": resq, "logvol_15m": f[FIDX["logvol_15m"]], + # "ec" is the training label's own cost model (market/exit_cost.cost_at_size), at the size the book trades + "ec": float(cost_at_size(f[FIDX["exit_cost_0p1"]], resq)), "age_h": (t_end - s.graduated_at) / 3600 if s.graduated_at else None, + "mcap": price * s.meta.supply, "fee_rate": a.get("fee_rate"), "decimals": s.decimals, "pool": s.pool, "program_label": s.program_label, "broken": s.broken, + "skill_missing": a.get("skill_buy") is None, "curve_known": bool(meta.get("curve_known"))} + return x, info + + @staticmethod + def eligible(info: dict) -> bool: + return not info["broken"] and info["resq"] is not None and info["resq"] >= MIN_RESQ_SOL and info["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL) + + def _stream_through(self) -> float | None: + with transaction() as conn: + r = conn.execute("SELECT value->>'flushed_through' AS ft FROM ui_settings WHERE key = 'pumpstream_status'").fetchone() + return datetime.fromisoformat(r["ft"]).timestamp() if r and r["ft"] else None + + def ready_through(self, m1_epoch: float) -> float: + """End of the newest minute (at most ``m1``) the stream has written; A minute is + never read before it is written, so a lagging stream delays minutes instead of feeding them empty or partial.""" + ft = self._stream_through() + return min(m1_epoch, ft + 60.0) if ft is not None else float("-inf") + + def stream_fresh(self, m0_epoch: float) -> bool: + ft = self._stream_through() + return ft is not None and m0_epoch - ft <= STREAM_STALE_S + + def _minute_rows(self, conn, agg: dict, m1_epoch: float) -> tuple[np.ndarray, list, list, dict]: + xs, infos, mints, bars = [], [], [], {} + self.market.add(m1_epoch, sum(float(a["buy"]) + float(a["sell"]) for a in agg.values())) + for mint, a in agg.items(): + x, info = self._features(conn, mint, a, m1_epoch) + bars[mint] = (m1_epoch - 60.0, info["open"], info["price"], info["ec"]) + if in_universe(x[None], X_COLS)[0] and self.eligible(info): # the models' universe (train/selector.in_universe) + xs.append(x); infos.append(info); mints.append(mint) + return (np.stack(xs) if xs else np.zeros((0, len(X_COLS)), np.float32)), infos, mints, bars + + def warm_up(self, m1_epoch: float, minutes: int = WARMUP_MIN) -> int: + """Feed the last ``minutes`` closed minutes through the feature states (and the books' bars) without trading.""" + n = 0 + with transaction() as conn: + for k in range(minutes, 0, -1): + t1 = m1_epoch - 60 * k + agg = self._aggregate(conn, datetime.fromtimestamp(t1 - 60, timezone.utc), datetime.fromtimestamp(t1, timezone.utc)) + bars = {} + self.market.add(t1, sum(float(a["buy"]) + float(a["sell"]) for a in agg.values())) + for mint, a in agg.items(): + _x, info = self._features(conn, mint, a, t1); n += 1 + bars[mint] = (t1 - 60.0, info["open"], info["price"], info["ec"]) + for b in self.books: + b.on_bars(t1 - 60.0, bars) + log.info("warm-up: %d token-minutes over the last %d minutes (%d tokens)", n, minutes, len(self.states)) + return n + + def _sweep(self, now_s: float, held: set[str]) -> None: + if now_s - self.last_sweep < 3600: + return + self.last_sweep = now_s + idle = [m for m, s in self.states.items() if m not in held and (s.st.last_ts is None or s.st.last_ts < now_s - IDLE_EVICT_S)] + for m in idle: + self.states.pop(m, None); self.meta_cache.pop(m, None) + if idle: + log.info("evicted %d mint states idle for more than %d h", len(idle), IDLE_EVICT_S // 3600) + + def run_minute(self, m1_epoch: float, trade: bool = True) -> dict: + m1 = datetime.fromtimestamp(m1_epoch, timezone.utc); m0 = datetime.fromtimestamp(m1_epoch - 60, timezone.utc) + out: dict = {"minute": m1.isoformat()} + with transaction() as conn: + agg = self._aggregate(conn, m0, m1) + X, infos, mints, bars = self._minute_rows(conn, agg, m1_epoch) + ctx = Minute(conn=conn, m0=m0, m1=m1, m1_epoch=m1_epoch, agg=agg, X=X, infos=infos, mints=mints, bars=bars, + prices={m: float(a["close"]) for m, a in agg.items()}, resqs={m: a["resq"] for m, a in agg.items()}, + fees={m: a.get("fee_rate") for m, a in agg.items()}, fresh=trade and self.stream_fresh(m1_epoch - 60), trade=trade, engine=self) + out.update(mints_traded=len(agg), eligible=len(mints)); held: set[str] = set() + for b in list(self.books): + try: + with conn.transaction(): # a savepoint per book + b.on_bars(m1_epoch - 60.0, bars) + out[b.name] = b.on_minute(ctx) + held |= set(b.open_mints(conn)) + except Exception: + log.exception("%s: minute %s failed", b.name, m1.isoformat()) + record_event("error", b.name, f"minute failed: {m1.isoformat()}") + self._sweep(m1_epoch, held) + for b in [b for b in self.books if b.done]: + b.finish(); self.books.remove(b) + return out + + +def _status(key: str, value: dict) -> None: + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (key, json.dumps({**value, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str))) + except Exception: + log.debug("status write failed", exc_info=True) + + +def admit_books(engine: MinuteEngine, live: bool) -> list: + """Books that qualify now and are not running yet (the fly first: its presence decides whether the selector trades).""" + from . import fly_session, handover + from .selector_session import NoModel, SelectorBook + new = [] + if not engine.has_book("fly"): + fb, why = fly_session.try_start(live) + if fb is not None: + new.append(fb) + else: + _status("fly_status", {"stage": "not trading", "detail": why}) + fly_on = engine.has_book("fly") or bool(new) + with transaction() as conn: + handed = handover.state(conn) is not None + if not engine.has_book("selector") and not (handed and fly_on): + try: + new.append(SelectorBook()) + except NoModel as e: + _status("selector_status", {"stage": "waiting for a model", "detail": str(e)}) + return new + + +def main(stop_event: threading.Event | None = None, live: bool = False) -> None: + engine = MinuteEngine(live=live) + while not (stop_event is not None and stop_event.is_set()): + engine.books = admit_books(engine, live) + if engine.books: + break + log.info("trading engine waiting: no book qualifies yet") + (stop_event or threading.Event()).wait(MODEL_CHECK_S) + else: + return + m_start = math.floor(time.time() / 60) * 60 + engine.warm_up(m_start); engine.last_minute = m_start - 60; last_check = time.time(); n_min = 0 + try: + while not (stop_event is not None and stop_event.is_set()): + now = time.time(); m1 = math.floor(now / 60) * 60 + if now - last_check >= MODEL_CHECK_S: + last_check = now + for b in engine.books: + try: + b.maybe_reload() + except Exception: + log.exception("%s: model reload check failed", b.name) + try: + engine.books += admit_books(engine, live) + except Exception: + log.exception("book admission failed") + if engine.books and m1 > engine.last_minute and now - m1 >= 4.0 and (upto := engine.ready_through(m1)) > engine.last_minute: # only minutes the stream has written + for minute in range(int(engine.last_minute) + 60, int(upto) + 1, 60): + try: + st = engine.run_minute(float(minute), trade=(minute == int(m1))) # missed minutes update features (and the fly's learning) only + n_min += 1 + if minute == int(m1) and n_min % 10 == 0: + log.info("minute %s: traded %d eligible %d | %s", st["minute"], st.get("mints_traded", 0), st.get("eligible", 0), + " | ".join(f"{b.name}: picks {(st.get(b.name) or {}).get('picks')} open {(st.get(b.name) or {}).get('open')} wealth {(st.get(b.name) or {}).get('wealth')}" + for b in engine.books)) + except Exception: + log.exception("minute %s failed", minute); record_event("error", "engine", f"minute failed: {minute}") + engine.last_minute = upto + time.sleep(0.5) + finally: + for b in engine.books: + try: + b.finish() + except Exception: + log.exception("%s: finish failed", b.name) diff --git a/fly_trader/agent/paper_trading.py b/fly_trader/agent/paper_trading.py new file mode 100644 index 0000000000000000000000000000000000000000..0f255f720af8117e32480a20e8f2023c3d4b3290 --- /dev/null +++ b/fly_trader/agent/paper_trading.py @@ -0,0 +1,89 @@ +"""One trading minute of a paper book — the rules both race books share (the selector's and the fly's): + +exits: positions held their own hold (the strategy's, stored on the position) or ``horizon_s`` are sold at this minute's +price (or the last traded price: the training label's exit); entries: scores at or above the book's line open a position sized from the book's certainty bands +(``agent/sizing.py``), one position per token, unless blocked (global kill switch, paused entries, the book's own +drawdown halt, or the caller's ``block``); marks and wealth per minute (positions net of exit cost, exposure at cost, +drawdown ≥ 0); the book's drawdown halt is checked on its own peak (``rails.check_book_drawdown``). Every score, entry, +exit and blocked pick is a ``decisions`` row. +""" +from __future__ import annotations + +import json + +import numpy as np + +from . import rails, sizing +from ..execution import ledger +from ..market.exit_cost import exit_cost_fraction + + +def trade_minute(ctx, *, book: str, run_id: str, beat_no: int, broker, kind: str, mints: list, infos: list, scores, threshold, + table: list | None, horizon_s: float, block: str | None = None, holds=None, strategies=None, allow=None, reasons=None, + tables: list | None = None) -> dict: + """Optional per-row arguments (the selector's strategy stack, train/strategies.decide): ``threshold`` may be per row, + ``holds`` (s), ``strategies``, ``tables`` (sizing) and ``allow`` / ``reasons`` (a blocked row is recorded with its reason). + Left out, the call behaves exactly as the single-strategy engine.""" + conn, m1 = ctx.conn, ctx.m1 + beat = conn.execute("INSERT INTO beats (run_id, ts, beat_no, n_slots_active, notes) VALUES (%s,%s,%s,%s,%s) RETURNING id", + (run_id, m1, beat_no, len(mints), json.dumps({"minute": m1.isoformat(), "mints_traded": len(ctx.agg), "book": book}))).fetchone()["id"] + n_exit = 0 + for p in ledger.open_positions(conn, book): + held = (m1 - p["opened_at"]).total_seconds() + if held < float(p.get("hold_s") or horizon_s): + continue + did = conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, size_sol, forced, reason, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,false,%s,%s) RETURNING id", + (beat, run_id, m1, p["mint"], p["pool"], f"{kind}_exit", float(p["cost_sol"]), f"held {held/60:.0f} min", json.dumps({"book": book}))).fetchone()["id"] + px = ctx.prices.get(p["mint"]) or float(p.get("last_mark_price") or p["entry_price"]) + broker.sell(conn, position=p, decision_id=did, price=ctx.prices.get(p["mint"]), res_quote_sol=ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"]), + mcap_sol=ctx.mcap(p["mint"], px), program_label=p.get("program_label"), forced_kind=None, pool_fee=ctx.fees.get(p["mint"]), ts=m1) + n_exit += 1 + opens_now = ledger.open_positions(conn, book); open_mints = {p["mint"] for p in opens_now} + cash = ledger.paper_cash(conn, book); n_enter = 0; picks = []; entries = [] + bankroll = cash + sum(float(p["cost_sol"]) for p in opens_now) # wealth at cost: the sizing base (agent/sizing.py) + circuit = conn.execute("SELECT kill_switch, entries_paused FROM circuit_state WHERE id = 1").fetchone() + blocked = (block or ("kill switch" if circuit and circuit["kill_switch"] else None) or ("paused" if circuit and circuit["entries_paused"] else None) + or ("book halted" if rails.book_halted(conn, book) else None)) + thr_v = np.broadcast_to(np.asarray(threshold, dtype=np.float64), (len(mints),)) if len(mints) else np.array([]) + for k, (m, i, sc) in enumerate(zip(mints, infos, scores)): + thr_k = float(thr_v[k]) + if not np.isfinite(sc) or sc < thr_k: + continue + picks.append((m, float(sc))) + tab_k = tables[k] if tables is not None else table + size, why = sizing.size_position(float(sc), thr_k, tab_k or [], bankroll, cash, i["resq"]) + denied = allow is not None and not bool(allow[k]) + if m in open_mints or blocked or denied or size <= 0: + rail = blocked or ("held" if m in open_mints else (str(reasons[k]) if denied and reasons is not None else "filtered") if denied else "sizing") + conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, m_hat, size_sol, forced, rail, reason, detail) VALUES (%s,%s,%s,%s,%s,'blocked',%s,%s,false,%s,%s,%s)", + (beat, run_id, m1, m, i["pool"], float(sc), size, rail, why if rail == "sizing" else f"{kind} pick not taken", + json.dumps({"book": book, "score": float(sc), "threshold": thr_k, "sizing": why, + "strategy": None if strategies is None else strategies[k]}))) + continue + hold_k = float(holds[k]) if holds is not None else None; strat_k = None if strategies is None else strategies[k] + did = conn.execute("INSERT INTO decisions (beat_id, run_id, ts, mint, pool, kind, m_hat, size_sol, forced, reason, book_targets, detail) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,false,%s,%s,%s) RETURNING id", + (beat, run_id, m1, m, i["pool"], f"{kind}_enter", float(sc), size, f"{strat_k + ': ' if strat_k else ''}score {sc:.3f} ≥ {thr_k:.3f}; {why}", [book], + json.dumps({"book": book, "score": float(sc), "threshold": thr_k, "size_sol": size, "sizing": why, "resq": i["resq"], "age_h": i["age_h"], "price": i["price"], + "strategy": strat_k, "hold_s": hold_k}))).fetchone()["id"] + fill = broker.buy(conn, decision_id=did, mint=m, pool=i["pool"], size_sol=size, price=i["price"], res_quote_sol=i["resq"], + mcap_sol=i["mcap"], decimals=i["decimals"], program_label=i["program_label"], pool_fee=i["fee_rate"], ts=m1, hold_s=hold_k, strategy=strat_k) + if fill.ok: + n_enter += 1; open_mints.add(m); cash -= size + entries.append({"mint": m, "decision_id": did, "score": float(sc), "size_sol": size, "info": i, "threshold": thr_k, "table": tab_k, "hold_s": hold_k, "strategy": strat_k}) + else: + conn.execute("UPDATE decisions SET rail = %s, reason = %s WHERE id = %s", ("paper_fill", fill.reason, did)) + ledger.mark_positions(conn, book, ctx.prices, {}, m1) + opens = ledger.open_positions(conn, book); gross_v = 0.0; exit_cost = 0.0; exposure = 0.0 + for p in opens: + px = float(p.get("last_mark_price") or p["entry_price"]) + gross = int(p["qty"]) / 10 ** int(p.get("decimals") or 6) * px + rq = ctx.resqs.get(p["mint"]) or ctx.last_resq(p["mint"]) + gross_v += gross; exposure += float(p["cost_sol"]) + exit_cost += gross * exit_cost_fraction(gross, rq, ctx.mcap(p["mint"], px), p.get("program_label"), ctx.fees.get(p["mint"])) + cash = ledger.paper_cash(conn, book); positions_net = gross_v - exit_cost; wealth = cash + positions_net + peak_row = conn.execute("SELECT max(wealth) AS pk FROM wealth_marks WHERE book = %s", (book,)).fetchone(); peak = max(float(peak_row["pk"] or 0.0), wealth) + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (beat, book, m1, cash, positions_net, exit_cost, wealth, peak, (1.0 - wealth / peak) if peak > 0 else 0.0, exposure, len(opens))) + rails.check_book_drawdown(conn, book, wealth, peak) # the book's own kill switch: blocks its next minutes' entries + return {"beat_id": beat, "entries": entries, "entered": n_enter, "exited": n_exit, "open": len(opens), "cash": cash, "wealth": wealth, "picks": len(picks), "blocked": blocked, + "top_scores": sorted(picks, key=lambda x: -x[1])[:5], "open_mints": sorted({p["mint"] for p in opens})} diff --git a/fly_trader/agent/rails.py b/fly_trader/agent/rails.py new file mode 100644 index 0000000000000000000000000000000000000000..888e8ecff6bcf174e857c3be5c29f93a81e8dddf --- /dev/null +++ b/fly_trader/agent/rails.py @@ -0,0 +1,122 @@ +"""Risk rails outside the model. KillSwitch (−KILL_SWITCH_DRAWDOWN from peak wealth → no entries), CircuitBreaker +(CIRCUIT_THRESHOLD consecutive execution failures), the rolling 24 h notional ledger, and PauseEntries (console +toggle). State lives in circuit_state / circuit_events / notional_ledger; the trading engine reads it before entries. +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction + +log = logging.getLogger(__name__) + + +@dataclass +class CircuitState: + fail_count: int = 0 + tripped: bool = False + kill_switch: bool = False + kill_reason: str | None = None + peak_wealth: float | None = None + entries_paused: bool = False + + +def load_circuit(conn) -> CircuitState: + r = conn.execute("SELECT * FROM circuit_state WHERE id = 1").fetchone() + if not r: + return CircuitState() + return CircuitState(r["fail_count"], r["tripped"], r["kill_switch"], r["kill_reason"], r["peak_wealth"], r["entries_paused"]) + + +def _event(conn, kind: str, detail: dict | None = None) -> None: + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES (%s, %s)", (kind, json.dumps(detail or {}, default=str))) + + +def record_failure(conn, reason: str) -> CircuitState: + r = conn.execute( + "UPDATE circuit_state SET fail_count = fail_count + 1, last_failure_ts = now(), updated_at = now() WHERE id = 1 RETURNING fail_count, tripped" + ).fetchone() + tripped = r["tripped"] + if r["fail_count"] >= config.CIRCUIT_THRESHOLD and not tripped: + conn.execute("UPDATE circuit_state SET tripped = true, updated_at = now() WHERE id = 1") + _event(conn, "tripped", {"reason": reason, "fail_count": r["fail_count"]}) + record_event("error", "rails", "circuit breaker tripped", {"reason": reason}) + return load_circuit(conn) + + +def record_success(conn) -> None: + conn.execute("UPDATE circuit_state SET fail_count = 0, updated_at = now() WHERE id = 1") + + +def check_drawdown(conn, wealth: float, peak: float) -> bool: + """Trip the kill switch when ``wealth`` has fallen KILL_SWITCH_DRAWDOWN below ``peak`` (the book's own peak: the + trading engine passes the highest wealth mark of the current run). Returns whether the kill switch is on.""" + st = load_circuit(conn) + if st.kill_switch: + return True + if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN): + conn.execute("UPDATE circuit_state SET kill_switch = true, kill_reason = %s, peak_wealth = %s, updated_at = now() WHERE id = 1", + (f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL", peak)) + _event(conn, "kill_switch", {"wealth": wealth, "peak": peak}) + record_event("error", "rails", "kill switch tripped: entries blocked", {"wealth": wealth, "peak": peak}) + return True + return False + + +def check_book_drawdown(conn, book: str, wealth: float, peak: float) -> bool: + """A paper race book's own kill switch: KILL_SWITCH_DRAWDOWN below the book's peak halts that book's entries only, + so one book's drawdown never stops the other during the race. Returns whether the book is halted.""" + if book_halted(conn, book): + return True + if peak > 0 and wealth <= peak * (1.0 - config.KILL_SWITCH_DRAWDOWN): + reason = f"wealth {wealth:.4f} SOL <= {1 - config.KILL_SWITCH_DRAWDOWN:.2f} x peak {peak:.4f} SOL" + conn.execute("INSERT INTO book_state (book, halted, reason, peak, updated_at) VALUES (%s, true, %s, %s, now()) " + "ON CONFLICT (book) DO UPDATE SET halted = true, reason = EXCLUDED.reason, peak = EXCLUDED.peak, updated_at = now()", + (book, reason, peak)) + _event(conn, "book_halt", {"book": book, "wealth": wealth, "peak": peak}) + record_event("error", "rails", f"{book}: drawdown halt, entries blocked", {"wealth": wealth, "peak": peak}) + return True + return False + + +def book_halted(conn, book: str) -> bool: + r = conn.execute("SELECT halted FROM book_state WHERE book = %s", (book,)).fetchone() + return bool(r and r["halted"]) + + +def clear_book_halt(book: str) -> None: + with transaction() as conn: + conn.execute("UPDATE book_state SET halted = false, reason = NULL, peak = NULL, updated_at = now() WHERE book = %s", (book,)) + _event(conn, "book_resumed", {"book": book}) + record_event("info", "rails", f"{book}: drawdown halt cleared") + + +def notional_24h(conn) -> float: + r = conn.execute("SELECT COALESCE(sum(sol), 0) AS s FROM notional_ledger WHERE ts > now() - interval '24 hours'").fetchone() + return float(r["s"]) + + +def record_notional(conn, sol: float, kind: str = "entry") -> None: + conn.execute("INSERT INTO notional_ledger (sol, kind) VALUES (%s, %s)", (sol, kind)) + + +def reset_circuit(kill: bool = False) -> None: + with transaction() as conn: + conn.execute("UPDATE circuit_state SET fail_count = 0, tripped = false, updated_at = now() WHERE id = 1") + _event(conn, "reset", {"kill": kill}) + if kill: + conn.execute("UPDATE circuit_state SET kill_switch = false, kill_reason = NULL, peak_wealth = NULL, updated_at = now() WHERE id = 1") + record_event("info", "rails", "circuit reset" + (" incl. kill switch (peak re-based)" if kill else "")) + print("circuit reset" + (" + kill switch cleared" if kill else "")) + + +def set_entries_paused(paused: bool) -> None: + with transaction() as conn: + conn.execute("UPDATE circuit_state SET entries_paused = %s, updated_at = now() WHERE id = 1", (paused,)) + _event(conn, "paused" if paused else "resumed", {}) + record_event("info", "rails", "entries paused" if paused else "entries resumed") + print("entries paused" if paused else "entries resumed") diff --git a/fly_trader/agent/runner.py b/fly_trader/agent/runner.py new file mode 100644 index 0000000000000000000000000000000000000000..d00acdd398ed1a009a6886a4ecefe8528d91084b --- /dev/null +++ b/fly_trader/agent/runner.py @@ -0,0 +1,64 @@ +"""The `runner` worker: the selector session (``agent/selector_session.py``) trading the paper book every minute on +the market feed. One runner per database (a Postgres advisory lock). ``LIVE_ENABLED=1`` passes the signing guard first.""" +from __future__ import annotations + +import logging + +from .. import config +from ..db.apilog import record_event +from ..db.connection import connect +from ..logging_setup import setup + +log = logging.getLogger(__name__) + + +RUNNER_LOCK_KEY = 0x666C795F72756E # "fly_run": one runner per database, held for the runner's lifetime + + +def acquire_runner_lock(): + """Session advisory lock on a dedicated autocommit connection; closing the connection releases it. + Raises RuntimeError when another runner (process or console thread) already holds it.""" + conn = connect(autocommit=True) + try: + ok = conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (RUNNER_LOCK_KEY,)).fetchone()["ok"] + except Exception: + conn.close() + raise + if not ok: + conn.close() + raise RuntimeError("another runner holds the runner lock; refusing to start a second one") + return conn + + +def main(stop_event=None) -> None: + setup("runner") + try: + lock_conn = acquire_runner_lock() + except RuntimeError as e: + log.error("%s", e) + record_event("error", "runner", str(e)) + raise + try: + _main(stop_event) + finally: + lock_conn.close() + + +def _main(stop_event=None) -> None: + live = False + if config.LIVE_ENABLED: + from ..chain.cluster_guard import SigningRefused, assert_signing_allowed + try: + assert_signing_allowed(); live = True + except SigningRefused as e: + if config.SOLANA_CLUSTER != "mainnet-beta": + raise # a wrong cluster is a configuration error, never a wait + live = True # the per-minute gate (fly_session._live) keeps it in paper until the wallet is ready + log.error("LIVE_ENABLED=1 but %s -- trading paper only until that is fixed", e) + record_event("error", "runner", "live trading requested but not possible yet: holding in paper", {"why": str(e)}) + if config.RESET_ON_START: # a fresh runner never carries over a previous run's books or stats + from ..ops.reset import reset_training_state + reset_training_state(reason="runner start") + from . import minute_engine + record_event("info", "runner", "runner started", {"live": live}) + minute_engine.main(stop_event=stop_event, live=live) diff --git a/fly_trader/agent/selector_session.py b/fly_trader/agent/selector_session.py new file mode 100644 index 0000000000000000000000000000000000000000..cac1d7801e7707a1b36e0b4cde24a0afe2a90373 --- /dev/null +++ b/fly_trader/agent/selector_session.py @@ -0,0 +1,162 @@ +"""The selector's book: the GBM selector trading ``paper_selector`` on the minute engine (agent/minute_engine.py). + +Each minute the engine's eligible rows (full feature vectors) are scored by the deployed selector (``brain_snapshots`` +kind 'selector') with its columns picked by name; the book trades them with the shared paper rules +(agent/paper_trading.py): scores at or above the model's line open positions sized from its measured certainty, held +``horizon_min`` minutes, then sold at the last traded price (the training label's exit). A newer deployable model is +picked up without a restart (``maybe_reload``). After the handover (agent/handover.py) the book stops entering while the +fly trades, exits to flat and retires. +""" +from __future__ import annotations + +import json +import logging +import uuid +from datetime import datetime, timezone + +import numpy as np + +from ..db.apilog import record_event +from ..db.connection import transaction +from ..execution import ledger +from ..execution.broker_paper import PaperBroker +from ..train.decisions import X_COLS +from . import handover, paper_trading + +log = logging.getLogger(__name__) +BOOK, KIND = "paper_selector", "selector" + + +class NoModel(RuntimeError): + """No selector has qualified to trade (current data and a profitable backtest after costs).""" + + +def _columns(model) -> np.ndarray: + missing = [c for c in model.cols if c not in X_COLS] + if missing: + raise RuntimeError(f"selector expects features the live engine does not produce: {missing}") + return np.asarray([X_COLS.index(c) for c in model.cols], dtype=int) + + +def pinned_snapshot() -> int | None: + """The selector snapshot the operator has pinned, or None. A pin freezes the book on one model: a forward test that + silently swapped models mid-run could not say which one earned the result.""" + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'pinned_selector_snapshot'").fetchone() + if not r: + return None + v = r["value"] + v = v.get("id") if isinstance(v, dict) else json.loads(v or "{}").get("id") + return int(v) if v is not None else None + + +class SelectorBook: + name = "selector" + + def __init__(self): + from ..train import selector as sel + pin = pinned_snapshot() + self.model = sel.load_snapshot(pin) if pin is not None else sel.load_latest() + if self.model is None: + raise NoModel("no model has qualified to trade yet (trained on the current data, with a backtest that made money after costs and beat random picks)") + self.idx = _columns(self.model) + self.horizon_s = self.model.horizon_min * 60 + self.broker = PaperBroker(BOOK); self.done = False + self.run_id = str(uuid.uuid4()); self.beat_no = 0 + with transaction() as conn: + self.snapshot_id = pin if pin is not None else sel.latest_current(conn)["id"] # the snapshot actually loaded + conn.execute("INSERT INTO runs (run_id, kind, config, brain_snapshot_id, status) VALUES (%s,%s,%s,%s,'running')", + (self.run_id, KIND, json.dumps({"threshold": self.model.threshold, "horizon_min": self.model.horizon_min, "book": BOOK}, default=str), self.snapshot_id)) + record_event("info", "selector", "selector session started", {"run_id": self.run_id, "snapshot": self.snapshot_id, "threshold": self.model.threshold, "horizon_min": self.model.horizon_min}) + log.info("selector session: snapshot %d threshold %.4f horizon %d min", self.snapshot_id, self.model.threshold, self.model.horizon_min) + + def maybe_reload(self) -> bool: + """Switch to a newer deployable model without a restart (the paper book and open positions carry on). While a + snapshot is pinned (``ui_settings['pinned_selector_snapshot']``) the book stays on it, so a forward test measures + one model instead of a blend of every retrain that lands during it.""" + import joblib + from ..train import selector as sel + if pinned_snapshot() is not None: + return False + with transaction() as conn: + r = sel.latest_current(conn) + if not r or r["id"] == self.snapshot_id: + return False + m = joblib.load(r["path"]) + try: + idx = _columns(m) + except RuntimeError as e: + log.error("model #%d: %s; keeping #%s", r["id"], e, self.snapshot_id) + return False + old = self.snapshot_id; self.model, self.idx, self.snapshot_id, self.horizon_s = m, idx, r["id"], m.horizon_min * 60 + with transaction() as conn: + conn.execute("UPDATE runs SET brain_snapshot_id = %s, config = %s WHERE run_id = %s", + (r["id"], json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "book": BOOK}, default=str), self.run_id)) + record_event("info", "selector", f"switched to model #{r['id']}", {"from": old, "to": r["id"], "threshold": m.threshold}) + log.info("switched from model #%s to #%d (threshold %.4f)", old, r["id"], m.threshold) + return True + + def on_bars(self, t_start: float, bars: dict) -> None: + pass + + def open_mints(self, conn) -> set[str]: + return {p["mint"] for p in ledger.open_positions(conn, BOOK)} + + def _decide(self, ctx) -> dict | None: + """The strategy stack's per-row decision (train/strategies.decide), with the live fail-closed rules: rows whose + inputs training had but live lacks are blocked — the skill table of the day missing while skill inputs are used, a + token graduated inside the archive whose curve facts are not known yet while rug inputs are used.""" + if not (getattr(self.model, "stack", None) or {}).get("strategies") or not len(ctx.mints): + return None + d = self.model.decide(ctx.X, X_COLS, ctx.t_start) + groups = set((getattr(self.model, "metrics", None) or {}).get("groups") or []) + for k, i in enumerate(ctx.infos): + if not d["allow"][k]: + continue + if "skill" in groups and i.get("skill_missing"): + d["allow"][k] = False; d["reason"][k] = "fail closed: no wallet-skill table for today" + elif "rug" in groups and i.get("age_h") is not None and not i.get("curve_known"): + d["allow"][k] = False; d["reason"][k] = "fail closed: curve facts not known yet" + return d + + def on_minute(self, ctx) -> dict: + conn = ctx.conn + d = self._decide(ctx) + if d is not None: + scores = np.where(np.isfinite(d["score"]), d["score"], -1.0); thr = d["threshold"] + else: + scores = self.model.score(ctx.X[:, self.idx]) if len(ctx.mints) else np.array([]); thr = self.model.threshold + summary = {"minute": ctx.m1.isoformat(), "mints_traded": len(ctx.agg), "eligible": len(ctx.mints), + "picks": int((scores >= thr).sum()) if len(scores) else 0, + "score_p99": float(np.percentile(scores, 99)) if len(scores) else None, "score_max": float(scores.max()) if len(scores) else None, + "threshold": self.model.threshold, "snapshot": self.snapshot_id, + "strategies": sorted(((getattr(self.model, "stack", None) or {}).get("strategies") or {"ev": None}).keys())} + if not ctx.trade: + return summary + if not ctx.fresh: + status = {**summary, "stage": "stream stale: holding (no entries or exits)", "updated_at": datetime.now(timezone.utc).isoformat()} + self._write_status(conn, status) + return status + handed = handover.state(conn) is not None and ctx.engine.has_book("fly") + self.beat_no += 1 + extra = {} if d is None else {"holds": d["hold_s"], "strategies": d["strategy"], "allow": d["allow"], "reasons": d["reason"], "tables": d["tables"]} + st = paper_trading.trade_minute(ctx, book=BOOK, run_id=self.run_id, beat_no=self.beat_no, broker=self.broker, kind=KIND, mints=ctx.mints, infos=ctx.infos, + scores=scores, threshold=thr, table=getattr(self.model, "sizing", None), horizon_s=self.horizon_s, + block="handed over to the fly" if handed else None, **extra) + if handed and st["open"] == 0: + self.done = True + record_event("info", "selector", "selector retired: the fly holds the seat and the book is flat", {"run_id": self.run_id}) + status = {**summary, **st, "stage": "retired (handed over to the fly)" if self.done else ("exiting to flat (handed over to the fly)" if handed else "trading"), + "latency_s": (datetime.now(timezone.utc) - ctx.m1).total_seconds(), "updated_at": datetime.now(timezone.utc).isoformat()} + self._write_status(conn, status) + return status + + @staticmethod + def _write_status(conn, status: dict) -> None: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('selector_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(status, default=str),)) + + def finish(self) -> None: + with transaction() as conn: + conn.execute("UPDATE runs SET ended_at = now(), status = 'finished' WHERE run_id = %s", (self.run_id,)) + record_event("info", "selector", "selector session stopped", {"run_id": self.run_id, "minutes": self.beat_no}) diff --git a/fly_trader/agent/sizing.py b/fly_trader/agent/sizing.py new file mode 100644 index 0000000000000000000000000000000000000000..e5dcdaa723b21f2772b74aa92462c894ec482e93 --- /dev/null +++ b/fly_trader/agent/sizing.py @@ -0,0 +1,106 @@ +"""Position sizing: how much of the bankroll a buy gets, from how certain the model is that the trade will profit. + +Certainty is measured, not assumed. During training the backtest's out-of-sample trades are grouped by how far their +score cleared the buy line (``build_table``: equal-count bands of ``score − threshold``). For each band the growth-optimal +(Kelly) fraction is found on the band's actual net returns — the bet f that maximises the average of log(1 + f·r), so the +rare −100 % rugs count in full; a band with no edge gets 0. Live (``size_position``): the current score's band → +``KELLY_FRACTION`` of that fraction of the deployable bankroll (wealth at cost minus the gas reserve), capped at +``MAX_POSITION_FRACTION`` of it, ``MAX_POOL_SHARE`` of the pool's quote reserve (bigger buys pay more price impact than +the 0.1 SOL the backtest measured) and the free cash above the reserve; below ``MIN_POSITION_SOL`` the buy is skipped. +A model without a table (trained before sizing) trades the fixed ``MAX_POSITION_SOL``. +""" +from __future__ import annotations + +import numpy as np + +from .. import config + +N_BANDS = 4 +MIN_BAND_TRADES = 50 +_F_GRID = np.linspace(0.0, 0.99, 100) + + +def kelly_fraction(r: np.ndarray) -> float: + """Fraction of the bankroll per bet that maximises mean log growth over the empirical net returns ``r`` (0 if no edge).""" + r = np.asarray(r, dtype=float) + r = r[np.isfinite(r)] + if len(r) == 0 or r.mean() <= 0: + return 0.0 + growth = np.log1p(np.clip(_F_GRID[:, None] * r[None, :], -0.999999, None)).mean(axis=1) + return float(_F_GRID[int(np.argmax(growth))]) + + +def build_table(margins: np.ndarray, returns: np.ndarray, n_bands: int = N_BANDS) -> list[dict]: + """Bands of score margin (score − the model's threshold) over out-of-sample trades, lowest first: + [{lo, n, mean, win, kelly}] where ``lo`` is the band's lowest margin (the first band starts at 0).""" + m = np.asarray(margins, dtype=float); r = np.asarray(returns, dtype=float) + ok = np.isfinite(m) & np.isfinite(r); m, r = m[ok], r[ok] + if len(r) < MIN_BAND_TRADES: + return [] + k = max(1, min(n_bands, len(r) // MIN_BAND_TRADES)) + edges = np.quantile(m, np.linspace(0, 1, k + 1)); edges[0] = 0.0 + out = [] + for b in range(k): + sel = (m >= edges[b]) & ((m < edges[b + 1]) if b < k - 1 else np.ones(len(m), bool)) + rb = r[sel] + out.append({"lo": float(edges[b]), "n": int(len(rb)), "mean": float(rb.mean()) if len(rb) else None, + "win": float((rb > 0).mean()) if len(rb) else None, "kelly": kelly_fraction(rb)}) + return out + + +def band_for(table: list[dict], margin: float) -> dict | None: + best = None + for b in table: + if margin >= b["lo"]: + best = b + return best + + +def size_position(score: float, threshold: float, table: list[dict] | None, bankroll_sol: float, cash_sol: float, + res_quote_sol: float | None) -> tuple[float, str]: + """(size in SOL, why). Size 0 means skip the buy.""" + reserve = config.GAS_RESERVE_SOL; free = cash_sol - reserve + if not table: + size = config.MAX_POSITION_SOL + return (size, f"fixed {size:g} SOL (model has no sizing table)") if free >= size else (0.0, "not enough free cash") + b = band_for(table, score - threshold) + if b is None or b["kelly"] <= 0: + return 0.0, "no edge in this score band" + deployable = max(0.0, bankroll_sol - reserve) + size = config.KELLY_FRACTION * b["kelly"] * deployable + caps = {"bankroll cap": config.MAX_POSITION_FRACTION * deployable, "free cash": free} + if res_quote_sol: + caps["pool depth"] = config.MAX_POOL_SHARE * res_quote_sol + limit = min(caps, key=caps.get) + why = f"band from +{b['lo']:.3f} over the line: Kelly {b['kelly']:.2f} × {config.KELLY_FRACTION:g} of {deployable:.2f} SOL" + if caps[limit] < size: + size = caps[limit]; why += f", capped by {limit}" + if size < config.MIN_POSITION_SOL: + return 0.0, why + f" → {size:.3f} SOL is below the {config.MIN_POSITION_SOL:g} SOL minimum" + return float(size), why + f" → {size:.3f} SOL" + + +def simulate_bankroll(ts: np.ndarray, horizon_s, margins: np.ndarray, returns: np.ndarray, table: list[dict] | None, + start_sol: float | None = None, res_quote: np.ndarray | None = None, tables: list | None = None) -> dict: + """Replay out-of-sample trades in entry order with the sizing rule (positions overlap for ``horizon_s``; cash, the + reserve and — given ``res_quote``, each trade's pool quote reserve — the pool-depth cap are enforced, as live). + Returns the final bankroll, its multiple and the worst drawdown.""" + start = float(start_sol or config.CAPITAL_SOL or 1.0); cash = start; open_: list[tuple[float, float, float]] = [] # (exit_ts, cost, return) + hv = np.asarray(horizon_s, dtype=np.float64); hold = np.full(len(ts), float(hv)) if hv.ndim == 0 else hv # per-trade holds allowed + peak = start; mdd = 0.0; n = 0 + for i in np.argsort(ts, kind="stable"): + t = float(ts[i]) + still = [] + for x in open_: + if x[0] <= t: + cash += x[1] * (1 + x[2]) + else: + still.append(x) + open_ = still + bankroll = cash + sum(x[1] for x in open_) + size, _ = size_position(float(margins[i]), 0.0, tables[i] if tables is not None else table, bankroll, cash, float(res_quote[i]) if res_quote is not None else None) + if size > 0: + cash -= size; open_.append((t + float(hold[i]), size, float(returns[i]))); n += 1 + w = cash + sum(x[1] for x in open_); peak = max(peak, w); mdd = max(mdd, 1 - w / peak if peak > 0 else 0.0) + final = cash + sum(x[1] * (1 + x[2]) for x in open_) + return {"start_sol": start, "final_sol": final, "multiple": final / start if start > 0 else None, "max_drawdown": mdd, "trades": n} diff --git a/fly_trader/brain/__init__.py b/fly_trader/brain/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/brain/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/brain/connectome.py b/fly_trader/brain/connectome.py new file mode 100644 index 0000000000000000000000000000000000000000..003af78dc9f3dd49855690c8912f0a0fbdb7f637 --- /dev/null +++ b/fly_trader/brain/connectome.py @@ -0,0 +1,171 @@ +"""The FlyWire connectome the fly (``train/fly_selector.py``) is built on: the artifact written by ``build-connectome`` +(``brain/connectome_build.py``), loaded as signed synapse counts with population ranges, and restricted to a set of +populations by ``SubConnectome``. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import numpy as np +import torch + +from .. import config + +CONNECTOME_DIR = config.BRAIN_DIR / "connectome" +CURRENT_FILE = CONNECTOME_DIR / "current.txt" +SCALE_FILE = CONNECTOME_DIR / "scale.json" +AFFERENT_POPS = ("ORN_FOOD", "ORN_DANGER", "GRN_SWEET", "GRN_BITTER", "GRN_OTHER", "MECH_JO", "MECH_BRISTLE", + "MECH_OTHER", "THERMO_WARM", "THERMO_COOL", "THERMO_OTHER") # sensory neurons: where the features enter +EFFERENT_POP = "DESCENDING" # brain-to-body neurons: where the decoder reads + + +def resolve_device(name: str | None = None) -> torch.device: + """config.DEVICE with automatic fallback to CPU when MPS (or CUDA) is unavailable.""" + name = (name or config.DEVICE or "cpu").lower() + if name.startswith("mps"): + if torch.backends.mps.is_available(): + return torch.device("mps") + print("[connectome] MPS unavailable; falling back to cpu", file=sys.stderr) + return torch.device("cpu") + if name.startswith("cuda"): + if torch.cuda.is_available(): + return torch.device(name) + print("[connectome] CUDA unavailable; falling back to cpu", file=sys.stderr) + return torch.device("cpu") + return torch.device("cpu") + + +def current_connectome_path() -> Path: + if not CURRENT_FILE.exists(): + raise FileNotFoundError(f"{CURRENT_FILE} missing: run `fly-trader build-connectome` first") + name = CURRENT_FILE.read_text().strip() + path = Path(name) if Path(name).is_absolute() else CONNECTOME_DIR / name + if not path.exists(): + raise FileNotFoundError(f"connectome {path} named in current.txt does not exist") + return path + + +def _scale_for(connectome_file: str) -> dict | None: + """The calibrated scale if scale.json exists and refers to this artifact.""" + if not SCALE_FILE.exists(): + return None + try: + d = json.loads(SCALE_FILE.read_text()) + except Exception: + return None + return d if d.get("connectome") in (connectome_file, Path(connectome_file).name) else None + + +MB_POPS = ("KC", "MBON_APP", "MBON_AV", "MBON_OTHER") # the mushroom-body block the build keeps apart from the sparse graph +DAN_POPS = ("DAN_PAM", "DAN_PPL1", "DAN_OTHER") + + +def mbon_compartments(indices: torch.Tensor, values_raw: torch.Tensor, pop_ranges: dict, cell_type=None) -> tuple[np.ndarray, list[str]]: + """Each MBON column's compartment: the dopamine cell type (or, without cell types, the DAN population) sending it the + most synapse mass in the raw graph — before scale.json's gains silence DAN→MBON as fast edges (Aso et al. 2014: a + compartment = the DAN type innervating that MBON's dendrites). −1 when no DAN reaches the MBON.""" + if not all(p in pop_ranges for p in ("MBON_APP", "MBON_OTHER")): + return np.zeros(0, int), [] + m0, m1 = pop_ranges["MBON_APP"][0], pop_ranges["MBON_OTHER"][1] + post, pre = indices[0].numpy(), indices[1].numpy(); w = np.abs(values_raw.numpy()) + dan = np.zeros(len(pre), bool); label = np.full(len(pre), "", dtype=object) + for pop in DAN_POPS: + if pop in pop_ranges: + a, b = pop_ranges[pop]; sel = (pre >= a) & (pre < b); dan |= sel + label[sel] = np.asarray(cell_type, dtype=object)[pre[sel]] if cell_type is not None else pop + e = dan & (post >= m0) & (post < m1) + names = sorted({str(x) or "DAN" for x in label[e]}) + comp = np.full(m1 - m0, -1, int) + if not e.any(): + return comp, names + mass = np.zeros((m1 - m0, len(names))); col = {n: i for i, n in enumerate(names)} + np.add.at(mass, (post[e] - m0, np.array([col[str(x) or "DAN"] for x in label[e]])), w[e]) + has = mass.sum(1) > 0; comp[has] = mass[has].argmax(1) + return comp, names + + +def apply_pathway_gains(indices: torch.Tensor, values_raw: torch.Tensor, pop_ranges: dict, gains: dict) -> tuple[torch.Tensor, torch.Tensor]: + """Multiply the edges from population A to population B by ``gains['A->B']``; edges scaled to 0 are dropped. + scale.json silences the direct DAN→MBON/KC edges: dopamine acts on the mushroom body through KC→MBON plasticity, + not fast excitation (Hige 2015, Handler 2019).""" + post, pre = indices[0], indices[1] + v = values_raw.clone() + for key, g in (gains or {}).items(): + a, _, b = key.partition("->") + if a not in pop_ranges or b not in pop_ranges: + continue + (a0, a1), (b0, b1) = pop_ranges[a], pop_ranges[b] + v[(pre >= a0) & (pre < a1) & (post >= b0) & (post < b1)] *= float(g) + keep = v != 0 + return indices[:, keep], v[keep] + + +class Connectome: + """Signed synapse counts ``values_raw`` [E] on edges ``indices`` [2, E] (row = post, col = pre) over ``N`` neurons, + population ``pop_ranges``, and the weight scale ``s`` (scale.json when it refers to this artifact, else the build's + s0). The KC→MBON synapses are not in the sparse graph: the build keeps them as the dense block ``W_KM0`` + [n_KC, n_MBON] (synapse counts; MBON columns = MBON_APP, MBON_AV, MBON_OTHER in population order) with the mask + ``M_KM`` of the pairs that exist. scale.json's ``pathway_gains`` are applied to the sparse graph. Tensors stay on + the CPU; ``device`` is where models built on it should run.""" + + def __init__(self, path: str | Path | None = None, device: torch.device | str | None = None): + path = Path(path) if path else current_connectome_path() + z = np.load(path, allow_pickle=False) + self.path = path + self.N = int(z["N"]) + self.pop_ranges = {k: (int(a), int(b)) for k, (a, b) in json.loads(str(z["pop_ranges"])).items()} + scale = _scale_for(path.name) + self.pathway_gains = dict((scale or {}).get("pathway_gains") or {}) + raw_i = torch.from_numpy(np.ascontiguousarray(z["indices"])).to(torch.int64); raw_v = torch.from_numpy(np.ascontiguousarray(z["values_raw"])).to(torch.float32) + self.mbon_compartment, self.compartment_names = mbon_compartments(raw_i, raw_v, self.pop_ranges, z["cell_type"] if "cell_type" in z.files else None) + self.indices, self.values_raw = apply_pathway_gains(torch.from_numpy(np.ascontiguousarray(z["indices"])).to(torch.int64), + torch.from_numpy(np.ascontiguousarray(z["values_raw"])).to(torch.float32), + self.pop_ranges, self.pathway_gains) + self.W_KM0 = torch.from_numpy(np.ascontiguousarray(z["W_KM0"])).to(torch.float32) + self.M_KM = torch.from_numpy(np.ascontiguousarray(z["M_KM"])).to(torch.bool) + self.spectral_radius_raw = float(z["spectral_radius_raw"]) + self.content_sha256 = str(z["content_sha256"]) + self.s = float(scale["s"]) if scale and "s" in scale else float(z["s0"]) + self.device = device if isinstance(device, torch.device) else resolve_device(device) + + @classmethod + def load(cls, path: str | Path | None = None, device: torch.device | str | None = None) -> "Connectome": + """The artifact named in data/brain/connectome/current.txt (or ``path``).""" + return cls(path, device=device) + + +class SubConnectome: + """A connectome restricted to a set of populations (e.g. everything but the optic lobes).""" + + def __init__(self, c, exclude: tuple[str, ...] = ("VISUAL",)): + keep = torch.ones(c.N, dtype=torch.bool) + for name in exclude: + if name in c.pop_ranges: + lo, hi = c.pop_ranges[name] + keep[lo:hi] = False + new_index = torch.cumsum(keep.to(torch.int64), 0) - 1 + post, pre = c.indices[0], c.indices[1] + m = keep[post] & keep[pre] + self.indices = torch.stack([new_index[post[m]], new_index[pre[m]]]) + self.values_raw = c.values_raw[m].clone() + self.N = int(keep.sum()) + self.pop_ranges = {} + for name, (lo, hi) in c.pop_ranges.items(): + if name in exclude or hi <= lo: + continue + self.pop_ranges[name] = (int(new_index[lo]), int(new_index[hi - 1]) + 1) + self.device = c.device + self.s = getattr(c, "s", None) + self.spectral_radius_raw = getattr(c, "spectral_radius_raw", None) + self.content_sha256 = getattr(c, "content_sha256", None) + lost = [p for p in MB_POPS if p in c.pop_ranges and not bool(keep[c.pop_ranges[p][0]:c.pop_ranges[p][1]].all())] + if lost: + raise ValueError(f"a sub-connectome must keep the whole mushroom body (KC→MBON block); excluded: {lost}") + self.W_KM0 = getattr(c, "W_KM0", None) # unchanged: KC and MBON rows keep their order and ranges + self.mbon_compartment = getattr(c, "mbon_compartment", None); self.compartment_names = getattr(c, "compartment_names", []) + self.M_KM = getattr(c, "M_KM", None) + self.pathway_gains = getattr(c, "pathway_gains", {}) + self.excluded = exclude + self.keep = keep diff --git a/fly_trader/brain/connectome_build.py b/fly_trader/brain/connectome_build.py new file mode 100644 index 0000000000000000000000000000000000000000..25c16753dfc79424e42c82cd928ad1c761062122 --- /dev/null +++ b/fly_trader/brain/connectome_build.py @@ -0,0 +1,337 @@ +"""Build the FAFB v783 connectome artifact (``fly-trader build-connectome``). + +Inputs (data/brain/raw/): flybrain ``connections.csv.gz`` (pre_root_id, post_root_id, neuropil, +syn_count, nt_type; one row per (pre, post, neuropil)), ``neurons.csv.gz``, ``classification.csv.gz`` +and the Schlegel et al. 2024 annotation TSV. Output: ``data/brain/connectome/fafb783_.npz`` +(COO ``indices`` int64[2, nnz] with row = post, col = pre, sorted; ``values`` float32 pre-calibration +weights; ``values_raw`` = signed synapse counts; the dense plastic KC->MBON block ``W_KM0``) plus +``current.txt`` naming the artifact, the ``populations`` table and a ``runs`` row. + +Sign convention (config.NT_SIGN_MODE): ``shiu`` = GABA and glutamate inhibitory, everything else +excitatory (Shiu et al. 2024 Nature 634:210); ``flybrain`` = only GABA inhibitory (flybrain's +NT_SIGN). Weights are summed over neuropils per (pre, post) pair with the edge's own nt_type +(nt_type varies within a pre neuron in FlyWire's connection table); zero sums are dropped, as +flybrain does, which reproduces its 2,698,236 edges in ``flybrain`` mode. + +Scale: s0 = 0.99 / rho(|W_raw|) (spectral radius by shifted power iteration) is only the INITIAL +guess, motivated by Costi et al. 2025 (PMC12109256), who ran the FlyWire connectome as an echo-state +reservoir rescaled to rho = 0.99 (tanh units, glutamate treated as excitatory there). The fly +(``train/fly_selector.py``) initialises its synaptic magnitudes from s (``scale.json`` when present, else s0). +KC->MBON magnitudes are forced positive because KC output synapses are cholinergic (Barnstedt et +al. 2016 Neuron), regardless of FlyWire's per-cell NT prediction. +""" +from __future__ import annotations + +import hashlib +import json +import math +import os +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import warnings + +import numpy as np +import pyarrow as pa +import pyarrow.csv as pc +import torch + +warnings.filterwarnings("ignore", message="Sparse CSR tensor support is in beta state") +warnings.filterwarnings("ignore", message="Sparse invariant checks are implicitly disabled") + +from .. import config +from . import populations as P + +CONNECTIONS_CSV = P.RAW_DIR / "connections.csv.gz" +CONNECTOME_DIR = config.BRAIN_DIR / "connectome" +CURRENT_FILE = CONNECTOME_DIR / "current.txt" +EXPECTED_EDGES = 2_698_236 +W_CLIP = 1.0 # = theta (plan section 1.7) +TARGET_RADIUS = 0.99 # reservoir operating point (Vienna 2025) +POWER_ITER_MAX = 5000 +POWER_ITER_TOL = 1e-9 +SOURCE_FILES = ["connections.csv.gz", "neurons.csv.gz", "classification.csv.gz", + "Supplemental_file1_neuron_annotations.tsv", "consolidated_cell_types.csv.gz", "neuron_meta.json"] + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def nt_sign(nt: np.ndarray, mode: str) -> np.ndarray: + """+1/-1 per edge from the edge's nt_type (flybrain vocabulary ACH/GABA/GLUT/DA/SER/OCT).""" + nt = np.char.upper(nt.astype(str)) + if mode == "shiu": + neg = (nt == "GABA") | (nt == "GLUT") + elif mode == "flybrain": + neg = nt == "GABA" + else: + raise ValueError(f"unknown NT_SIGN_MODE {mode!r} (shiu | flybrain)") + return np.where(neg, -1.0, 1.0) + + +def read_connections(path: Path = CONNECTIONS_CSV) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + tbl = pc.read_csv(str(path), convert_options=pc.ConvertOptions(column_types={ + "pre_root_id": pa.int64(), "post_root_id": pa.int64(), "neuropil": pa.string(), + "syn_count": pa.int32(), "nt_type": pa.string()})) + pre = tbl["pre_root_id"].to_numpy() + post = tbl["post_root_id"].to_numpy() + syn = tbl["syn_count"].to_numpy().astype(np.int64) + nt = tbl["nt_type"].to_numpy(zero_copy_only=False).astype(str) + return pre, post, syn, nt + + +def root_id_lookup(canonical_root_ids: np.ndarray): + order = np.argsort(canonical_root_ids) + srt = canonical_root_ids[order] + + def lookup(x: np.ndarray) -> np.ndarray: + pos = np.searchsorted(srt, x) + pos[pos >= len(srt)] = 0 + return np.where(srt[pos] == x, order[pos], -1) + return lookup + + +def aggregate_edges(pre: np.ndarray, post: np.ndarray, syn: np.ndarray, nt: np.ndarray, N: int, mode: str) -> dict: + """Sum syn_count * sign over neuropils per (pre, post). Returns arrays over distinct pairs.""" + key = pre.astype(np.int64) * N + post.astype(np.int64) + uniq, inv = np.unique(key, return_inverse=True) + n = len(uniq) + sgn = nt_sign(nt, mode) + signed = np.bincount(inv, weights=syn * sgn, minlength=n) + signed_fb = np.bincount(inv, weights=syn * nt_sign(nt, "flybrain"), minlength=n) + unsigned = np.bincount(inv, weights=syn, minlength=n) + return { + "pre": (uniq // N).astype(np.int64), "post": (uniq % N).astype(np.int64), + "signed": signed, "unsigned": unsigned, + "n_pairs": int(n), "nnz_mode": int((signed != 0).sum()), "nnz_flybrain": int((signed_fb != 0).sum()), + } + + +def spectral_radius(row: np.ndarray, col: np.ndarray, values_abs: np.ndarray, N: int, + max_iter: int = POWER_ITER_MAX, tol: float = POWER_ITER_TOL) -> dict: + """rho(|W|) by power iteration on the shifted matrix |W| + I (rho(|W| + I) = rho(|W|) + 1 for a + non-negative matrix; the shift makes the iteration converge for periodic components). Also + returns the Collatz-Wielandt bracket min_i (|W|x)_i / x_i <= rho <= max_i (|W|x)_i / x_i.""" + idx = torch.from_numpy(np.stack([row, col]).astype(np.int64)) + W = torch.sparse_coo_tensor(idx, torch.from_numpy(values_abs.astype(np.float64)), (N, N)).coalesce().to_sparse_csr() + x = torch.full((N, 1), 1.0 / math.sqrt(N), dtype=torch.float64) + lam_prev, it, converged = 0.0, 0, False + for it in range(1, max_iter + 1): + y = W @ x + x + lam = float(torch.linalg.vector_norm(y)) + x = y / lam + if abs(lam - lam_prev) <= tol * lam: + converged = True + break + lam_prev = lam + wx = (W @ x).squeeze(1) + xs = x.squeeze(1) + ratio = wx / xs + return {"rho": lam - 1.0, "iterations": it, "converged": converged, + "cw_lower": float(ratio.min()), "cw_upper": float(ratio.max())} + + +def content_hash(indices: np.ndarray, values_raw: np.ndarray, W_KM0: np.ndarray, root_ids: np.ndarray, + pop_ranges_json: str, nt_sign_mode: str) -> str: + h = hashlib.sha256() + for part in (indices.tobytes(), values_raw.tobytes(), W_KM0.tobytes(), root_ids.tobytes(), + pop_ranges_json.encode(), nt_sign_mode.encode()): + h.update(part) + return h.hexdigest() + + +def _git_sha() -> str | None: + try: + import subprocess + out = subprocess.run(["git", "rev-parse", "HEAD"], cwd=config.REPO_ROOT, capture_output=True, text=True, timeout=5) + return out.stdout.strip() or None if out.returncode == 0 else None + except Exception: + return None + + +def persist(pops: P.Populations, metrics: dict, build_cfg: dict, file_sha: str, started_at: datetime, + annotations_label: str) -> str: + """populations rows (id = order index) + runs row; returns the run_id.""" + from psycopg.types.json import Jsonb + from ..db.connection import transaction + run_id = str(uuid.uuid4()) + with transaction() as conn: + with conn.cursor() as cur: + for i, name in enumerate(pops.order): + cur.execute("DELETE FROM populations WHERE name = %s AND id <> %s", (name, i)) + cur.execute( + "INSERT INTO populations (id, name, n, kind, source) VALUES (%s, %s, %s, %s, %s) " + "ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, n = EXCLUDED.n, kind = EXCLUDED.kind, " + "source = EXCLUDED.source", + (i, name, pops.n(name), pops.kind.get(name, "other"), annotations_label)) + cur.execute( + "INSERT INTO runs (run_id, kind, started_at, ended_at, git_sha, config, connectome_sha256, status, metrics) " + "VALUES (%s, 'connectome_build', %s, now(), %s, %s, %s, 'done', %s)", + (run_id, started_at, _git_sha(), Jsonb(build_cfg), file_sha, Jsonb(metrics))) + if pops.source != "annotations": + cur.execute( + "INSERT INTO events (level, source, message, detail) VALUES ('warning', 'connectome_build', %s, %s)", + ("cell types from flybrain classification.csv fallback (annotation TSV unavailable); " + "MBON valence by neurotransmitter, DAN valence by synapse counts", + Jsonb({"run_id": run_id}))) + return run_id + + +def main(annotations: str | None = None, nt_sign_mode: str | None = None, source: str | None = None, + out_dir: str | Path | None = None, persist_db: bool = True) -> Path: + started_at = datetime.now(timezone.utc) + t0 = time.perf_counter() + mode = nt_sign_mode or config.NT_SIGN_MODE + out_dir = Path(out_dir) if out_dir else CONNECTOME_DIR + out_dir.mkdir(parents=True, exist_ok=True) + + # ---- source hashes ---- + source_sha = {} + for name in SOURCE_FILES: + p = P.RAW_DIR / name + if p.exists(): + source_sha[name] = sha256_file(p) + if annotations: + source_sha[Path(annotations).name] = sha256_file(Path(annotations)) + + # ---- connections (canonical flybrain indices) ---- + pre_rid, post_rid, syn, nt = read_connections() + n_rows = len(pre_rid) + neu_ids = np.asarray(__import__("pandas").read_csv(P.NEURONS_CSV, usecols=["root_id"], dtype={"root_id": np.int64})["root_id"]) + N = len(neu_ids) + lookup = root_id_lookup(neu_ids) + pre_c, post_c = lookup(pre_rid), lookup(post_rid) + ok = (pre_c >= 0) & (post_c >= 0) + n_skipped = int((~ok).sum()) + unknown_nt = int((~np.isin(np.char.upper(nt), ["ACH", "GABA", "GLUT", "DA", "SER", "OCT", "OA"])).sum()) + pre_c, post_c, syn, nt = pre_c[ok], post_c[ok], syn[ok], nt[ok] + print(f"[build] connections rows={n_rows:,} skipped_unknown_root_id={n_skipped} unknown_nt_rows={unknown_nt} " + f"({time.perf_counter() - t0:.1f}s)") + + # ---- populations ---- + pops = P.build_populations(annotations=annotations, source=source, edges=(pre_c, post_c, syn.astype(np.float64))) + assert pops.N == N + inv = pops.inverse_index() # canonical -> sorted position + print(f"[build] populations source={pops.source} N={N:,} ({time.perf_counter() - t0:.1f}s)") + + # ---- aggregate over neuropils ---- + agg = aggregate_edges(pre_c, post_c, syn, nt, N, mode) + nnz_fb, nnz_mode, n_pairs = agg["nnz_flybrain"], agg["nnz_mode"], agg["n_pairs"] + edges_match = nnz_fb == EXPECTED_EDGES + print(f"[build] distinct (pre,post) pairs={n_pairs:,}; nonzero edges: flybrain-sign={nnz_fb:,} " + f"(expected {EXPECTED_EDGES:,} -> {'MATCH' if edges_match else 'MISMATCH'}), {mode}-sign={nnz_mode:,}") + if not edges_match: + print(f"[build] WARNING flybrain-mode edge count {nnz_fb:,} != {EXPECTED_EDGES:,}; continuing", file=sys.stderr) + keep = agg["signed"] != 0 + pre_s = inv[agg["pre"][keep]] # sorted (population-order) indices + post_s = inv[agg["post"][keep]] + signed = agg["signed"][keep] + unsigned = agg["unsigned"][keep] + + # ---- KC -> MBON block (dense, positive magnitudes) ---- + kc0, kc1 = pops.ranges["KC"] + m0 = pops.ranges["MBON_APP"][0] + m1 = pops.ranges["MBON_OTHER"][1] + assert pops.ranges["MBON_APP"][1] == pops.ranges["MBON_AV"][0] and pops.ranges["MBON_AV"][1] == pops.ranges["MBON_OTHER"][0] + n_kc, n_mbon = kc1 - kc0, m1 - m0 + is_km = (pre_s >= kc0) & (pre_s < kc1) & (post_s >= m0) & (post_s < m1) + W_KM0 = np.zeros((n_kc, n_mbon), dtype=np.float32) + W_KM0[pre_s[is_km] - kc0, post_s[is_km] - m0] = np.abs(unsigned[is_km]).astype(np.float32) + M_KM = W_KM0 != 0 + n_km_neg = int((signed[is_km] < 0).sum()) + rest = ~is_km + row = post_s[rest] + col = pre_s[rest] + values_raw = signed[rest].astype(np.float32) + order = np.lexsort((col, row)) # coalesced: sorted by (row, col) + row, col, values_raw = row[order], col[order], values_raw[order] + indices = np.stack([row, col]).astype(np.int64) + nnz = int(len(values_raw)) + print(f"[build] KC->MBON pairs={int(M_KM.sum()):,} (non-cholinergic sign on {n_km_neg} pairs ignored, magnitudes kept); " + f"sparse nnz={nnz:,} ({time.perf_counter() - t0:.1f}s)") + + # ---- spectral radius and initial scale ---- + sr = spectral_radius(row, col, np.abs(values_raw), N) + rho = sr["rho"] + s0 = TARGET_RADIUS / rho + values = np.clip(s0 * values_raw, -W_CLIP, W_CLIP).astype(np.float32) + print(f"[build] rho(|W_raw|)={rho:.3f} (Collatz-Wielandt upper bound {sr['cw_upper']:.3f}, " + f"{sr['iterations']} iters, converged={sr['converged']}) s0={s0:.6g} ({time.perf_counter() - t0:.1f}s)") + + # ---- artifact ---- + pop_ranges_json = json.dumps(pops.ranges) + chash = content_hash(indices, values_raw, W_KM0, pops.root_ids, pop_ranges_json, mode) + fname = f"fafb783_{chash[:12]}.npz" + path = out_dir / fname + build_cfg = { + "nt_sign_mode": mode, "cell_types_source": pops.source, + "annotations": str(Path(annotations) if annotations else P.ANNOTATIONS_TSV), + "w_clip": W_CLIP, "target_radius": TARGET_RADIUS, "power_iter_max": POWER_ITER_MAX, "power_iter_tol": POWER_ITER_TOL, + "expected_edges": EXPECTED_EDGES, "torch": torch.__version__, "numpy": np.__version__, + "built_at": started_at.isoformat(), "content_sha256": chash, "file": fname, + "population_order": pops.order, + } + counts = pops.counts() + metrics = { + "N": N, "rows": n_rows, "distinct_pairs": n_pairs, "nnz_flybrain_mode": nnz_fb, "nnz_mode": nnz_mode, + "edges_expected": EXPECTED_EDGES, "edges_match": edges_match, "nnz_sparse": nnz, + "skipped_rows": n_skipped, "unknown_nt_rows": unknown_nt, + "kc_mbon_pairs": int(M_KM.sum()), "kc_mbon_max_syn": float(W_KM0.max()), "kc_mbon_mean_syn": float(W_KM0[M_KM].mean()), + "spectral_radius_raw": rho, "spectral_radius_bracket": [sr["cw_lower"], sr["cw_upper"]], + "power_iterations": sr["iterations"], "power_converged": sr["converged"], "s0": s0, + "n_glomeruli": len(pops.glomerulus_names), "orn_food_untyped": int((pops.glomerulus_of_orn < 0).sum()), + "populations": counts, "cell_types_source": pops.source, + "build_seconds": None, + } + if path.exists(): + print(f"[build] {path} already exists with identical content hash; keeping it") + else: + tmp = out_dir / f"{fname}.tmp.npz" # numpy appends .npz to names that lack it + np.savez_compressed( + tmp, indices=indices, values=values, values_raw=values_raw, N=np.int64(N), + pop_ranges=np.array(pop_ranges_json), pop_order=np.array(pops.order), + root_ids=pops.root_ids, nt=pops.nt.astype(str), flybrain_group=pops.flybrain_group, + flybrain_group_names=np.array(pops.flybrain_group_names), cell_type=pops.cell_type.astype(str), + W_KM0=W_KM0, M_KM=M_KM, + kc_idx=pops.index("KC"), mbon_app_idx=pops.index("MBON_APP"), mbon_av_idx=pops.index("MBON_AV"), + mbon_other_idx=pops.index("MBON_OTHER"), dan_pam_idx=pops.index("DAN_PAM"), dan_ppl1_idx=pops.index("DAN_PPL1"), + glomerulus_of_orn=pops.glomerulus_of_orn, glomerulus_names=np.array(pops.glomerulus_names, dtype=str), + spectral_radius_raw=np.float64(rho), s0=np.float64(s0), nt_sign_mode=np.array(mode), + w_clip=np.float32(W_CLIP), source_sha256=np.array(json.dumps(source_sha)), + build_config=np.array(json.dumps(build_cfg)), content_sha256=np.array(chash), + canonical_index=pops.canonical_index, + ) + os.replace(tmp, path) + file_sha = sha256_file(path) + CURRENT_FILE.parent.mkdir(parents=True, exist_ok=True) + if out_dir == CONNECTOME_DIR: + CURRENT_FILE.write_text(fname + "\n") + metrics["build_seconds"] = round(time.perf_counter() - t0, 1) + metrics["file_sha256"] = file_sha + + # ---- report ---- + print(f"[build] N={N:,} nnz={nnz:,} KC={counts['KC']} MBON_APP={counts['MBON_APP']} MBON_AV={counts['MBON_AV']} " + f"MBON_OTHER={counts['MBON_OTHER']} DAN_PAM={counts['DAN_PAM']} DAN_PPL1={counts['DAN_PPL1']} " + f"glomeruli={len(pops.glomerulus_names)}") + for name in pops.order: + print(f" {name:<13} {counts[name]:>7} kind={pops.kind.get(name, 'other')}") + print(f"[build] wrote {path} ({path.stat().st_size / 1e6:.1f} MB, sha256 {file_sha[:12]}); current.txt -> {fname}") + if counts["KC"] != P.EXPECTED_KC or (counts["MBON_APP"] + counts["MBON_AV"] + counts["MBON_OTHER"]) != P.EXPECTED_MBON: + raise RuntimeError("KC/MBON counts do not match FAFB v783 expectations") # build_populations already checks + if persist_db: + label = f"{pops.source}:{Path(build_cfg['annotations']).name if pops.source == 'annotations' else 'classification.csv.gz'}" + run_id = persist(pops, metrics, {"build": build_cfg, "env": config.summary()}, file_sha, started_at, label) + print(f"[build] runs row {run_id} (kind=connectome_build) and {len(pops.order)} populations rows written") + return path + + +if __name__ == "__main__": + main() diff --git a/fly_trader/brain/plastic.py b/fly_trader/brain/plastic.py new file mode 100644 index 0000000000000000000000000000000000000000..6914912c482f82d2ba2401c83bd52b2ee304f315 --- /dev/null +++ b/fly_trader/brain/plastic.py @@ -0,0 +1,275 @@ +"""Mushroom-body plasticity: after its bootstrap the fly keeps learning from the market, at the KC→MBON synapses only. + +Rule (dopamine-gated three-factor; Hige 2015, Cohn 2015, Bennett 2021; masked like the connectome, FlyModel-style +associative updates, Shen/Dasgupta/Navlakha 2023). A plastic change ``D`` [n_KC, n_MBON] of the KC→MBON weights acts +at the network's last step (``FlyNet.readout``). For a scored minute with KC code ``k`` (presynaptic activity), MBON +activity ``m`` and, 120 minutes later, its realized net return ``r``: + + δ = clip(clip(r, ±1) − ŷ, ±DELTA_CLIP) dopamine: the prediction error (ŷ recomputed with the current D) + g_j = c_j γ_j (1 − m_j²) / (σ_j · scale) how MBON j moves the prediction (∂ŷ/∂D_ij = k_i g_j, exact) + ΔD = (α/ν) · M ⊙ Σ_rows w δ k gᵀ w = 1 on minutes scored at/above the line, else 0 + +which is gradient descent on the weighted squared prediction error: reward (δ > 0) depresses KC→avoid-MBON synapses +(c < 0) and potentiates KC→approach ones; punishment the reverse. ``ν`` (the median squared norm of one row's update, +fixed at bootstrap) makes ``α`` the share of a row's error one update corrects. Changes decay toward the bootstrap weights +with half-life ``τ½`` in event time (dopamine-gated forgetting, Berry 2012), each update is capped at ``STEP_CAP`` of the +bootstrap weights' norm, and ``D`` stays within [−w0, 2·w0] (effective weights within [0, 3·w0]). ``β_pot`` < 1 damps +potentiation (depression-dominant; still a descent direction). ``D = 0`` is the frozen fly (the shadow). + +The label arrives two hours after the decision, far beyond any biological eligibility window (seconds: Handler 2019), +so each scored minute leaves a synaptic tag (``PendingTags``: its KC code, MBON pre-activation and descending output) +that is captured when the label resolves (synaptic tag-and-capture, Frey & Morris 1997). + +``PlasticBank`` holds ``C`` configurations side by side (the replay's grid; 1 live). +""" +from __future__ import annotations + +import bisect +import itertools +import math +from dataclasses import dataclass + +import numpy as np +import torch + +DELTA_CLIP = 0.5 +RET_CLIP = 1.0 +TOP_WEIGHT = 10.0 +STEP_CAP = 1e-3 # ‖ΔD‖_F per update ≤ STEP_CAP · ‖w0‖_F +DAY_S = 86400.0 + + +@dataclass +class Tags: + """A batch of captured tags: row keys, the parts cached at scoring and each configuration's weight [C, B].""" + keys: list + ts: np.ndarray + y_dn: torch.Tensor # [B] + u0: torch.Tensor # [B, n_MBON] + k: torch.Tensor # [B, n_KC] dense KC code + w: torch.Tensor # [C, B] + s: np.ndarray = None # [B] strategy (dopamine channel) of each tag; None = channel 0 + + def __post_init__(self): + if self.s is None: + self.s = np.zeros(len(self.keys), dtype=np.int64) + + def __len__(self) -> int: + return len(self.keys) + + def subset(self, keep: np.ndarray) -> "Tags": + keep = np.asarray(keep, dtype=bool); i = torch.as_tensor(np.flatnonzero(keep), device=self.y_dn.device) + return Tags([k for k, m in zip(self.keys, keep) if m], self.ts[keep], self.y_dn[i], self.u0[i], self.k[i], self.w[:, i], self.s[keep]) + + +class PendingTags: + """Scored minutes waiting for their labels, kept in due order (strategies hold for different times). The KC code is + stored sparsely (its active cells).""" + + def __init__(self, n_kc: int, k_active: int): + self.n_kc, self.k_active = n_kc, k_active + self.q: list = []; self.n = 0; self._seq = itertools.count() + + def __len__(self) -> int: + return self.n + + def push(self, keys: list, ts: np.ndarray, due: float, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor, w: torch.Tensor, s=None) -> None: + """Tags capturable from event time ``due`` on (all rows of one push share it); ``s``: their strategy channel(s).""" + if len(keys) == 0: + return + vals, idx = k.detach().topk(min(self.k_active, k.shape[1]), dim=1) + ch = np.broadcast_to(np.asarray(0 if s is None else s, dtype=np.int64), (len(keys),)).copy() + chunk = {"due": float(due), "keys": list(keys), "ts": np.asarray(ts, dtype=np.float64), "y_dn": y_dn.detach().float().cpu(), "s": ch, + "u0": u0.detach().float().cpu(), "idx": idx.to(torch.int32).cpu(), "vals": vals.float().cpu(), "w": w.detach().float().cpu()} + bisect.insort(self.q, (float(due), next(self._seq), chunk)) + self.n += len(keys) + + def pop_due(self, now: float, device=None) -> Tags | None: + out = [] + while self.q and self.q[0][0] <= now: + out.append(self.q.pop(0)[2]) + if not out: + return None + self.n -= sum(len(c["keys"]) for c in out) + return self._tags(out, device) + + def clear(self) -> None: + self.q.clear(); self.n = 0 + + def _tags(self, chunks: list[dict], device=None) -> Tags: + idx = torch.cat([c["idx"] for c in chunks]).long(); vals = torch.cat([c["vals"] for c in chunks]) + k = torch.zeros(len(idx), self.n_kc).scatter_(1, idx, vals) + dev = device or "cpu" + return Tags(keys=[key for c in chunks for key in c["keys"]], ts=np.concatenate([c["ts"] for c in chunks]), + y_dn=torch.cat([c["y_dn"] for c in chunks]).to(dev), u0=torch.cat([c["u0"] for c in chunks]).to(dev), k=k.to(dev), + w=torch.cat([c["w"] for c in chunks], dim=1).to(dev), s=np.concatenate([c["s"] for c in chunks])) + + +def assign_channels(comp: np.ndarray, km_mass: np.ndarray, c_sign: np.ndarray, n_strategies: int) -> tuple[np.ndarray, np.ndarray]: + """Each strategy's own dopamine channel: its mushroom-body compartments. ``comp`` [J]: each MBON's compartment (−1: no + DAN), ``km_mass`` [J]: KC→MBON synapse mass per MBON, ``c_sign`` [J]: readout sign (+1 approach, −1 avoid, 0 other). + Compartments are split by their MBONs' majority valence and, within each valence, dealt in descending synapse mass to + the strategy with the least mass so far (deterministic, balanced, every strategy gets both valences when there are + enough). Returns ``learn`` [S, J] (the strategy's own MBONs: its outcomes change only their KC synapses) and ``read`` + [S, J] (its own + the MBONs no compartment claims, which stay fixed). One strategy: everything, as before; no DAN + data: every MBON is its own compartment.""" + J = len(c_sign); S = int(n_strategies) + if S <= 1: + return np.ones((1, J), bool), np.ones((1, J), bool) + comp = np.asarray(comp, int) if comp is not None and len(comp) == J else np.full(J, -1) + if (comp < 0).all(): + comp = np.arange(J) + learn = np.zeros((S, J), bool) + ids = sorted(set(comp[comp >= 0].tolist())) + val = {k: int(np.sign(np.sum(np.sign(c_sign[comp == k])))) for k in ids} + mass = {k: float(np.asarray(km_mass)[comp == k].sum()) for k in ids} + for v in (1, -1, 0): + load = np.zeros(S) + for k in sorted([k for k in ids if val[k] == v], key=lambda k: (-mass[k], k)): + s_ = int(np.argmin(load)); learn[s_, comp == k] = True; load[s_] += mass[k] + free = ~learn.any(0) + return learn, learn | free[None, :] + + +def row_weights(scores: torch.Tensor, lines: torch.Tensor) -> torch.Tensor: + """[C, B]: 1 where a configuration scored the row at or above its own line, else 0 — it learns from the outcome of + its own decisions and nothing else. + + Until 2026-09-21 every candidate row was tagged, weighting the ones above the line 10x. Candidates are overwhelmingly + losers (4.58M of them averaged -5.70% over the 2026-07-27 replay while the rows actually traded averaged +6.30%), so + even at 10x the losing population outweighed the tradeable rows by more than 100:1 and dragged every prediction + toward the candidate mean. Learning then made the fly worse than the same network frozen (PF 1.012 vs 1.098). A row + below the line is not a decision, so it carries no weight.""" + return torch.where(scores >= lines[:, None], 1.0, 0.0) + + +class PlasticBank: + """Plastic changes ``D`` [C, n_KC, n_MBON] of a FlyNet's KC→MBON weights under ``C`` configurations (α, τ½ in days; + τ½ = inf: no decay). The net is read, never modified.""" + + def __init__(self, net, configs: list[tuple[float, float]], scale: float, nu: float = 1.0, beta_pot: float = 1.0, step_cap: float = STEP_CAP, + learn: np.ndarray | None = None, read: np.ndarray | None = None): + self.net, self.scale, self.nu, self.beta_pot, self.step_cap = net, float(scale), float(nu), float(beta_pot), float(step_cap) + dev = net.dev + with torch.no_grad(): + self.w0 = net.w_km().detach().clone() + self.mask = net.mask.float() + self.c = net.c.detach().clone(); self.gamma = net.gain[net.mb0:net.mb1].detach().clone() + mu, sd = net.mbon_stats(); self.mu, self.sigma = mu.detach().clone(), sd.detach().clone() + self.w0_norm = float(torch.linalg.norm(self.w0)) + self.configs = [(float(a), float(h)) for a, h in configs] + C = len(self.configs) + J = self.w0.shape[1] + self.learn = torch.as_tensor(np.ones((1, J), bool) if learn is None else learn, device=dev) # [S, J] per-strategy channels + self.read = torch.as_tensor(np.ones((1, J), bool) if read is None else read, device=dev) + self.alpha = torch.tensor([a for a, _ in self.configs], device=dev)[:, None].repeat(1, J) # [C, J]: per-channel rates + self.half_life_s = torch.tensor([h * DAY_S if math.isfinite(h) else float("inf") for _, h in self.configs], device=dev)[:, None].repeat(1, J) + self.D = torch.zeros(C, *self.w0.shape, device=dev) + self.t_last: float | None = None + self.dev = dev + + @property + def C(self) -> int: + return len(self.configs) + + # ---- prediction ---- + def _read(self, s, B: int) -> torch.Tensor: + """[B, J] the MBONs each row's strategy reads (all MBONs with one strategy).""" + if self.read.shape[0] == 1 or s is None: + return self.read[0].float()[None].expand(B, -1) + return self.read[torch.as_tensor(np.asarray(s), device=self.dev)].float() + + def predict(self, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor, D: torch.Tensor | None = None, s=None) -> tuple[torch.Tensor, torch.Tensor]: + """(scores [C, B] in net-return units, MBON activity [C, B, n_MBON]) under each configuration's ``D``; ``s``: each + row's strategy (its head output is ``y_dn``; it reads its own compartments and the unclaimed ones).""" + D = self.D if D is None else D + u = u0[None] + self.gamma * torch.einsum("bi,cij->cbj", k, D * self.mask) + m = torch.tanh(u) + return (y_dn[None] + (((m - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale, m + + def frozen(self, y_dn: torch.Tensor, u0: torch.Tensor, s=None) -> torch.Tensor: + """The shadow: the bootstrap fly's score (D = 0) [B].""" + return (y_dn + (((torch.tanh(u0) - self.mu) / self.sigma) * self._read(s, len(y_dn))) @ self.c) / self.scale + + def grad_factor(self, m: torch.Tensor, s=None) -> torch.Tensor: + """g [C, B, n_MBON] with ∂ŷ/∂D_ij = k_i g_j (zero on MBONs the row's strategy does not read).""" + return self.c * self.gamma * (1.0 - m * m) / (self.sigma * self.scale) * self._read(s, m.shape[1]) + + # ---- learning ---- + def decay_to(self, t: float) -> None: + """Decay toward the bootstrap weights up to event time ``t`` (seconds).""" + if self.t_last is not None and t > self.t_last: + f = torch.pow(2.0, -(t - self.t_last) / self.half_life_s) # [C, J]; inf half-life → 1 + self.D *= f[:, None, :] + self.t_last = t if self.t_last is None else max(self.t_last, t) + + def estimate_nu(self, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor) -> float: + """Median squared norm of one row's update direction at D = 0 (sets the scale ``α`` is measured in).""" + m = torch.tanh(u0) + g = self.c * self.gamma * (1.0 - m * m) / (self.sigma * self.scale) # [B, J] + sq = ((k * k) @ self.mask * g * g).sum(1) + v = float(sq.median()) if len(sq) else 0.0 + self.nu = v if v > 0 else 1.0 + return self.nu + + def update(self, tags: Tags, r: torch.Tensor, t: float) -> dict: + """Capture ``tags`` with realized net returns ``r`` [B] at event time ``t``; returns per-configuration statistics.""" + self.decay_to(t) + if len(tags) == 0: + return {"n": 0} + r = r.to(self.dev).float().clamp(-RET_CLIP, RET_CLIP) + yhat, m = self.predict(tags.y_dn, tags.u0, tags.k, s=tags.s) + delta = (r[None] - yhat).clamp(-DELTA_CLIP, DELTA_CLIP) # [C, B] + own = self.learn.float()[torch.as_tensor(tags.s, device=self.dev)] if self.learn.shape[0] > 1 else self.learn[0].float()[None] + g = self.grad_factor(m, tags.s) * own # [C, B, J]: only the row's own channel learns + dD = torch.einsum("bi,cbj->cij", tags.k, (tags.w * delta)[:, :, None] * g) * self.mask + if self.beta_pot != 1.0: + dD = torch.where(dD > 0, dD * self.beta_pot, dD) + dD = dD * (self.alpha / self.nu)[:, None, :] + norms = torch.linalg.norm(dD.flatten(1), dim=1) + cap = self.step_cap * self.w0_norm + capped = norms > cap + dD = dD * torch.where(capped, cap / norms.clamp(min=1e-30), torch.ones_like(norms))[:, None, None] + D = self.D + dD + self.D = torch.maximum(torch.minimum(D, 2.0 * self.w0), -self.w0) * self.mask + return {"n": len(tags), "mean_delta": delta.mean(1).tolist(), "mean_abs_delta": delta.abs().mean(1).tolist(), + "step": torch.linalg.norm(dD.flatten(1), dim=1).tolist(), "capped": capped.tolist(), "drift": self.drift().tolist()} + + def drift(self, s: int | None = None) -> torch.Tensor: + """‖D‖_F / ‖w0‖_F per configuration (``s``: over that strategy's own channel only).""" + if s is None: + return torch.linalg.norm(self.D.flatten(1), dim=1) / max(self.w0_norm, 1e-30) + cols = self.learn[s] + return torch.linalg.norm(self.D[:, :, cols].flatten(1), dim=1) / max(float(torch.linalg.norm(self.w0[:, cols])), 1e-30) + + # ---- state ---- + def state(self) -> dict: + m = self.mask.bool() + return {"D": self.D[:, m].cpu(), "t_last": self.t_last, "configs": self.configs, "nu": self.nu, "beta_pot": self.beta_pot, "step_cap": self.step_cap, + "alpha": self.alpha.cpu(), "half_life_s": self.half_life_s.cpu(), "learn": self.learn.cpu(), "read": self.read.cpu()} + + def load_state(self, s: dict) -> None: + if [tuple(c) for c in s["configs"]] != self.configs: + raise ValueError(f"plastic state for configurations {s['configs']} does not match {self.configs}") + D = torch.zeros_like(self.D); D[:, self.mask.bool()] = s["D"].to(self.dev).float() + self.D, self.t_last, self.nu = D, s["t_last"], float(s["nu"]) + if "alpha" in s: + self.alpha, self.half_life_s = s["alpha"].to(self.dev), s["half_life_s"].to(self.dev) + self.learn, self.read = s["learn"].to(self.dev), s["read"].to(self.dev) + self.beta_pot, self.step_cap = float(s.get("beta_pot", self.beta_pot)), float(s.get("step_cap", self.step_cap)) + + def select(self, i: int) -> "PlasticBank": + """A single-configuration bank holding configuration ``i``'s current ``D`` (the replay's winner, going live).""" + b = PlasticBank(self.net, [self.configs[i]], self.scale, self.nu, self.beta_pot, self.step_cap, self.learn.cpu().numpy(), self.read.cpu().numpy()) + b.D = self.D[i:i + 1].clone(); b.t_last = self.t_last + return b + + def select_per_strategy(self, best: list[int]) -> "PlasticBank": + """One bank with each strategy's own winning configuration: strategy s's channel columns (D, α, half-life) come from + configuration ``best[s]``. Exact, because a strategy's value depends only on its own channel.""" + b = self.select(best[0]) + for s_, i in enumerate(best): + cols = self.learn[s_] + b.D[0][:, cols] = self.D[i][:, cols]; b.alpha[0, cols] = self.alpha[i, cols]; b.half_life_s[0, cols] = self.half_life_s[i, cols] + b.configs = [("per-strategy", tuple(best))] + return b diff --git a/fly_trader/brain/populations.py b/fly_trader/brain/populations.py new file mode 100644 index 0000000000000000000000000000000000000000..73298361a9c7b764d0f297ad9554e645f30933e3 --- /dev/null +++ b/fly_trader/brain/populations.py @@ -0,0 +1,611 @@ +"""Population index sets for the FlyWire FAFB v783 connectome. + +The neuron table is flybrain's ``neurons.csv.gz`` (139,255 canonical root_ids, the same set and +order the connectome edges refer to) left-joined on ``root_id`` with the Schlegel et al. 2024 +annotation TSV (``Supplemental_file1_neuron_annotations.tsv``: cell_class / cell_sub_class / +cell_type / top_nt). Neurotransmitter = ``top_nt`` when annotated, else flybrain's ``nt_type`` +mapped to the same vocabulary. + +Populations are *contiguous index ranges*: neurons are sorted by (population_id, root_id) so a +population is a slice, never a gather, on the GPU. Membership is first-match in +``POPULATION_ORDER``; everything unmatched is ``OTHER``. + +Valence tables (config/mbon_valence.json): Aso et al. 2014 eLife 3:e04580 (aversion-driving MBONs +are glutamatergic, attraction-driving MBONs GABAergic/cholinergic) and 3:e04577 (PAM = reward, +PPL1 = punishment). KC sparse code: Dasgupta, Stevens, Navlakha 2017 Science. + +flybrain's 63 behavioural groups (``determine_group`` from snedea/flybrain +``scripts/build_connectome.py``) are recomputed verbatim so group rates stay comparable with the +JavaScript demo; the DRIVE_HUNGER group (46 pars-intercerebralis neurons) is our HUNGER population. + +Fallback (``config.CELL_TYPES_SOURCE == 'flybrain'`` or the TSV is absent): flybrain's +``classification.csv.gz`` supplies the classes; MBON valence is assigned by neurotransmitter +(glutamate -> avoid, acetylcholine/gaba -> approach) and DAN valence by synapse counts onto avoid +vs approach MBONs (reward if more onto avoid MBONs: reward DANs depress avoidance synapses); +glomeruli come from the Codex ``consolidated_cell_types.csv.gz`` if it types >= 30 ORN +glomeruli, else a seeded random partition into 50 pseudo-glomeruli. +""" +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import pandas as pd + +from .. import config + +RAW_DIR = config.BRAIN_DIR / "raw" +NEURONS_CSV = RAW_DIR / "neurons.csv.gz" +CLASSIFICATION_CSV = RAW_DIR / "classification.csv.gz" +ANNOTATIONS_TSV = RAW_DIR / "Supplemental_file1_neuron_annotations.tsv" +CONSOLIDATED_TYPES_CSV = RAW_DIR / "consolidated_cell_types.csv.gz" +MBON_VALENCE_JSON = config.REPO_ROOT / "config" / "mbon_valence.json" + +EXPECTED_N = 139_255 +EXPECTED_KC = 5_177 +EXPECTED_MBON = 96 + +# Local constants (not in config.py): pseudo-glomerulus fallback. +MIN_TYPED_GLOMERULI = 30 +PSEUDO_GLOMERULI = 50 + +# First match wins; the remainder is OTHER. +POPULATION_ORDER: list[str] = [ + "KC", "MBON_APP", "MBON_AV", "MBON_OTHER", + "DAN_PAM", "DAN_PPL1", "DAN_OTHER", + "ORN_DANGER", "ORN_FOOD", "ALPN", "ALLN", "LH", + "GRN_SWEET", "GRN_BITTER", "GRN_OTHER", + "MECH_JO", "MECH_BRISTLE", "MECH_OTHER", + "THERMO_WARM", "THERMO_COOL", "THERMO_OTHER", + "CX", "HUNGER", "DESCENDING", "VISUAL", "OTHER", +] + +# kind for the ``populations`` table: sensory | central | mb | motor | other +POPULATION_KIND: dict[str, str] = { + "KC": "mb", "MBON_APP": "mb", "MBON_AV": "mb", "MBON_OTHER": "mb", + "DAN_PAM": "mb", "DAN_PPL1": "mb", "DAN_OTHER": "mb", + "ORN_DANGER": "sensory", "ORN_FOOD": "sensory", "ALPN": "central", "ALLN": "central", "LH": "central", + "GRN_SWEET": "sensory", "GRN_BITTER": "sensory", "GRN_OTHER": "sensory", + "MECH_JO": "sensory", "MECH_BRISTLE": "sensory", "MECH_OTHER": "sensory", + "THERMO_WARM": "sensory", "THERMO_COOL": "sensory", "THERMO_OTHER": "sensory", + "CX": "central", "HUNGER": "central", "DESCENDING": "motor", "VISUAL": "sensory", "OTHER": "other", +} + +NT_FLYBRAIN_TO_NAME: dict[str, str] = { + "ACH": "acetylcholine", "GABA": "gaba", "GLUT": "glutamate", "DA": "dopamine", + "SER": "serotonin", "OA": "octopamine", "OCT": "octopamine", +} + +# --------------------------------------------------------------------------------------------- +# flybrain groups, ported verbatim from snedea/flybrain scripts/build_connectome.py +# --------------------------------------------------------------------------------------------- +FLYBRAIN_GROUPS: list[tuple[str, str]] = [ + ("VIS_R1R6", "sensory"), ("VIS_R7R8", "sensory"), ("VIS_ME", "sensory"), ("VIS_LO", "sensory"), + ("VIS_LC", "sensory"), ("VIS_LPTC", "sensory"), ("OLF_ORN_FOOD", "sensory"), ("OLF_ORN_DANGER", "sensory"), + ("OLF_LN", "sensory"), ("OLF_PN", "sensory"), ("MECH_BRISTLE", "sensory"), ("MECH_JO", "sensory"), + ("MECH_CHORD", "sensory"), ("ANTENNAL_MECH", "sensory"), ("THERMO_WARM", "sensory"), ("THERMO_COOL", "sensory"), + ("NOCI", "sensory"), + ("MB_KC", "central"), ("MB_APL", "central"), ("MB_MBON_APP", "central"), ("MB_MBON_AV", "central"), + ("MB_DAN_REW", "central"), ("MB_DAN_PUN", "central"), ("LH_APP", "central"), ("LH_AV", "central"), + ("CX_EPG", "central"), ("CX_PFN", "central"), ("CX_FC", "central"), ("CX_HDELTA", "central"), + ("SEZ_FEED", "central"), ("SEZ_GROOM", "central"), ("SEZ_WATER", "central"), ("GUS_GRN_SWEET", "central"), + ("GUS_GRN_BITTER", "central"), ("GUS_GRN_WATER", "central"), ("GNG_DESC", "central"), ("CLOCK_DN", "central"), + ("DRIVE_HUNGER", "drives"), ("DRIVE_FEAR", "drives"), ("DRIVE_FATIGUE", "drives"), ("DRIVE_CURIOSITY", "drives"), + ("DRIVE_GROOM", "drives"), + ("DN_WALK", "motor"), ("DN_FLIGHT", "motor"), ("DN_TURN", "motor"), ("DN_BACKUP", "motor"), + ("DN_STARTLE", "motor"), ("VNC_CPG", "motor"), ("MN_LEG_L1", "motor"), ("MN_LEG_R1", "motor"), + ("MN_LEG_L2", "motor"), ("MN_LEG_R2", "motor"), ("MN_LEG_L3", "motor"), ("MN_LEG_R3", "motor"), + ("MN_WING_L", "motor"), ("MN_WING_R", "motor"), ("MN_PROBOSCIS", "motor"), ("MN_HEAD", "motor"), + ("MN_ABDOMEN", "motor"), + ("GENERIC_SENSORY", "sensory"), ("GENERIC_CENTRAL", "central"), ("GENERIC_DRIVES", "drives"), + ("GENERIC_MOTOR", "motor"), +] +FLYBRAIN_GROUP_NAMES: list[str] = [name for name, _ in FLYBRAIN_GROUPS] +FLYBRAIN_GROUP_ID: dict[str, int] = {name: i for i, name in enumerate(FLYBRAIN_GROUP_NAMES)} +assert len(FLYBRAIN_GROUPS) == 63 + + +def determine_group(flow: str, super_class: str, cls: str, sub_class: str, region: str, side: str = "") -> str: + """Map classification fields to one of the 63 group names (59 named + 4 generic). + + Verbatim port of flybrain ``scripts/build_connectome.py::determine_group`` (inputs are the + lower-cased flow / super_class / class / sub_class / side columns of classification.csv). + """ + # --- Visual system --- + if "visual" in cls or "optic" in cls or "visual" in super_class or "optic" in super_class: + if "photo_receptor" in sub_class or "photo" in sub_class: + if "r7" in sub_class or "r8" in sub_class or "uv" in sub_class or "pale" in sub_class: + return "VIS_R7R8" + return "VIS_R1R6" + if "ocellar" in sub_class or "ocellar" in cls: + return "VIS_R1R6" + if "lptc" in sub_class or "tangential" in sub_class: + return "VIS_LPTC" + if "lc" in sub_class or "loom" in sub_class or "lobula_columnar" in sub_class: + return "VIS_LC" + if "lobula" in sub_class and "plate" in sub_class: + return "VIS_LPTC" + if "lobula" in sub_class or sub_class == "lo": + return "VIS_LO" + if "medulla" in sub_class or "tm" in sub_class or "mi" in sub_class: + return "VIS_ME" + return "VIS_ME" + + # --- Olfactory system --- + if "olfact" in cls: + if "pheromone" in sub_class or "avers" in sub_class or "danger" in sub_class: + return "OLF_ORN_DANGER" + return "OLF_ORN_FOOD" + if cls == "alpn": + return "OLF_PN" + if cls == "alln": + return "OLF_LN" + + # --- Gustatory system --- + if "gustat" in cls: + if "bitter" in sub_class: + return "GUS_GRN_BITTER" + if "water" in sub_class: + return "GUS_GRN_WATER" + return "GUS_GRN_SWEET" + + # --- Mechanosensory system --- + if "mechano" in cls: + if "wind" in sub_class or "gravity" in sub_class or "auditory" in sub_class: + return "MECH_JO" + if "groom" in sub_class: + return "MECH_BRISTLE" + if "bristle" in sub_class or "taste_peg" in sub_class: + return "MECH_BRISTLE" + if "chord" in sub_class or "propriocep" in sub_class: + return "MECH_CHORD" + if "antenna" in sub_class: + return "ANTENNAL_MECH" + return "MECH_BRISTLE" + + # --- Thermosensory --- + if "thermo" in cls: + if "cool" in sub_class or "cold" in sub_class: + return "THERMO_COOL" + return "THERMO_WARM" + + # --- Hygrosensory -> thermosensory --- + if "hygro" in cls: + if "dry" in sub_class: + return "THERMO_WARM" + return "THERMO_COOL" + + # --- Nociceptive --- + if "nocicep" in cls: + return "NOCI" + + # --- Unknown sensory -> generic sensory --- + if "unknown_sensory" in cls: + return "MECH_BRISTLE" + + # --- Mushroom body: Kenyon cells --- + if "kenyon" in cls or cls == "kc": + return "MB_KC" + + # --- Mushroom body output neurons (MBONs) --- + if cls == "mbon": + return "MB_MBON_APP" + + # --- Mushroom body input neurons (MBINs) --- + if cls == "mbin": + return "MB_DAN_REW" + + # --- Dopaminergic neurons (DANs) --- + if cls == "dan": + return "MB_DAN_REW" + + # --- Lateral horn --- + if cls == "lhln" or cls == "lhcent": + return "LH_APP" + + # --- Central complex --- + if cls == "cx": + if "ring" in sub_class: + return "CX_EPG" + if "tangential" in sub_class: + return "CX_HDELTA" + if "columnar" in sub_class: + return "CX_PFN" + return "CX_FC" + + # --- Antennal lobe interneurons --- + if cls == "alin": + return "OLF_LN" + if cls == "alon": + return "OLF_PN" + + # --- TuBu neurons (tubercle to bulb, visual -> CX pathway) --- + if cls == "tubu": + return "CX_EPG" + + # --- TPN (taste projection neurons) --- + if cls == "tpn": + if "water" in sub_class: + return "GUS_GRN_WATER" + return "GUS_GRN_SWEET" + + # --- MAL (medial accessory lobe) neurons --- + if cls == "mal": + return "CX_FC" + + # --- Ascending neurons --- + if "ascending" in super_class or cls == "an": + return "GNG_DESC" + + # --- Descending neurons --- + if "descend" in super_class or "descend" in cls: + if "dn1p" in sub_class: + return "DN_STARTLE" + if "dn3" in sub_class: + return "DN_WALK" + if "walk" in sub_class or "locomot" in sub_class: + return "DN_WALK" + if "flight" in sub_class: + return "DN_FLIGHT" + if "turn" in sub_class: + return "DN_TURN" + if "back" in sub_class: + return "DN_BACKUP" + if "startle" in sub_class or "escape" in sub_class or "giant" in sub_class: + return "DN_STARTLE" + return "GNG_DESC" + + # --- Motor neurons --- + if "motor" in cls or "motor" in super_class: + if "proboscis" in sub_class: + return "MN_PROBOSCIS" + if "neck" in sub_class or "head" in sub_class: + return "MN_HEAD" + if "abdom" in sub_class or "crop" in sub_class: + return "MN_ABDOMEN" + if "ingestion" in sub_class or "haustellum" in sub_class or "salivary" in sub_class: + return "SEZ_FEED" + if "eye" in sub_class or "antenna" in sub_class: + return "MN_HEAD" + return "VNC_CPG" + + # --- Clock neurons --- + if "clock" in cls or "circadian" in cls: + return "CLOCK_DN" + + # --- Endocrine / neurosecretory --- + if "endocrine" in super_class: + if "pars_lateralis" in cls: + return "DRIVE_FATIGUE" + return "DRIVE_HUNGER" + + # --- Fallback: use flow to pick a region-appropriate generic --- + if flow == "afferent": + return "GENERIC_SENSORY" + if flow == "efferent": + return "GENERIC_MOTOR" + return "GENERIC_CENTRAL" + + +# --------------------------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------------------------- +def load_neuron_table(annotations: str | Path | None = None, source: str | None = None) -> tuple[pd.DataFrame, str]: + """flybrain neurons (canonical order) + classification + annotations (left join). + + Returns (table, source) where source is 'annotations' or 'flybrain'. Columns: + root_id (int64), fb_nt, fb_flow, fb_super_class, fb_class, fb_sub_class, fb_side (lower-cased, + as flybrain sees them), flybrain_group (uint16), annotated (bool), super_class, cell_class, + cell_sub_class, cell_type, top_nt, nt. + """ + source = source or config.CELL_TYPES_SOURCE + neu = pd.read_csv(NEURONS_CSV, usecols=["root_id", "nt_type"], dtype={"root_id": np.int64, "nt_type": str}, + keep_default_na=False) + if len(neu) != EXPECTED_N: + print(f"[populations] WARNING neurons.csv has {len(neu)} rows, expected {EXPECTED_N}", file=sys.stderr) + cls = pd.read_csv(CLASSIFICATION_CSV, dtype=str, keep_default_na=False) + cls["root_id"] = cls["root_id"].astype(np.int64) + for c in ("flow", "super_class", "class", "sub_class", "side"): + cls[c] = cls[c].str.strip().str.lower() + cls = cls.rename(columns={"flow": "fb_flow", "super_class": "fb_super_class", "class": "fb_class", + "sub_class": "fb_sub_class", "side": "fb_side"}) + cls = cls[["root_id", "fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"]] + df = neu.rename(columns={"nt_type": "fb_nt"}).merge(cls, on="root_id", how="left") + for c in ("fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"): + df[c] = df[c].fillna("") + df["fb_nt"] = df["fb_nt"].str.strip().str.upper() + + # flybrain group labels (classification.csv rows; neurons without a row keep GENERIC_CENTRAL) + groups = np.full(len(df), FLYBRAIN_GROUP_ID["GENERIC_CENTRAL"], dtype=np.uint16) + keyed = df[["fb_flow", "fb_super_class", "fb_class", "fb_sub_class", "fb_side"]].drop_duplicates() + cache: dict[tuple, int] = {} + for row in keyed.itertuples(index=False): + cache[tuple(row)] = FLYBRAIN_GROUP_ID[determine_group(row[0], row[1], row[2], row[3], "central", row[4])] + keys = list(zip(df["fb_flow"], df["fb_super_class"], df["fb_class"], df["fb_sub_class"], df["fb_side"])) + has_row = df["root_id"].isin(cls["root_id"]).to_numpy() + groups[has_row] = np.array([cache[k] for k, h in zip(keys, has_row) if h], dtype=np.uint16) + df["flybrain_group"] = groups + + ann_path = Path(annotations) if annotations else ANNOTATIONS_TSV + use_ann = source == "annotations" and ann_path.exists() + if source == "annotations" and not ann_path.exists(): + print(f"[populations] WARNING annotations TSV missing at {ann_path}; falling back to flybrain " + f"classification", file=sys.stderr) + if use_ann: + ann = pd.read_csv(ann_path, sep="\t", dtype=str, keep_default_na=False, low_memory=False, + usecols=["root_id", "super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"]) + ann["root_id"] = ann["root_id"].astype(np.int64) + ann = ann.drop_duplicates("root_id") + ann["annotated"] = True + df = df.merge(ann, on="root_id", how="left") + df["annotated"] = df["annotated"].fillna(False).astype(bool) + for c in ("super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"): + df[c] = df[c].fillna("").str.strip() + src = "annotations" + else: + df["annotated"] = False + for c in ("super_class", "cell_class", "cell_sub_class", "cell_type", "top_nt"): + df[c] = "" + src = "flybrain" + fb_nt_named = df["fb_nt"].map(NT_FLYBRAIN_TO_NAME).fillna("") + df["nt"] = np.where(df["top_nt"] != "", df["top_nt"].str.lower(), fb_nt_named) + return df, src + + +def load_mbon_valence(path: str | Path | None = None) -> dict: + with open(path or MBON_VALENCE_JSON) as f: + return json.load(f) + + +def mbon_base_type(cell_type: str) -> str: + """Exact type only: '-like' annotations and combined labels ('MBON25,MBON34') stay unassignable + (no valence evidence for '-like' cells; literature review 2026-09-12).""" + return cell_type.strip() + + +# --------------------------------------------------------------------------------------------- +# Populations +# --------------------------------------------------------------------------------------------- +@dataclass +class Populations: + order: list[str] + ranges: dict[str, tuple[int, int]] + root_ids: np.ndarray # int64 [N], population order + nt: np.ndarray # str [N] + flybrain_group: np.ndarray # uint16 [N] + flybrain_group_names: list[str] + glomerulus_of_orn: np.ndarray # int32 [n_ORN_FOOD], -1 unknown + glomerulus_names: list[str] + canonical_index: np.ndarray # int64 [N]: sorted position i holds flybrain row canonical_index[i] + source: str = "annotations" + cell_type: np.ndarray = field(default_factory=lambda: np.zeros(0, dtype=object)) + kind: dict[str, str] = field(default_factory=lambda: dict(POPULATION_KIND)) + + @property + def N(self) -> int: + return int(len(self.root_ids)) + + def n(self, name: str) -> int: + a, b = self.ranges[name] + return b - a + + def index(self, name: str) -> np.ndarray: + a, b = self.ranges[name] + return np.arange(a, b, dtype=np.int64) + + def slice(self, name: str) -> slice: + a, b = self.ranges[name] + return slice(a, b) + + def population_id(self) -> np.ndarray: + pid = np.empty(self.N, dtype=np.int16) + for i, name in enumerate(self.order): + a, b = self.ranges[name] + pid[a:b] = i + return pid + + def counts(self) -> dict[str, int]: + return {name: self.n(name) for name in self.order} + + def inverse_index(self) -> np.ndarray: + """canonical flybrain row -> sorted position.""" + inv = np.empty(self.N, dtype=np.int64) + inv[self.canonical_index] = np.arange(self.N, dtype=np.int64) + return inv + + +def _assign_annotations(df: pd.DataFrame, valence: dict) -> np.ndarray: + cc = df["cell_class"].to_numpy(dtype=object) + sc = df["cell_sub_class"].to_numpy(dtype=object) + ct = df["cell_type"].to_numpy(dtype=object) + base = np.array([mbon_base_type(t) for t in ct], dtype=object) + sup = np.where(df["annotated"].to_numpy(), df["super_class"].to_numpy(dtype=object), + df["fb_super_class"].to_numpy(dtype=object)) + approach = set(valence["approach"]) + avoid = set(valence["avoid"]) + rew = valence.get("dan_reward_prefix", "PAM") + pun = valence.get("dan_punishment_prefix", "PPL1") + is_mbon = cc == "MBON" + is_dan = cc == "DAN" + is_olf = cc == "olfactory" + is_gus = cc == "gustatory" + is_mech = cc == "mechanosensory" + is_thermo = cc == "thermosensory" + is_hygro = cc == "hygrosensory" + starts = np.char.startswith + ct_s = ct.astype(str) + rules = { + "KC": cc == "Kenyon_Cell", + "MBON_APP": is_mbon & np.isin(base, list(approach)), + "MBON_AV": is_mbon & np.isin(base, list(avoid)), + "MBON_OTHER": is_mbon, + "DAN_PAM": is_dan & starts(ct_s, rew), + "DAN_PPL1": is_dan & starts(ct_s, pun), + "DAN_OTHER": is_dan, + "ORN_DANGER": is_olf & (sc == "pheromone"), + "ORN_FOOD": is_olf, + "ALPN": cc == "ALPN", + "ALLN": cc == "ALLN", + "LH": np.isin(cc, ["LHLN", "LHCENT"]), + "GRN_SWEET": is_gus & (sc == "sugar/water"), + "GRN_BITTER": is_gus & (sc == "bitter"), + "GRN_OTHER": is_gus, + "MECH_JO": is_mech & np.isin(sc, ["wind_gravity", "auditory"]), + "MECH_BRISTLE": is_mech & np.isin(sc, ["eye bristle", "head bristle"]), + "MECH_OTHER": is_mech, + "THERMO_WARM": (is_thermo & (sc == "heating")) | (is_hygro & (sc == "dry")), + "THERMO_COOL": (is_thermo & (sc == "cold")) | (is_hygro & np.isin(sc, ["cooling", "moist", "evaporative_cooling"])), + "THERMO_OTHER": is_thermo | is_hygro, + "CX": cc == "CX", + "HUNGER": df["flybrain_group"].to_numpy() == FLYBRAIN_GROUP_ID["DRIVE_HUNGER"], + "DESCENDING": sup == "descending", + "VISUAL": np.isin(sup, ["optic", "visual_projection", "visual_centrifugal"]) | (cc == "visual"), + } + return _first_match(rules, len(df)) + + +def _contains(arr: np.ndarray, needle: str) -> np.ndarray: + return np.array([needle in s for s in arr], dtype=bool) + + +def _assign_flybrain(df: pd.DataFrame, dan_valence: np.ndarray | None) -> np.ndarray: + """Fallback partition from flybrain classification.csv (lower-cased fields).""" + cls = df["fb_class"].to_numpy(dtype=object) + sub = df["fb_sub_class"].to_numpy(dtype=object) + sup = df["fb_super_class"].to_numpy(dtype=object) + nt = df["nt"].to_numpy(dtype=object) + is_kc = _contains(cls, "kenyon") | (cls == "kc") + is_mbon = cls == "mbon" + is_dan = cls == "dan" + is_olf = _contains(cls, "olfact") + is_gus = _contains(cls, "gustat") + is_mech = _contains(cls, "mechano") + is_thermo = _contains(cls, "thermo") + is_hygro = _contains(cls, "hygro") + if dan_valence is None: + dan_valence = np.zeros(len(df), dtype=np.int8) + rules = { + "KC": is_kc, + "MBON_APP": is_mbon & np.isin(nt, ["acetylcholine", "gaba"]), + "MBON_AV": is_mbon & (nt == "glutamate"), + "MBON_OTHER": is_mbon, + "DAN_PAM": is_dan & (dan_valence > 0), + "DAN_PPL1": is_dan & (dan_valence < 0), + "DAN_OTHER": is_dan, + "ORN_DANGER": is_olf & (_contains(sub, "pheromone") | _contains(sub, "avers") | _contains(sub, "danger")), + "ORN_FOOD": is_olf, + "ALPN": cls == "alpn", + "ALLN": cls == "alln", + "LH": (cls == "lhln") | (cls == "lhcent"), + "GRN_SWEET": is_gus & (_contains(sub, "sugar") | _contains(sub, "water")), + "GRN_BITTER": is_gus & _contains(sub, "bitter"), + "GRN_OTHER": is_gus, + "MECH_JO": is_mech & (_contains(sub, "wind") | _contains(sub, "gravity") | _contains(sub, "auditory")), + "MECH_BRISTLE": is_mech & _contains(sub, "bristle"), + "MECH_OTHER": is_mech, + "THERMO_WARM": (is_thermo & (_contains(sub, "heat") | _contains(sub, "warm"))) | (is_hygro & _contains(sub, "dry")), + "THERMO_COOL": (is_thermo & (_contains(sub, "cold") | _contains(sub, "cool"))) + | (is_hygro & (_contains(sub, "cool") | _contains(sub, "moist") | _contains(sub, "evap"))), + "THERMO_OTHER": is_thermo | is_hygro, + "CX": cls == "cx", + "HUNGER": df["flybrain_group"].to_numpy() == FLYBRAIN_GROUP_ID["DRIVE_HUNGER"], + "DESCENDING": _contains(sup, "descend"), + "VISUAL": np.isin(sup, ["optic", "visual_projection", "visual_centrifugal"]) | (cls == "visual"), + } + return _first_match(rules, len(df)) + + +def _first_match(rules: dict[str, np.ndarray], n: int) -> np.ndarray: + pid = np.full(n, -1, dtype=np.int16) + for name in POPULATION_ORDER: + if name == "OTHER": + continue + m = rules[name] & (pid < 0) + pid[m] = POPULATION_ORDER.index(name) + pid[pid < 0] = POPULATION_ORDER.index("OTHER") + return pid + + +def fallback_dan_valence(df: pd.DataFrame, pid_partial: np.ndarray, edges: tuple[np.ndarray, np.ndarray, np.ndarray]) -> np.ndarray: + """+1 reward / -1 punishment / 0 unknown per canonical neuron, from synapse counts onto avoid vs + approach MBONs (reward DANs innervate the compartments whose MBONs drive avoidance).""" + pre, post, syn = edges + app = POPULATION_ORDER.index("MBON_APP") + av = POPULATION_ORDER.index("MBON_AV") + n = len(df) + onto_app = np.bincount(pre, weights=syn * (pid_partial[post] == app), minlength=n) + onto_av = np.bincount(pre, weights=syn * (pid_partial[post] == av), minlength=n) + val = np.zeros(n, dtype=np.int8) + val[onto_av > onto_app] = 1 + val[(onto_app > 0) & (onto_app >= onto_av)] = -1 + return val + + +def _glomeruli_annotations(cell_types: np.ndarray) -> tuple[np.ndarray, list[str]]: + names: list[str] = sorted({t[len("ORN_"):] for t in cell_types if t.startswith("ORN_") and len(t) > 4}) + lut = {n: i for i, n in enumerate(names)} + gid = np.array([lut.get(t[len("ORN_"):], -1) if t.startswith("ORN_") else -1 for t in cell_types], dtype=np.int32) + return gid, names + + +def _glomeruli_fallback(root_ids: np.ndarray, seed: int) -> tuple[np.ndarray, list[str]]: + if CONSOLIDATED_TYPES_CSV.exists(): + ct = pd.read_csv(CONSOLIDATED_TYPES_CSV, dtype=str, keep_default_na=False, usecols=["root_id", "primary_type"]) + ct["root_id"] = ct["root_id"].astype(np.int64) + lut = dict(zip(ct["root_id"].to_numpy(), ct["primary_type"].to_numpy())) + types = np.array([lut.get(r, "") for r in root_ids], dtype=object) + gid, names = _glomeruli_annotations(types) + if len(names) >= MIN_TYPED_GLOMERULI: + return gid, names + rng = np.random.default_rng(seed) + gid = rng.integers(0, PSEUDO_GLOMERULI, size=len(root_ids)).astype(np.int32) + return gid, [f"pseudo_{i:02d}" for i in range(PSEUDO_GLOMERULI)] + + +def build_populations(annotations: str | Path | None = None, source: str | None = None, + edges: tuple[np.ndarray, np.ndarray, np.ndarray] | None = None, + valence_path: str | Path | None = None, seed: int = 0, verbose: bool = True) -> Populations: + """Partition the neuron table into contiguous populations. + + ``edges`` = (pre_canonical, post_canonical, syn_count) arrays; only needed for the flybrain + fallback (DAN valence by synapse counts). + """ + df, src = load_neuron_table(annotations, source) + valence = load_mbon_valence(valence_path) + if src == "annotations": + pid = _assign_annotations(df, valence) + else: + pid = _assign_flybrain(df, None) + if edges is not None: + pid = _assign_flybrain(df, fallback_dan_valence(df, pid, edges)) + elif verbose: + print("[populations] WARNING flybrain fallback without edges: DANs are all DAN_OTHER", file=sys.stderr) + root_ids = df["root_id"].to_numpy(dtype=np.int64) + perm = np.lexsort((root_ids, pid)) # by population, then root_id + pid_sorted = pid[perm] + ranges: dict[str, tuple[int, int]] = {} + for i, name in enumerate(POPULATION_ORDER): + a = int(np.searchsorted(pid_sorted, i, side="left")) + b = int(np.searchsorted(pid_sorted, i, side="right")) + ranges[name] = (a, b) + cell_type = df["cell_type"].to_numpy(dtype=object)[perm] + a, b = ranges["ORN_FOOD"] + if src == "annotations": + gid, gnames = _glomeruli_annotations(cell_type[a:b]) + else: + gid, gnames = _glomeruli_fallback(root_ids[perm][a:b], seed) + pops = Populations( + order=list(POPULATION_ORDER), ranges=ranges, root_ids=root_ids[perm], + nt=df["nt"].to_numpy(dtype=object)[perm].astype(str), flybrain_group=df["flybrain_group"].to_numpy()[perm], + flybrain_group_names=list(FLYBRAIN_GROUP_NAMES), glomerulus_of_orn=gid, glomerulus_names=gnames, + canonical_index=perm.astype(np.int64), source=src, cell_type=cell_type, + ) + n_kc = pops.n("KC") + n_mbon = pops.n("MBON_APP") + pops.n("MBON_AV") + pops.n("MBON_OTHER") + if n_kc != EXPECTED_KC or n_mbon != EXPECTED_MBON: + raise RuntimeError(f"population sanity check failed: KC={n_kc} (expected {EXPECTED_KC}), " + f"MBON={n_mbon} (expected {EXPECTED_MBON}); source={src}") + return pops diff --git a/fly_trader/chain/__init__.py b/fly_trader/chain/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/chain/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/chain/balances.py b/fly_trader/chain/balances.py new file mode 100644 index 0000000000000000000000000000000000000000..52caf8f1aa70dcfa14be3dcf54bcf585d721ae96 --- /dev/null +++ b/fly_trader/chain/balances.py @@ -0,0 +1,49 @@ +"""Wallet balance snapshots and deltas (pattern: flywheel utils/balance_tracker.py:41-80). + +A fill is verified ONLY by comparing a snapshot taken before the order with one taken after +confirmation (plan §9). SOL is tracked as lamports of the wallet itself; SPL balances are raw +integer amounts summed per mint across all accounts under both token programs (Jupiter wraps and +unwraps WSOL inside the transaction, so a SOL leg shows up as a lamport delta, not a WSOL delta). +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime + +from .. import config + + +@dataclass +class Snapshot: + lamports: int + tokens: dict[str, int] + ts: datetime + decimals: dict[str, int] = field(default_factory=dict) + + def token(self, mint: str) -> int: + return int(self.tokens.get(mint, 0)) + + def to_json(self) -> dict: + return {"lamports": int(self.lamports), "tokens": {m: int(a) for m, a in self.tokens.items()}, + "decimals": {m: int(d) for m, d in self.decimals.items()}, "ts": self.ts.isoformat()} + + +def snapshot_balances(rpc, pubkey) -> Snapshot: + lamports = int(rpc.get_balance(str(pubkey))) + tokens: dict[str, int] = {} + decimals: dict[str, int] = {} + for acct in rpc.get_token_accounts_by_owner(str(pubkey)): + mint = acct["mint"] + tokens[mint] = tokens.get(mint, 0) + int(acct["amount"]) + if acct.get("decimals") is not None: + decimals[mint] = int(acct["decimals"]) + return Snapshot(lamports=lamports, tokens=tokens, ts=config.utcnow(), decimals=decimals) + + +def compute_delta(pre: Snapshot, post: Snapshot) -> dict: + """``{"lamports_delta": int, "token_deltas": {mint: post - pre}}`` over every mint seen in either snapshot.""" + mints = set(pre.tokens) | set(post.tokens) + return { + "lamports_delta": int(post.lamports) - int(pre.lamports), + "token_deltas": {m: int(post.tokens.get(m, 0)) - int(pre.tokens.get(m, 0)) for m in sorted(mints)}, + } diff --git a/fly_trader/chain/cluster_guard.py b/fly_trader/chain/cluster_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..5b350a09196daa175ad584d96cd55c218a57fef3 --- /dev/null +++ b/fly_trader/chain/cluster_guard.py @@ -0,0 +1,35 @@ +"""Default-deny signing gate (plan §9; pattern from better_bot code/solana_client.py:28-52). + +Every path that signs a transaction calls ``assert_signing_allowed()`` first. Reads (balances, +quotes, signature statuses) never do. Signing is allowed only when ALL of these hold: +``SOLANA_CLUSTER == 'mainnet-beta'`` (the only cluster where the graduated-memecoin universe +exists), ``LIVE_ENABLED=1``, and ``config.live_prerequisites_missing()`` is empty (capital, position +cap, gas reserve, bot key and API keys all set). ``config`` attributes are read at call time so a +test can monkeypatch them. +""" +from __future__ import annotations + +from .. import config + + +class SigningRefused(RuntimeError): + """Raised instead of signing when the live gate is not fully open.""" + + +def assert_signing_allowed() -> None: + cluster = config.SOLANA_CLUSTER + if cluster != "mainnet-beta": + raise SigningRefused(f"refusing to sign on cluster {cluster!r}: only mainnet-beta is allowed") + if not config.LIVE_ENABLED: + raise SigningRefused("refusing to sign: LIVE_ENABLED is not set (default-deny)") + missing = config.live_prerequisites_missing() + if missing: + raise SigningRefused("refusing to sign: live prerequisites missing: " + ", ".join(missing)) + + +def signing_allowed() -> bool: + try: + assert_signing_allowed() + return True + except SigningRefused: + return False diff --git a/fly_trader/chain/jupiter_swap.py b/fly_trader/chain/jupiter_swap.py new file mode 100644 index 0000000000000000000000000000000000000000..b030cbe7c33123fe50918fe3e1cc1f7be9c771c2 --- /dev/null +++ b/fly_trader/chain/jupiter_swap.py @@ -0,0 +1,221 @@ +"""Jupiter Swap API v2 client: ``GET /order``, ``POST /execute``, ``GET /program-id-to-label``. + +Base ``config.JUPITER_SWAP_BASE`` (https://api.jup.ag/swap/v2), header ``x-api-key``. Token-bucket +rate limits: one bucket (``config.JUPITER_RPS``) for ``/order`` and a separate one for ``/execute`` +(Jupiter meters ``/execute`` separately). Every call is persisted to ``api_calls`` +(service='jupiter'); the key is never stored (the request jsonb holds query params / a summary). + +``GET /order`` verified 2026-09-12 against mainnet (0.01 SOL -> USDC): + quote-only (no taker) -> HTTP 200 with fields: swapType ("aggregator"), inAmount, outAmount, + otherAmountThreshold, swapMode ("ExactIn"), slippageBps (0 when not requested), priceImpactPct + (string), routePlan [{percent, bps, usdValue, swapInfo {ammKey, label, inputMint, outputMint, + inAmount, outAmount}}], feeMint, feeBps (2 for SOL/USDC), platformFee {feeBps, feeMint}, + signatureFeeLamports (0), signatureFeePayer (null), prioritizationFeeLamports (0), + prioritizationFeePayer (null), rentFeeLamports, rentFeePayer, transaction (null), gasless, + jitOptimized, taker (null), inputMint, outputMint, router ("metis"), guaranteedPrice, requestId, + inUsdValue, outUsdValue, swapUsdValue, priceImpact (number, percentage points), mode ("ultra" and + "manual" both observed), totalTime. No lastValidBlockHeight / expireAt / errorCode in the + quote-only response. feeBps was 2 for SOL/USDC (docs: 10 bps most pairs, 50 bps for tokens < 24 h). + with taker -> adds transaction (base64 v0, taker = fee payer, 1 required signature, no + pre-filled signatures), lastValidBlockHeight (STRING, e.g. "424496168"), signatureFeePayer == + taker, signatureFeeLamports 5000, prioritizationFeeLamports/Payer, rentFeeLamports/Payer, + mode "manual". Documented error fields when the order cannot be built: errorCode + (aggregator: 1 insufficient funds, 2 insufficient SOL for gas, 3 below gasless minimum; + jupiterz: 1 insufficient balance, 2 missing ATA, 3 quote could not be built) + errorMessage. + Rate-limit headers: x-ratelimit-remaining / x-ratelimit-current / x-ratelimit-reset. + +``POST /execute`` (per the v2 reference; never exercised here): body {signedTransaction, requestId, + lastValidBlockHeight?(string)} -> {status "Success"|"Failed", signature, slot (string), code + (0 ok; -1 missing cached order, -2 invalid signed tx, -3 invalid message bytes; -1000 failed to + land, -1001 unknown, -1002 invalid tx, -1003 not fully signed, -1004 invalid block height; + -2000 RFQ failed to land, -2001 unknown, -2002 invalid payload, -2003 quote expired, -2004 swap + rejected), error, totalInputAmount, totalOutputAmount, inputAmountResult, outputAmountResult, + swapEvents [{inputMint, inputAmount, outputMint, outputAmount}]}. The same signedTransaction + + requestId is idempotent for ~2 minutes. + +``GET /program-id-to-label`` verified keyless at https://api.jup.ag/swap/v2/program-id-to-label +(107 entries; /swap/v1 and lite-api.jup.ag serve the same map). Cached to +``data/jupiter_program_labels.json``. +""" +from __future__ import annotations + +import json +import logging +import os +import time +from pathlib import Path + +import httpx + +from .. import config +from ..db.apilog import record_api_call +from ..logging_setup import scrub +from .jupiter_tokens import TokenBucket + +log = logging.getLogger(__name__) + +JUPITER_EXECUTE_RPS = config.JUPITER_RPS # separate bucket, same rate +JUPITER_LABELS_URL = "https://api.jup.ag/swap/v2/program-id-to-label" +JUPITER_LABELS_CACHE = config.DATA_DIR / "jupiter_program_labels.json" +JUPITER_LABELS_MAX_AGE_S = 86400.0 +DEFAULT_EXCLUDE_ROUTERS = "jupiterz" + + +class JupiterError(RuntimeError): + def __init__(self, endpoint: str, status: int | None, detail: str = ""): + self.endpoint = endpoint + self.status = status + self.detail = detail + super().__init__(f"jupiter {endpoint}: HTTP {status}: {detail}") + + +def _read_labels_cache(path: Path) -> dict | None: + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict) and isinstance(data.get("labels"), dict): + return data + except (OSError, ValueError): + pass + return None + + +def fetch_program_labels(cache_path: Path | None = None, refresh: bool = False, + max_age_s: float = JUPITER_LABELS_MAX_AGE_S) -> dict[str, str]: + """Keyless ``GET /program-id-to-label`` with a JSON file cache; stale cache on network failure.""" + path = Path(cache_path) if cache_path else JUPITER_LABELS_CACHE + cached = _read_labels_cache(path) + if cached and not refresh and time.time() - float(cached.get("fetched_at", 0)) < max_age_s: + return cached["labels"] + t0 = time.monotonic() + status = None + err = None + nbytes = None + try: + r = httpx.get(JUPITER_LABELS_URL, timeout=20) + status = r.status_code + nbytes = len(r.content) + body = r.json() if status < 400 else None + if status >= 400 or not isinstance(body, dict): + raise JupiterError("/program-id-to-label", status, scrub(r.text[:200])) + labels = {str(k): str(v) for k, v in body.items()} + except (JupiterError, httpx.HTTPError, ValueError) as e: + err = f"{type(e).__name__}: {scrub(str(e))}" + if cached: + log.warning("program-id-to-label fetch failed (%s); using stale cache", type(e).__name__) + return cached["labels"] + raise + finally: + record_api_call("jupiter", "swap/program-id-to-label", "GET", status, + int((time.monotonic() - t0) * 1000), err is None, err, response_bytes=nbytes) + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + with open(tmp, "w", encoding="utf-8") as f: + json.dump({"fetched_at": time.time(), "url": JUPITER_LABELS_URL, "labels": labels}, f, indent=0) + os.replace(tmp, path) + except OSError as e: + log.warning("could not write %s: %s", path, type(e).__name__) + return labels + + +class JupiterSwap: + def __init__(self, api_key: str | None = None, rps: float | None = None, timeout_s: float = 30.0): + self.api_key = api_key or config.JUPITER_API_KEY + if not self.api_key: + raise RuntimeError("JUPITER_API_KEY is not set") + self.base = config.JUPITER_SWAP_BASE + self.order_bucket = TokenBucket(rps or config.JUPITER_RPS) + self.execute_bucket = TokenBucket(rps or JUPITER_EXECUTE_RPS) + self.client = httpx.Client(timeout=timeout_s, headers={"x-api-key": self.api_key}) + + def _request(self, method: str, path: str, *, bucket: TokenBucket, params: dict | None = None, + json_body: dict | None = None, log_request: dict | None = None, accept_error_body, + retries: int = 3) -> tuple[int, object, int]: + """One rate-limited request. Returns ``(http_status, parsed_body, latency_ms)``. + Raises ``JupiterError`` on transport errors and on HTTP >= 400 unless ``accept_error_body(body)`` + says the JSON body is a structured Jupiter result the caller wants to see.""" + bucket.acquire() + t0 = time.monotonic() + status: int | None = None + err: str | None = None + nbytes: int | None = None + ok = False + try: + for attempt in range(retries + 1): + try: + r = self.client.request(method, self.base + path, params=params, json=json_body) + except httpx.HTTPError as e: + raise JupiterError(path, None, f"{type(e).__name__}: {scrub(str(e))}") from None + status = r.status_code + nbytes = len(r.content) + if status == 429 and attempt < retries: + time.sleep(1.0 + attempt) + bucket.acquire() + continue + break + try: + body = r.json() + except ValueError: + body = None + latency = int((time.monotonic() - t0) * 1000) + if status >= 400 and not accept_error_body(body): + snippet = json.dumps(body)[:300] if body is not None else r.text[:300] + raise JupiterError(path, status, scrub(snippet)) + if isinstance(body, dict): + if body.get("errorCode") is not None: + err = f"errorCode {body.get('errorCode')}: {body.get('errorMessage') or body.get('error') or ''}" + elif body.get("code") not in (None, 0): + err = f"code {body.get('code')}: {body.get('error') or body.get('status') or ''}" + ok = status < 400 and err is None + return status, body, latency + except JupiterError as e: + err = str(e) + raise + finally: + record_api_call("jupiter", f"swap{path}", method, status, int((time.monotonic() - t0) * 1000), + ok, scrub(err) if err else None, request=log_request, response_bytes=nbytes) + + def order(self, input_mint: str, output_mint: str, amount: int, taker: str | None = None, + slippage_bps: int | None = None, exclude_routers: str | None = DEFAULT_EXCLUDE_ROUTERS, + exclude_dexes: str | None = None) -> dict: + """``GET /order``. Without ``taker`` this is a quote (no transaction is built). Returns the + parsed JSON plus ``_latency_ms`` and ``_http_status``; a body carrying ``errorCode`` is returned + as-is (the caller decides), other HTTP errors raise ``JupiterError``.""" + params: dict = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(int(amount))} + if taker: + params["taker"] = str(taker) + if slippage_bps is not None: + params["slippageBps"] = int(slippage_bps) + if exclude_routers: + params["excludeRouters"] = exclude_routers + if exclude_dexes: + params["excludeDexes"] = exclude_dexes + status, body, latency = self._request( + "GET", "/order", bucket=self.order_bucket, params=params, log_request={"params": params}, + accept_error_body=lambda b: isinstance(b, dict) and "errorCode" in b) + if not isinstance(body, dict): + raise JupiterError("/order", status, "non-JSON response") + body["_latency_ms"] = latency + body["_http_status"] = status + return body + + def execute(self, signed_transaction_b64: str, request_id: str, last_valid_block_height: int | None = None) -> dict: + """``POST /execute``. Returns the parsed JSON plus ``_latency_ms``/``_http_status``; a body with + ``status``/``code`` is returned even on HTTP errors so the broker can see the code.""" + payload: dict = {"signedTransaction": signed_transaction_b64, "requestId": request_id} + if last_valid_block_height is not None: + payload["lastValidBlockHeight"] = str(int(last_valid_block_height)) + summary = {"requestId": request_id, "lastValidBlockHeight": payload.get("lastValidBlockHeight"), + "signedTransactionBytes": len(signed_transaction_b64)} + status, body, latency = self._request( + "POST", "/execute", bucket=self.execute_bucket, json_body=payload, log_request=summary, + accept_error_body=lambda b: isinstance(b, dict) and ("code" in b or "status" in b)) + if not isinstance(body, dict): + raise JupiterError("/execute", status, "non-JSON response") + body["_latency_ms"] = latency + body["_http_status"] = status + return body + + def program_id_to_label(self, refresh: bool = False) -> dict[str, str]: + return fetch_program_labels(refresh=refresh) diff --git a/fly_trader/chain/jupiter_tokens.py b/fly_trader/chain/jupiter_tokens.py new file mode 100644 index 0000000000000000000000000000000000000000..6262d0294e48ea0493bc01550a35a0d97e212d5c --- /dev/null +++ b/fly_trader/chain/jupiter_tokens.py @@ -0,0 +1,82 @@ +"""Jupiter Tokens API v2 client (https://api.jup.ag/tokens/v2). + +Endpoint (verified 2026-09-12 with the paid key): /search?query=. Tokens carry organicScore, +holderCount, liquidity, mcap, stats5m/1h/6h/24h and audit: mintAuthorityDisabled, +freezeAuthorityDisabled, topHoldersPercentage, devBalancePercentage, devMints, devMigrations (isSus is +absent unless flagged). Rate limit headers: x-ratelimit-remaining/current/reset. +""" +from __future__ import annotations + +import logging +import threading +import time + +import httpx + +from .. import config +from ..db.apilog import record_api_call + +log = logging.getLogger(__name__) + + +class TokenBucket: + def __init__(self, rate_per_s: float, burst: int | None = None): + self.rate = rate_per_s + self.capacity = burst or max(1, int(rate_per_s)) + self.tokens = float(self.capacity) + self.ts = time.monotonic() + self.lock = threading.Lock() + + def acquire(self) -> None: + while True: + with self.lock: + now = time.monotonic() + self.tokens = min(self.capacity, self.tokens + (now - self.ts) * self.rate) + self.ts = now + if self.tokens >= 1: + self.tokens -= 1 + return + wait = (1 - self.tokens) / self.rate + time.sleep(wait) + + +class JupiterTokens: + def __init__(self, api_key: str | None = None, rps: float | None = None): + self.api_key = api_key or config.JUPITER_API_KEY + if not self.api_key: + raise RuntimeError("JUPITER_API_KEY is not set") + self.base = config.JUPITER_TOKENS_BASE + self.bucket = TokenBucket(rps or config.JUPITER_RPS) + self.client = httpx.Client(timeout=30, headers={"x-api-key": self.api_key}) + + def _get(self, path: str, params: dict | None = None) -> list[dict]: + self.bucket.acquire() + t0 = time.monotonic() + status = None + err = None + try: + for attempt in range(4): + r = self.client.get(self.base + path, params=params) + status = r.status_code + if r.status_code == 429: + time.sleep(1.0 + attempt) + continue + r.raise_for_status() + data = r.json() + return data if isinstance(data, list) else [] + raise httpx.HTTPStatusError("rate limited", request=r.request, response=r) + except Exception as e: + err = f"{type(e).__name__}: {e}" + raise + finally: + record_api_call("jupiter", f"tokens{path}", "GET", status, int((time.monotonic() - t0) * 1000), + err is None, err, request={"params": params} if params else None) + + def search(self, query: str) -> list[dict]: + return self._get("/search", {"query": query}) + + def search_many(self, mints: list[str]) -> list[dict]: + out: list[dict] = [] + for i in range(0, len(mints), 100): + out.extend(self.search(",".join(mints[i:i + 100]))) + return out diff --git a/fly_trader/chain/keys.py b/fly_trader/chain/keys.py new file mode 100644 index 0000000000000000000000000000000000000000..204c92801a6c5e18a987018bfedc497536fec92b --- /dev/null +++ b/fly_trader/chain/keys.py @@ -0,0 +1,141 @@ +"""Bot wallet keys (plan §9; pattern lifted from VOC dexlp/keys.py). + +The secret lives ONLY in the gitignored ``.env`` as ``BOT_PRIVATE_KEY`` — base58 of the 64-byte +secret key (what ``create_wallet`` writes) or a JSON int array (what ``solana-keygen`` writes). It +is never logged, never printed and never placed in an exception message: errors name the source +and the failure type only. Note that ``str(Keypair)`` IS the base58 secret, so no Keypair object +is ever formatted, repr'd or passed to a logger in this module. +""" +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path + +import base58 +from psycopg.types.json import Jsonb +from solders.keypair import Keypair + +from .. import config +from ..db.connection import transaction + +log = logging.getLogger(__name__) + +ENV_NAME = "BOT_PRIVATE_KEY" +ENV_PATH = config.REPO_ROOT / ".env" +_ENV_LINE = re.compile(r"^\s*(?:export\s+)?BOT_PRIVATE_KEY\s*=") + + +class WalletKeyError(RuntimeError): + """Missing or malformed bot key. The message never includes secret material.""" + + +class WalletExists(RuntimeError): + """``create_wallet`` refused because a BOT_PRIVATE_KEY already exists.""" + + +def _keypair_from(raw: str | None) -> Keypair: + raw = (raw or "").strip() + if not raw: + raise WalletKeyError(f"{ENV_NAME} not set") + try: + if raw.startswith("["): + arr = json.loads(raw) + if not isinstance(arr, list): + raise ValueError("not a list") + data = bytes(int(x) & 0xFF for x in arr) + else: + data = base58.b58decode(raw) + except Exception as e: # never surface the value + raise WalletKeyError(f"could not decode {ENV_NAME}: {type(e).__name__}") from None + try: + if len(data) == 64: + return Keypair.from_bytes(data) + if len(data) == 32: # a bare seed (solana-keygen never writes this, but accept it) + return Keypair.from_seed(data) + except Exception as e: + raise WalletKeyError(f"invalid {ENV_NAME} bytes: {type(e).__name__}") from None + raise WalletKeyError(f"{ENV_NAME} must decode to 64 bytes (got {len(data)})") + + +def load_keypair(raw: str | None = None) -> Keypair: + """Keypair from ``BOT_PRIVATE_KEY`` (or ``raw``). Accepts base58 or a JSON int array.""" + return _keypair_from(raw if raw is not None else os.environ.get(ENV_NAME)) + + +def bot_pubkey(raw: str | None = None) -> str: + return str(load_keypair(raw).pubkey()) + + +def key_available() -> bool: + try: + load_keypair() + return True + except WalletKeyError: + return False + + +def env_file_has_key(path: Path | None = None) -> bool: + path = Path(path) if path else ENV_PATH + if not path.exists(): + return False + with open(path, encoding="utf-8") as f: + return any(_ENV_LINE.match(line) for line in f) + + +def create_wallet(env_path: Path | None = None, url: str | None = None) -> str: + """Generate the bot keypair once and append it to ``.env`` (mode 600). Returns the pubkey. + + Refuses if ``BOT_PRIVATE_KEY`` is already present in the process environment or in the file. + A preceding newline is written when the file does not end with one. Only the pubkey is + logged; a ``wallet_events(kind='created')`` row records the event. + """ + path = Path(env_path) if env_path else ENV_PATH + if os.environ.get(ENV_NAME, "").strip(): + raise WalletExists(f"{ENV_NAME} is already set in the environment; refusing to create another wallet") + if env_file_has_key(path): + raise WalletExists(f"{path} already contains {ENV_NAME}; refusing to overwrite") + kp = Keypair() + pubkey = str(kp.pubkey()) + secret_b58 = base58.b58encode(bytes(kp)).decode() + needs_newline = path.exists() and path.stat().st_size > 0 and not path.read_bytes().endswith(b"\n") + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + try: + with os.fdopen(fd, "a", encoding="utf-8") as f: + if needs_newline: + f.write("\n") + f.write(f"{ENV_NAME}={secret_b58}\n") + finally: + os.chmod(path, 0o600) + del secret_b58, kp + try: + with transaction(url) as conn: + conn.execute( + "INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('created', %s, %s)", + (pubkey, Jsonb({"env_path": str(path)})), + ) + except Exception as e: # the key is already safely on disk; do not lose it over a DB hiccup + log.warning("wallet_events insert failed after wallet creation: %s", type(e).__name__) + log.info("bot wallet created: %s", pubkey) + return pubkey + + +def show_wallet(rpc=None, url: str | None = None) -> str: + """Pubkey + SOL balance (read-only RPC); records ``wallet_events(kind='balance_snapshot')``.""" + pubkey = bot_pubkey() + if rpc is None: + from .rpc import HttpSolanaRpc + rpc = HttpSolanaRpc() + lamports = int(rpc.get_balance(pubkey)) + sol = lamports / config.LAMPORTS_PER_SOL + try: + with transaction(url) as conn: + conn.execute( + "INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('balance_snapshot', %s, %s)", + (pubkey, Jsonb({"lamports": lamports, "sol": sol})), + ) + except Exception as e: + log.warning("wallet_events insert failed: %s", type(e).__name__) + return f"pubkey={pubkey} sol={sol:.9f} lamports={lamports}" diff --git a/fly_trader/chain/rpc.py b/fly_trader/chain/rpc.py new file mode 100644 index 0000000000000000000000000000000000000000..82ebfcbe849efd2b06f0bcaa0608f7ce2461438f --- /dev/null +++ b/fly_trader/chain/rpc.py @@ -0,0 +1,200 @@ +"""Solana JSON-RPC over Helius using httpx + solders only (no solana-py). Pattern: VOC +dexlp/ingest/helius_swaps.py HttpSolanaRpc. + +The RPC URL carries the Helius API key: it is never logged, never stored in ``api_calls`` and never +placed in an exception message (httpx error text is scrubbed and re-raised as our own types). +Every call is persisted to ``api_calls`` (service='helius', endpoint=) through +``fly_trader.db.apilog.record_api_call``, which swallows DB errors so a database hiccup never +breaks an RPC call. + +Response shapes verified against mainnet on 2026-09-12: + getBalance -> {"context": {"slot", "apiVersion"}, "value": } + getTokenAccountsByOwner (jsonParsed) -> {"context", "value": [{"pubkey", "account": {"lamports", + "owner": , "data": {"program": "spl-token"|"spl-token-2022", + "parsed": {"type": "account", "info": {"mint", "owner", "state", "isNative", + "tokenAmount": {"amount": "", "decimals", "uiAmount", "uiAmountString"}, + "extensions": [...] (token-2022 only)}}}}}]} + getSignatureStatuses -> {"context", "value": [{"slot", "confirmations", "err", "status", + "confirmationStatus": processed|confirmed|finalized} | null]} + getLatestBlockhash -> {"context", "value": {"blockhash", "lastValidBlockHeight": }} + getBlockHeight -> (same units as lastValidBlockHeight) + getTransaction -> {"transaction", "meta": {"err", "status", "fee", "preBalances", + "postBalances", "preTokenBalances", "postTokenBalances", "innerInstructions", + "logMessages", "computeUnitsConsumed", ...}, "version", "slot", "blockTime"} + sendTransaction -> ""; malformed input returns error {"code": -32602, ...} + JSON-RPC batches are supported (array in, array out, matched by id). +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Any + +import httpx + +from .. import config +from ..db.apilog import record_api_call +from ..logging_setup import scrub + +log = logging.getLogger(__name__) + +TOKEN_PROGRAM_ID = "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA" +TOKEN_2022_PROGRAM_ID = "TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb" +TOKEN_PROGRAM_IDS = (TOKEN_PROGRAM_ID, TOKEN_2022_PROGRAM_ID) +RPC_TIMEOUT_S = 15.0 +COMMITMENT = "confirmed" + + +class RpcError(RuntimeError): + """A JSON-RPC level error ({"error": {code, message, data}}).""" + + def __init__(self, method: str, error: dict | Any): + err = error if isinstance(error, dict) else {"message": str(error)} + self.method = method + self.code = err.get("code") + self.message = str(err.get("message", "")) + self.data = err.get("data") + super().__init__(f"rpc {method} error {self.code}: {scrub(self.message)}") + + +class RpcTransportError(RuntimeError): + """HTTP/transport failure. Message never contains the URL.""" + + +def _request_summary(method: str, params: list | None) -> dict: + if method == "sendTransaction" and params: + opts = params[1] if len(params) > 1 and isinstance(params[1], dict) else {} + return {"tx_b64_len": len(params[0]) if isinstance(params[0], str) else None, **opts} + if method == "getSignatureStatuses" and params: + # full signatures are 88 base58 chars and the log scrubber would redact them as secrets + return {"n": len(params[0]), "signature_prefixes": [str(s)[:12] for s in params[0]]} + if method == "getTransaction" and params: + return {"signature_prefix": str(params[0])[:12]} + return {"params": params} + + +class HttpSolanaRpc: + def __init__(self, url: str | None = None, timeout_s: float = RPC_TIMEOUT_S): + self._url = url or config.helius_http_url() + self._client = httpx.Client(timeout=timeout_s) + self._id = 0 + self._lock = threading.Lock() + + def _next_id(self) -> int: + with self._lock: + self._id += 1 + return self._id + + def _post(self, payload: dict | list, endpoint: str, summary: dict | None): + t0 = time.monotonic() + status: int | None = None + err: str | None = None + nbytes: int | None = None + try: + try: + r = self._client.post(self._url, json=payload) + except httpx.HTTPError as e: + raise RpcTransportError(f"{endpoint}: {type(e).__name__}: {scrub(str(e))}") from None + status = r.status_code + nbytes = len(r.content) + if status >= 400: + raise RpcTransportError(f"{endpoint}: HTTP {status}: {scrub(r.text[:200])}") + try: + body = r.json() + except ValueError: + raise RpcTransportError(f"{endpoint}: non-JSON response ({nbytes} bytes)") from None + if isinstance(payload, dict): + if not isinstance(body, dict): + raise RpcTransportError(f"{endpoint}: unexpected response shape {type(body).__name__}") + if body.get("error"): + raise RpcError(endpoint, body["error"]) + return body.get("result") + if not isinstance(body, list): + raise RpcTransportError(f"{endpoint}: batch response is not a list") + by_id = {item.get("id"): item for item in body if isinstance(item, dict)} + results = [] + for req in payload: + item = by_id.get(req["id"]) + if item is None: + raise RpcTransportError(f"{endpoint}: missing batch response for id {req['id']}") + if item.get("error"): + raise RpcError(req["method"], item["error"]) + results.append(item.get("result")) + return results + except Exception as e: + err = str(e) + raise + finally: + record_api_call("helius", endpoint, "POST", status, int((time.monotonic() - t0) * 1000), + err is None, err, request=summary, response_bytes=nbytes) + + # ---- generic ---- + def call(self, method: str, params: list | None = None): + payload = {"jsonrpc": "2.0", "id": self._next_id(), "method": method, "params": params or []} + return self._post(payload, method, _request_summary(method, params)) + + def batch(self, calls: list[tuple[str, list | None]]) -> list: + """Send several calls in one HTTP request; results in call order. Raises on the first error.""" + if not calls: + return [] + payload = [{"jsonrpc": "2.0", "id": self._next_id(), "method": m, "params": p or []} for m, p in calls] + methods = [m for m, _ in calls] + return self._post(payload, "batch:" + ",".join(methods), {"methods": methods}) + + # ---- typed helpers ---- + def get_balance(self, pubkey) -> int: + res = self.call("getBalance", [str(pubkey), {"commitment": COMMITMENT}]) + return int(res["value"]) + + def get_token_accounts_by_owner(self, pubkey) -> list[dict]: + """All token accounts of ``pubkey`` under BOTH token programs, as + ``{address, mint, amount:int, decimals, program}``.""" + out: list[dict] = [] + for program in TOKEN_PROGRAM_IDS: + res = self.call("getTokenAccountsByOwner", + [str(pubkey), {"programId": program}, {"encoding": "jsonParsed", "commitment": COMMITMENT}]) + for item in (res or {}).get("value") or []: + try: + info = item["account"]["data"]["parsed"]["info"] + amt = info["tokenAmount"] + out.append({"address": item["pubkey"], "mint": info["mint"], "amount": int(amt["amount"]), + "decimals": int(amt["decimals"]), "program": program}) + except (KeyError, TypeError, ValueError): + log.warning("unparsed token account %s", item.get("pubkey") if isinstance(item, dict) else "?") + return out + + def get_signature_statuses(self, sigs: list[str], search_history: bool = False) -> list[dict | None]: + res = self.call("getSignatureStatuses", [list(sigs), {"searchTransactionHistory": bool(search_history)}]) + value = (res or {}).get("value") + if not isinstance(value, list): + return [None] * len(sigs) + return value + + def get_transaction(self, sig: str, encoding: str = "jsonParsed") -> dict | None: + return self.call("getTransaction", [sig, {"encoding": encoding, "maxSupportedTransactionVersion": 0, + "commitment": COMMITMENT}]) + + def get_latest_blockhash(self) -> dict: + """``{"blockhash": str, "last_valid_block_height": int}``.""" + v = self.call("getLatestBlockhash", [{"commitment": COMMITMENT}])["value"] + return {"blockhash": v["blockhash"], "last_valid_block_height": int(v["lastValidBlockHeight"])} + + def get_block_height(self) -> int: + return int(self.call("getBlockHeight", [{"commitment": COMMITMENT}])) + + def get_account_info(self, pubkey) -> dict | None: + res = self.call("getAccountInfo", [str(pubkey), {"encoding": "jsonParsed", "commitment": COMMITMENT}]) + return (res or {}).get("value") + + def get_mint_decimals(self, mint: str) -> int | None: + try: + info = self.get_account_info(mint) + return int(info["data"]["parsed"]["info"]["decimals"]) + except Exception: + return None + + def send_transaction(self, tx_b64: str, skip_preflight: bool = False, max_retries: int = 3) -> str: + sig = self.call("sendTransaction", [tx_b64, {"encoding": "base64", "skipPreflight": bool(skip_preflight), + "preflightCommitment": COMMITMENT, "maxRetries": int(max_retries)}]) + return str(sig) diff --git a/fly_trader/chain/signing.py b/fly_trader/chain/signing.py new file mode 100644 index 0000000000000000000000000000000000000000..a5e50355d421b7778908feb856467a5b066a37df --- /dev/null +++ b/fly_trader/chain/signing.py @@ -0,0 +1,86 @@ +"""Sign only our own signer slot of a Jupiter-built VersionedTransaction (plan §9). + +The multi-signer trap (flywheel utils/transaction_signing.py:99-205): ``VersionedTransaction(msg, +[keypair])`` demands keypairs for EVERY required signer and replaces all signatures, which pushed +that code into rewriting account keys and the header — Jupiter then rejects the transaction (and a +rewritten fee payer would make us pay for someone else's message). Here the message, header and fee +payer are never touched: we locate our pubkey among the static account keys, require its index to +be below ``num_required_signatures``, sign the versioned message bytes, and drop the signature into +that slot while keeping any signatures already present (RFQ makers pre-sign their slot). + +Verified 2026-09-12: a Swap v2 ``/order`` transaction with a taker is v0 with +``num_required_signatures=1``, the taker as static key 0 (fee payer), 4 address-table lookups and no +pre-filled signatures. +""" +from __future__ import annotations + +import base64 + +from solders.message import to_bytes_versioned +from solders.signature import Signature +from solders.transaction import VersionedTransaction + + +class NotASigner(ValueError): + """Our pubkey is absent from the static keys or sits outside the signer range.""" + + +def _as_tx(tx) -> VersionedTransaction: + if isinstance(tx, VersionedTransaction): + return tx + if isinstance(tx, str): + tx = base64.b64decode(tx) + return VersionedTransaction.from_bytes(bytes(tx)) + + +def signer_index(tx, pubkey) -> int: + """Index of ``pubkey`` among the static account keys, verified to be a required signer.""" + tx = _as_tx(tx) + msg = tx.message + keys = list(msg.account_keys) + if pubkey not in keys: + raise NotASigner(f"{pubkey} is not among the transaction's static account keys") + idx = keys.index(pubkey) + n_req = msg.header.num_required_signatures + if idx >= n_req: + raise NotASigner(f"{pubkey} is account {idx} but only the first {n_req} account(s) sign") + return idx + + +def sign_transaction_b64(tx_b64: str, keypair) -> tuple[str, int]: + """Return ``(base64 signed transaction, our signer index)``. Message/header/fee payer untouched.""" + tx = _as_tx(tx_b64) + msg = tx.message + idx = signer_index(tx, keypair.pubkey()) + n_req = msg.header.num_required_signatures + sigs = list(tx.signatures) + if len(sigs) < n_req: + sigs.extend([Signature.default()] * (n_req - len(sigs))) + sigs[idx] = keypair.sign_message(to_bytes_versioned(msg)) + signed = VersionedTransaction.populate(msg, sigs) + return base64.b64encode(bytes(signed)).decode(), idx + + +def transaction_id(tx) -> str: + """The transaction signature (first signature) of a signed transaction.""" + return str(_as_tx(tx).signatures[0]) + + +def static_program_ids(tx) -> list[str]: + """Program ids of the compiled instructions that resolve to STATIC account keys, unique, in order. + Program ids that live in address lookup tables are not resolved in v1 (see ``n_lookup_programs``).""" + msg = _as_tx(tx).message + keys = [str(k) for k in msg.account_keys] + out: list[str] = [] + for ci in msg.instructions: + i = ci.program_id_index + if i < len(keys) and keys[i] not in out: + out.append(keys[i]) + return out + + +def n_lookup_programs(tx) -> int: + """How many compiled instructions reference a program id from an address lookup table.""" + msg = _as_tx(tx).message + n_static = len(msg.account_keys) + return sum(1 for ci in msg.instructions if ci.program_id_index >= n_static) diff --git a/fly_trader/config.py b/fly_trader/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4d91fa499ad80c300c19cb619c3d4335a0f56ba6 --- /dev/null +++ b/fly_trader/config.py @@ -0,0 +1,196 @@ +"""Typed configuration from the environment (.env is loaded once, never overriding real env vars). + +Every knob lives here. Secret values are never printed; ``summary()`` returns the non-secret +configuration for persistence in ``runs.config``. All time handling is UTC. +""" +from __future__ import annotations + +import os +from datetime import datetime, timezone +from pathlib import Path + +from dotenv import load_dotenv + +REPO_ROOT = Path(__file__).resolve().parent.parent +load_dotenv(REPO_ROOT / ".env", override=False) + +SECRET_ENV_NAMES = ("HELIUS_API_KEY", "JUPITER_API_KEY", "BOT_PRIVATE_KEY") + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def env_str(name: str, default: str | None = None) -> str | None: + v = os.environ.get(name) + if v is None or v.strip() == "": + return default + return v.strip() + + +def env_int(name: str, default: int) -> int: + v = env_str(name) + return int(v) if v is not None else default + + +def env_float(name: str, default: float) -> float: + v = env_str(name) + return float(v) if v is not None else default + + +def env_bool(name: str, default: bool = False) -> bool: + v = env_str(name) + if v is None: + return default + return v.lower() in ("1", "true", "yes", "on") + + +def env_list(name: str, default: list[str]) -> list[str]: + v = env_str(name) + if v is None: + return list(default) + return [x.strip() for x in v.split(",") if x.strip()] + +# ---- secrets (names only ever appear in logs) ---- +HELIUS_API_KEY = env_str("HELIUS_API_KEY") +JUPITER_API_KEY = env_str("JUPITER_API_KEY") + +# ---- paths ---- +DATA_DIR = (REPO_ROOT / env_str("DATA_DIR", "data")).resolve() +LOG_DIR = (REPO_ROOT / env_str("LOG_DIR", "logs")).resolve() +BRAIN_DIR = DATA_DIR / "brain" +PG_ARCHIVE_DIR = DATA_DIR / "pg_archive" + +# ---- database ---- +DATABASE_URL = env_str("DATABASE_URL", "postgresql:///fly_trader") + +# ---- chain / execution (default-deny) ---- +SOLANA_CLUSTER = env_str("SOLANA_CLUSTER", "mainnet-beta") +LIVE_ENABLED = env_bool("LIVE_ENABLED", False) +CAPITAL_SOL = env_float("CAPITAL_SOL", 0.0) +MAX_POSITION_SOL = env_float("MAX_POSITION_SOL", 0.0) +GAS_RESERVE_SOL = env_float("GAS_RESERVE_SOL", 0.30) +# position sizing (agent/sizing.py): a fraction of the growth-optimal (Kelly) bet measured per score band in the backtest +KELLY_FRACTION = env_float("KELLY_FRACTION", 0.25) # quarter Kelly: the band estimates are noisy +MAX_POSITION_FRACTION = env_float("MAX_POSITION_FRACTION", 0.10) # of the deployable bankroll (wealth minus the gas reserve) +MAX_POOL_SHARE = env_float("MAX_POOL_SHARE", 0.02) # of the pool's quote reserve (bounds price impact) +# The training label charges what an exit really costs at the size the book takes. market/features.py prices +# exit_cost_0p1 for a fixed 0.1 SOL, but positions are Kelly-sized up to MAX_POSITION_FRACTION of the bankroll, and +# impact = value/(value + reserves) grows with size. 0 = the sizing rule's own ceiling (train/decisions.py). +LABEL_SIZE_SOL = env_float("LABEL_SIZE_SOL", 0.0) +MIN_POSITION_SOL = env_float("MIN_POSITION_SOL", 0.02) # smaller sizes are skipped +KILL_SWITCH_DRAWDOWN = env_float("KILL_SWITCH_DRAWDOWN", 0.30) +KILL_SWITCH_LIQUIDATE = env_bool("KILL_SWITCH_LIQUIDATE", False) # a tripped kill switch also sells every open live position (agent/fly_live.py) +CIRCUIT_THRESHOLD = env_int("CIRCUIT_THRESHOLD", 3) +EXECUTABILITY_MAX_IMPACT = env_float("EXECUTABILITY_MAX_IMPACT", 0.30) +SLIPPAGE_ENTRY_BPS = env_int("SLIPPAGE_ENTRY_BPS", 150) +SLIPPAGE_EXIT_BPS = env_int("SLIPPAGE_EXIT_BPS", 300) +SLIPPAGE_FORCED_BPS = env_int("SLIPPAGE_FORCED_BPS", 500) +SLIPPAGE_STEP_BPS = env_int("SLIPPAGE_STEP_BPS", 100) +MAX_SLIPPAGE_ENTRY_BPS = env_int("MAX_SLIPPAGE_ENTRY_BPS", 300) + +HELIUS_HTTP = "https://mainnet.helius-rpc.com/" +JUPITER_SWAP_BASE = "https://api.jup.ag/swap/v2" +JUPITER_TOKENS_BASE = "https://api.jup.ag/tokens/v2" +JUPITER_RPS = env_float("JUPITER_RPS", 8.0) + +WSOL_MINT = "So11111111111111111111111111111111111111112" +USDC_MINT = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v" +LAMPORTS_PER_SOL = 1_000_000_000 + +# ---- token stats (Jupiter) ---- +STATS_REFRESH_S = env_float("STATS_REFRESH_S", 600.0) + +# ---- connectome (the fly) ---- +DEVICE = env_str("DEVICE", "mps") +NT_SIGN_MODE = env_str("NT_SIGN_MODE", "shiu") # shiu | flybrain +CELL_TYPES_SOURCE = env_str("CELL_TYPES_SOURCE", "annotations") # annotations | flybrain + +# ---- costs ---- +LAMBDA_IMPACT_DLMM = env_float("LAMBDA_IMPACT_DLMM", 0.02) + +# ---- training lifecycle ---- +RESET_ON_START = env_bool("RESET_ON_START", True) # wipe the paper book and its stats when the runner starts + +# ---- the fly's handover (agent/fly_session.py): when its paper race earns it the live seat ---- +HANDOVER_DAYS = env_int("HANDOVER_DAYS", 14) # days of racing first; 0 = as soon as the trades are in +HANDOVER_MIN_TRADES = env_int("HANDOVER_MIN_TRADES", 30) # closed paper_fly trades required +HANDOVER_BEAT_SELECTOR = env_bool("HANDOVER_BEAT_SELECTOR", True) # and realized P&L at least the paper selector's over the window + + +def helius_http_url() -> str: + if not HELIUS_API_KEY: + raise RuntimeError("HELIUS_API_KEY is not set") + return f"{HELIUS_HTTP}?api-key={HELIUS_API_KEY}" + + +def summary() -> dict: + """Non-secret configuration snapshot for ``runs.config``.""" + out = {} + for k, v in globals().items(): + if k.isupper() and k not in SECRET_ENV_NAMES and not k.startswith("_"): + if isinstance(v, (str, int, float, bool, list, tuple)): + out[k] = v + elif isinstance(v, Path): + out[k] = str(v) + elif isinstance(v, dict): + out[k] = v + if "DATABASE_URL" in out: # may carry a password (URL or key=value form); the database name is enough + try: + from psycopg.conninfo import conninfo_to_dict + db = conninfo_to_dict(str(out["DATABASE_URL"])).get("dbname") or "?" + except Exception: + db = "?" + out["DATABASE_URL"] = f"postgresql:///{db}" + return out + + +def live_prerequisites_missing() -> list[str]: + missing = [] + if SOLANA_CLUSTER != "mainnet-beta": + missing.append("SOLANA_CLUSTER must be mainnet-beta for live memecoin trading") + if not LIVE_ENABLED: + missing.append("LIVE_ENABLED=1") + if CAPITAL_SOL <= 0: + missing.append("CAPITAL_SOL") + if MAX_POSITION_SOL <= 0: + missing.append("MAX_POSITION_SOL") + if GAS_RESERVE_SOL <= 0: + missing.append("GAS_RESERVE_SOL") + if not env_str("BOT_PRIVATE_KEY"): + missing.append("BOT_PRIVATE_KEY") + if not JUPITER_API_KEY: + missing.append("JUPITER_API_KEY") + if not HELIUS_API_KEY: + missing.append("HELIUS_API_KEY") + return missing + +# ---- historical corpus (every pump.fun graduation, pulled from the migration wallet + pump.fun's swap-api) ---- +CORPUS_DIR = DATA_DIR / "corpus" +CORPUS_DAYS = env_int("CORPUS_DAYS", 240) # enumerate graduations this far back (must reach before CORPUS_PULL_BEFORE) +CORPUS_MIGRATION_AUTHORITY = env_str("CORPUS_MIGRATION_AUTHORITY", "39azUYFWPz3VHgKCf3VChUwbpURdCHRxjWVowf5jUJjg") +CORPUS_SWAP_API = env_str("CORPUS_SWAP_API", "https://swap-api.pump.fun") +CORPUS_PACE_S = env_float("CORPUS_PACE_S", 2.2) # seconds between swap-api requests (~30/min bucket; 1/s throttles after ~20) +CORPUS_PACE_MIN_S = env_float("CORPUS_PACE_MIN_S", 2.2) +CORPUS_CANDLE_1M_H = env_float("CORPUS_CANDLE_1M_H", 12.0) # 1-minute candles for this long after graduation +CORPUS_CANDLE_5M_D = env_float("CORPUS_CANDLE_5M_D", 7.0) # then 5-minute candles until this many days +CORPUS_MIN_LIFE_H = env_float("CORPUS_MIN_LIFE_H", 1.0) # per-trade rows only for tokens that traded at least this long +CORPUS_TRADES_H = env_float("CORPUS_TRADES_H", 3.0) # per-trade rows for the first hours after graduation +CORPUS_TRADES_DAYS = env_int("CORPUS_TRADES_DAYS", 400) # ... and only for graduations this recent (the 10 % sample bounds the cost) +CORPUS_TRADES_MAX_PAGES = env_int("CORPUS_TRADES_MAX_PAGES", 200) # 100 trades per page +CORPUS_TRADES_SAMPLE = env_float("CORPUS_TRADES_SAMPLE", 0.10) # share of surviving tokens that get per-trade rows (deterministic by mint) +CORPUS_MIN_AGE_H = env_float("CORPUS_MIN_AGE_H", 6.0) # pull a token only once its first hours are complete +CORPUS_RQ0_SOL = env_float("CORPUS_RQ0_SOL", 79.0) # PumpSwap quote reserve at migration (85 SOL curve − 6 SOL fee); R_q(t) ≈ RQ0·sqrt(p/p_grad) +CORPUS_FEATURES_DIR = CORPUS_DIR / "features" + +# ---- PumpAPI historical replay (free hourly archive of decoded pump.fun / PumpSwap events since 2026-04-18) ---- +REPLAY_URL = env_str("REPLAY_URL", "https://replay.pumpapi.io") +REPLAY_START = env_str("REPLAY_START", "2026-04-18") +REPLAY_PARALLEL = env_int("REPLAY_PARALLEL", 4) # concurrent hour downloads (~2-4 MB/s each) +REPLAY_ASSEMBLE = env_bool("REPLAY_ASSEMBLE", True) # 0: download and parse only (no assembly / corpus_meta / mature rounds) +REPLAY_DIR = CORPUS_DIR / "replay" +CORPUS_PULL_BEFORE = env_str("CORPUS_PULL_BEFORE", REPLAY_START) # swap-api puller only handles graduations before the replay archive begins + +# ---- PumpAPI live stream (free firehose, same events as the replay archive; the selector's parity feed) ---- +PUMPSTREAM_URL = env_str("PUMPSTREAM_URL", "wss://stream.pumpapi.io") +PUMP_MINUTES_KEEP_DAYS = env_int("PUMP_MINUTES_KEEP_DAYS", 7) # older minute rows are archived to Parquet, then removed from the table diff --git a/fly_trader/db/__init__.py b/fly_trader/db/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fly_trader/db/apilog.py b/fly_trader/db/apilog.py new file mode 100644 index 0000000000000000000000000000000000000000..e4f085d5b69ef504244a11faa98da3aad61d38d3 --- /dev/null +++ b/fly_trader/db/apilog.py @@ -0,0 +1,49 @@ +"""Persist every external API call to ``api_calls``. Never stores keys; never breaks the caller.""" +from __future__ import annotations + +import json +import logging + +from .connection import transaction +from ..logging_setup import scrub + +log = logging.getLogger(__name__) + + +def record_api_call(service: str, endpoint: str, method: str = "GET", status: int | None = None, + latency_ms: int | None = None, ok: bool = True, error: str | None = None, + request: dict | None = None, response_bytes: int | None = None) -> None: + try: + req_json = None + if request is not None: + req_json = json.loads(scrub(json.dumps(request, default=str))) + with transaction() as conn: + conn.execute( + "INSERT INTO api_calls (service, endpoint, method, status, latency_ms, ok, error, request, response_bytes) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", + (service, endpoint, method, status, latency_ms, ok, scrub(error) if error else None, + json.dumps(req_json) if req_json is not None else None, response_bytes), + ) + except Exception as e: # a DB hiccup must never break an API call + log.warning("api_calls insert failed: %s", type(e).__name__) + + +def _finite(x): + if isinstance(x, float) and (x != x or x in (float("inf"), float("-inf"))): + return None + if isinstance(x, dict): + return {k: _finite(v) for k, v in x.items()} + if isinstance(x, (list, tuple)): + return [_finite(v) for v in x] + return x + + +def record_event(level: str, source: str, message: str, detail: dict | None = None) -> None: + try: + with transaction() as conn: + conn.execute( + "INSERT INTO events (level, source, message, detail) VALUES (%s,%s,%s,%s)", + (level, source, scrub(message), json.dumps(_finite(detail), default=str) if detail is not None else None), + ) + except Exception as e: + log.warning("events insert failed: %s", type(e).__name__) diff --git a/fly_trader/db/connection.py b/fly_trader/db/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..9c6ef3d11c8e82f62b333531d9673878de149490 --- /dev/null +++ b/fly_trader/db/connection.py @@ -0,0 +1,85 @@ +"""psycopg 3 connections to the fly_trader database. + +Pattern from better_bot code/db_connection.py: a production database is refused while pytest is +running unless its name contains "test" (the autouse fixture in tests/conftest.py points +DATABASE_URL at fly_trader_test). +""" +from __future__ import annotations + +import os +import time +from contextlib import contextmanager + +import psycopg +from psycopg.conninfo import conninfo_to_dict +from psycopg.rows import dict_row + +from .. import config + + +class ProductionDatabaseUnderPytest(RuntimeError): + pass + + +def database_url() -> str: + return os.environ.get("DATABASE_URL") or config.DATABASE_URL + + +def database_name(url: str | None = None) -> str: + return conninfo_to_dict(url or database_url()).get("dbname") or "fly_trader" # URL or key=value DSN + + +def _refuse_production_under_pytest(url: str) -> None: + if "PYTEST_CURRENT_TEST" in os.environ and "test" not in database_name(url): + raise ProductionDatabaseUnderPytest( + f"refusing to open non-test database {database_name(url)!r} under pytest" + ) + + +def connect(url: str | None = None, *, autocommit: bool = False) -> psycopg.Connection: + url = url or database_url() + _refuse_production_under_pytest(url) + last: Exception | None = None + for attempt in range(3): # a busy machine can refuse one connection; a long training run must not die for it, + # but an unreachable database must still report in seconds, not half a minute: three short attempts, ~10 s in all + try: + return psycopg.connect(url, autocommit=autocommit, row_factory=dict_row, options="-c timezone=UTC", connect_timeout=3) + except psycopg.OperationalError as e: + last = e + if attempt < 2: + time.sleep(0.25 * 2 ** attempt) + raise last + + +@contextmanager +def transaction(url: str | None = None): + """One transaction; commits on success, rolls back on exception.""" + conn = connect(url) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def fetch_all(sql: str, params=None, url: str | None = None) -> list[dict]: + with connect(url) as conn: + with conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchall() + + +def fetch_one(sql: str, params=None, url: str | None = None) -> dict | None: + with connect(url) as conn: + with conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchone() + + +def execute(sql: str, params=None, url: str | None = None) -> None: + with transaction(url) as conn: + with conn.cursor() as cur: + cur.execute(sql, params) diff --git a/fly_trader/db/queries.py b/fly_trader/db/queries.py new file mode 100644 index 0000000000000000000000000000000000000000..c52a1ab6ee6915b864e21cf37df73706c60da630 --- /dev/null +++ b/fly_trader/db/queries.py @@ -0,0 +1,37 @@ +"""Read-side helpers for the console and `status`. Everything counts rows; nothing reads status flags.""" +from __future__ import annotations + +import json + +from .connection import connect + + +def q(sql: str, params=None) -> list[dict]: + with connect() as conn: + return [dict(r) for r in conn.execute(sql, params).fetchall()] + + +def q1(sql: str, params=None) -> dict | None: + rows = q(sql, params) + return rows[0] if rows else None + + +def status() -> dict: + out: dict = {} + out["wealth"] = q("""SELECT DISTINCT ON (book) book, ts, wealth, sol_free, positions_value, exposure, n_open, peak, drawdown + FROM wealth_marks ORDER BY book, ts DESC""") + out["positions"] = q("SELECT book, status, count(*) AS n, COALESCE(sum(realized_sol),0) AS realized FROM positions GROUP BY 1,2 ORDER BY 1,2") + out["fills"] = q("SELECT book, count(*) AS n, min(ts) AS first, max(ts) AS last FROM fills GROUP BY 1 ORDER BY 1") + out["beats_last_hour"] = (q1("SELECT count(*) AS n, max(ts) AS last FROM beats WHERE ts > now() - interval '1 hour'") or {}) + out["decisions_last_hour"] = q("SELECT kind, COALESCE(rail,'') AS rail, count(*) AS n FROM decisions WHERE ts > now() - interval '1 hour' GROUP BY 1,2 ORDER BY 3 DESC") + out["tokens"] = q("SELECT watch_status, count(*) AS n FROM tokens GROUP BY 1 ORDER BY 1") + out["circuit"] = q1("SELECT * FROM circuit_state WHERE id = 1") + out["brain"] = q1("SELECT * FROM brain_state") + out["processes"] = q("SELECT name, pid, started_at, stopped_at, exit_code FROM processes ORDER BY started_at DESC LIMIT 10") + out["events"] = q("SELECT ts, level, source, message FROM events ORDER BY ts DESC LIMIT 10") + return out + + +def print_status() -> None: + s = status() + print(json.dumps(s, indent=1, default=str)) diff --git a/fly_trader/db/schema.py b/fly_trader/db/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..89f8b1bd9ccc05f3f363f2ec7babf10b460a2066 --- /dev/null +++ b/fly_trader/db/schema.py @@ -0,0 +1,363 @@ +"""Versioned DDL ladder for the fly_trader database. + +Pattern from better_bot code/db_schema.py: a singleton ``schema_version`` row, a cross-process +advisory lock around migrations, and an idempotent list of statements. Adding a migration means +appending a version-gated block AND bumping SCHEMA_VERSION, or it never runs. +""" +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone + +import psycopg +from psycopg import sql + +from .. import config +from .connection import connect, database_name, database_url + +log = logging.getLogger(__name__) + +SCHEMA_VERSION = 6 # 5: the plastic fly's tables and per-book halts (BASE_DDL); 6: the selector's strategy stack (MIGRATIONS[6]) +_LOCK_KEY = 0x666C795F6D6967 # "fly_mig" + +BASE_DDL: list[str] = [ + """CREATE TABLE IF NOT EXISTS schema_version ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + version bigint NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS wallet_events ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), + kind text NOT NULL, pubkey text, detail jsonb)""", + """CREATE TABLE IF NOT EXISTS tokens ( + mint text PRIMARY KEY, symbol text, name text, decimals int, token_program text, + launchpad text, partner_config text, graduated_pool text, graduated_at timestamptz, + first_pool_id text, first_pool_created_at timestamptz, + mint_auth_disabled boolean, freeze_auth_disabled boolean, + first_seen timestamptz NOT NULL DEFAULT now(), last_seen timestamptz NOT NULL DEFAULT now(), + watch_status text NOT NULL DEFAULT 'candidate', raw jsonb)""", + "CREATE INDEX IF NOT EXISTS tokens_graduated_at_idx ON tokens (graduated_at DESC)", + "CREATE INDEX IF NOT EXISTS tokens_watch_status_idx ON tokens (watch_status)", + """CREATE TABLE IF NOT EXISTS token_stats ( + id bigserial PRIMARY KEY, mint text NOT NULL, ts timestamptz NOT NULL DEFAULT now(), + organic_score real, organic_label text, holder_count int, liquidity_usd double precision, + usd_price double precision, mcap double precision, top_holders_pct real, dev_balance_pct real, + dev_mints int, is_sus boolean, mint_auth_disabled boolean, freeze_auth_disabled boolean, + is_verified boolean, stats jsonb)""", + "CREATE INDEX IF NOT EXISTS token_stats_mint_ts_idx ON token_stats (mint, ts DESC)", + """CREATE TABLE IF NOT EXISTS watch_pools ( + pool text PRIMARY KEY, mint text NOT NULL, quote_mint text, program_id text, program_label text, + base_vault text, quote_vault text, base_decimals int, quote_decimals int, source text, + active boolean NOT NULL DEFAULT true, tradable boolean NOT NULL DEFAULT true, + added_at timestamptz NOT NULL DEFAULT now(), deactivated_at timestamptz, reason text)""", + "CREATE INDEX IF NOT EXISTS watch_pools_active_idx ON watch_pools (active)", + "CREATE INDEX IF NOT EXISTS watch_pools_mint_idx ON watch_pools (mint)", + """CREATE TABLE IF NOT EXISTS swap_tape ( + id bigserial, ts timestamptz NOT NULL, slot bigint, sig text, tx_index int, + pool text NOT NULL, mint text, side smallint NOT NULL, + amount_base numeric(30,0), amount_quote bigint, price_sol double precision, + signer text, res_base numeric(30,0), res_quote bigint, program_label text, + PRIMARY KEY (id, ts)) PARTITION BY RANGE (ts)""", + "CREATE INDEX IF NOT EXISTS swap_tape_pool_ts_idx ON swap_tape (pool, ts)", + """CREATE TABLE IF NOT EXISTS capture_status ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + updated_at timestamptz NOT NULL DEFAULT now(), last_swap_ts timestamptz, rows_total bigint DEFAULT 0, + pools_subscribed int DEFAULT 0, subscription_id bigint, reconnects int DEFAULT 0, + decode_failures bigint DEFAULT 0, last_error text)""", + """CREATE TABLE IF NOT EXISTS runs ( + run_id uuid PRIMARY KEY, kind text NOT NULL, started_at timestamptz NOT NULL DEFAULT now(), + ended_at timestamptz, git_sha text, config jsonb, connectome_sha256 text, encoder_sha256 text, + brain_snapshot_id bigint, corpus text, split jsonb, status text NOT NULL DEFAULT 'running', + metrics jsonb)""", + """CREATE TABLE IF NOT EXISTS populations ( + id smallint PRIMARY KEY, name text NOT NULL UNIQUE, n int NOT NULL, kind text, source text)""", + """CREATE TABLE IF NOT EXISTS beats ( + id bigserial PRIMARY KEY, run_id uuid, ts timestamptz NOT NULL DEFAULT now(), beat_no bigint, + sim_ts timestamptz, tape_last_id bigint, n_slots_active int, n_held_live int, n_held_paper int, + ticks int, gpu_ms int, total_ms int, feed_age_ms int, notes jsonb)""", + "CREATE INDEX IF NOT EXISTS beats_run_ts_idx ON beats (run_id, ts)", + "CREATE INDEX IF NOT EXISTS beats_ts_idx ON beats (ts)", + """CREATE TABLE IF NOT EXISTS beat_slots ( + beat_id bigint NOT NULL, slot smallint NOT NULL, ts timestamptz NOT NULL, + mint text, pool text, dwell_beats int, features real[], feature_mask bigint, danger real, + portfolio real[], glomeruli real[], stim real[], kc_active_frac real, m_hat real, + rho_app real, rho_av real, dan_rew real, dan_pun real, delta_in real, + decision_kind text, decision_size real, softmax_p real, + PRIMARY KEY (beat_id, slot, ts)) PARTITION BY RANGE (ts)""", + "CREATE INDEX IF NOT EXISTS beat_slots_mint_ts_idx ON beat_slots (mint, ts)", + """CREATE TABLE IF NOT EXISTS brain_activity ( + beat_id bigint PRIMARY KEY, group_rates real[], kc_sparsity real, mbon_app_rate real, + mbon_av_rate real, dan_rew_rate real, dan_pun_rate real, total_spikes bigint, + v_mean real, v_max real, nan_flag boolean DEFAULT false)""", + """CREATE TABLE IF NOT EXISTS decisions ( + id bigserial PRIMARY KEY, beat_id bigint, run_id uuid, ts timestamptz NOT NULL DEFAULT now(), + slot smallint, mint text, pool text, kind text NOT NULL, m_hat real, size_sol real, + forced boolean NOT NULL DEFAULT false, rail text, reason text, softmax_p real, + book_targets text[], detail jsonb)""", + "CREATE INDEX IF NOT EXISTS decisions_ts_idx ON decisions (ts)", + "CREATE INDEX IF NOT EXISTS decisions_mint_ts_idx ON decisions (mint, ts)", + "CREATE INDEX IF NOT EXISTS decisions_kind_ts_idx ON decisions (kind, ts)", + """CREATE TABLE IF NOT EXISTS orders ( + id bigserial PRIMARY KEY, decision_id bigint, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(), + attempt int NOT NULL DEFAULT 1, side text NOT NULL, input_mint text, output_mint text, + amount_in numeric(30,0), slippage_bps int, request jsonb, order_response jsonb, + execute_request jsonb, execute_response jsonb, request_id text, signature text, + last_valid_block_height bigint, router text, fee_bps int, program_ids text[], + status text, error_code int, error text, latency_ms int)""", + "CREATE INDEX IF NOT EXISTS orders_decision_idx ON orders (decision_id)", + "CREATE INDEX IF NOT EXISTS orders_signature_idx ON orders (signature)", + "CREATE INDEX IF NOT EXISTS orders_ts_idx ON orders (ts)", + """CREATE TABLE IF NOT EXISTS fills ( + id bigserial PRIMARY KEY, order_id bigint, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(), + signature text, slot bigint, mint text, side text, token_delta numeric(30,0), + sol_delta_lamports bigint, price_sol double precision, fee_lamports bigint, platform_fee jsonb, + verified_by text, pre_snapshot jsonb, post_snapshot jsonb)""", + "CREATE INDEX IF NOT EXISTS fills_book_ts_idx ON fills (book, ts)", + "CREATE INDEX IF NOT EXISTS fills_mint_ts_idx ON fills (mint, ts)", + """CREATE TABLE IF NOT EXISTS positions ( + id bigserial PRIMARY KEY, book text NOT NULL, mint text NOT NULL, pool text, + opened_at timestamptz NOT NULL DEFAULT now(), closed_at timestamptz, + entry_decision_id bigint, exit_decision_id bigint, qty numeric(30,0) NOT NULL DEFAULT 0, + cost_sol double precision NOT NULL DEFAULT 0, entry_price double precision, peak_price double precision, + last_mark_price double precision, last_mark_ts timestamptz, exit_price double precision, + realized_sol double precision, fees_sol double precision NOT NULL DEFAULT 0, + status text NOT NULL DEFAULT 'open', forced_exit_kind text, last_swap_ts timestamptz, satiety double precision NOT NULL DEFAULT 0)""", + "CREATE UNIQUE INDEX IF NOT EXISTS positions_open_uniq ON positions (book, mint) WHERE status = 'open'", + "CREATE INDEX IF NOT EXISTS positions_book_status_idx ON positions (book, status)", + """CREATE TABLE IF NOT EXISTS wealth_marks ( + beat_id bigint NOT NULL, book text NOT NULL, ts timestamptz NOT NULL DEFAULT now(), + sol_free double precision, positions_value double precision, exit_cost double precision, + wealth double precision, peak double precision, drawdown real, exposure double precision, n_open int, + PRIMARY KEY (beat_id, book))""", + "CREATE INDEX IF NOT EXISTS wealth_marks_book_ts_idx ON wealth_marks (book, ts)", + """CREATE TABLE IF NOT EXISTS rewards ( + beat_id bigint NOT NULL, slot smallint NOT NULL, book text, mint text, + r_slot real, r_global real, r_tilde real, m_prev real, m_now real, delta real, source text, + PRIMARY KEY (beat_id, slot))""", + """CREATE TABLE IF NOT EXISTS synapse_updates ( + beat_id bigint PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), eta real, gamma real, + n_slots int, reward_source text, sum_abs real, max_abs real, frob real, frob_capped boolean, + n_pos int, n_neg int, n_clipped int, w_mean real, w_min real, w_max real, + delta_path text, delta_sha256 text)""", + """CREATE TABLE IF NOT EXISTS brain_snapshots ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), run_id uuid, beat_id bigint, + path text NOT NULL, sha256 text, kind text NOT NULL, promoted_at timestamptz, promoted_by text, note text)""", + """CREATE TABLE IF NOT EXISTS brain_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + live_snapshot_id bigint, pending_snapshot_id bigint, updated_at timestamptz NOT NULL DEFAULT now())""", + "INSERT INTO brain_state (singleton) VALUES (true) ON CONFLICT DO NOTHING", + """CREATE TABLE IF NOT EXISTS encoder_state ( + name text PRIMARY KEY, version int NOT NULL DEFAULT 1, seed bigint, path text, sha256 text, + running_mean real[], running_var real[], n bigint NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS circuit_state ( + id int PRIMARY KEY CHECK (id = 1), fail_count int NOT NULL DEFAULT 0, last_failure_ts timestamptz, + tripped boolean NOT NULL DEFAULT false, kill_switch boolean NOT NULL DEFAULT false, kill_reason text, + peak_wealth double precision, entries_paused boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now())""", + "INSERT INTO circuit_state (id) VALUES (1) ON CONFLICT DO NOTHING", + """CREATE TABLE IF NOT EXISTS circuit_events ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), kind text NOT NULL, detail jsonb)""", + """CREATE TABLE IF NOT EXISTS notional_ledger ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), sol double precision NOT NULL, kind text)""", + "CREATE INDEX IF NOT EXISTS notional_ledger_ts_idx ON notional_ledger (ts)", + """CREATE TABLE IF NOT EXISTS api_calls ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), service text NOT NULL, endpoint text, + method text, status int, latency_ms int, ok boolean, error text, request jsonb, response_bytes int)""", + "CREATE INDEX IF NOT EXISTS api_calls_ts_idx ON api_calls (ts)", + "CREATE INDEX IF NOT EXISTS api_calls_service_ok_ts_idx ON api_calls (service, ok, ts)", + """CREATE TABLE IF NOT EXISTS events ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), level text NOT NULL DEFAULT 'info', + source text, message text NOT NULL, detail jsonb)""", + "CREATE INDEX IF NOT EXISTS events_ts_idx ON events (ts)", + """CREATE TABLE IF NOT EXISTS slot_visits ( + mint text PRIMARY KEY, last_visit_ts timestamptz, visits bigint NOT NULL DEFAULT 0)""", + """CREATE TABLE IF NOT EXISTS processes ( + id bigserial PRIMARY KEY, name text NOT NULL, pid int NOT NULL, cmd text[], + started_at timestamptz NOT NULL DEFAULT now(), stopped_at timestamptz, exit_code int, + log_path text, started_by text)""", + "CREATE UNIQUE INDEX IF NOT EXISTS processes_live_uniq ON processes (name) WHERE stopped_at IS NULL", + """CREATE TABLE IF NOT EXISTS corpus_tokens ( + mint text PRIMARY KEY, graduated_at timestamptz NOT NULL, grad_sig text, grad_slot bigint, + source text NOT NULL DEFAULT 'migration_wallet', status text NOT NULL DEFAULT 'pending', + life_h double precision, candles_1m int, candles_5m int, trades int, trades_through timestamptz, + candle_path text, trade_path text, last_error text, + added_at timestamptz NOT NULL DEFAULT now(), updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS corpus_tokens_status_idx ON corpus_tokens (status, graduated_at DESC)", + """CREATE TABLE IF NOT EXISTS corpus_features ( + mint text PRIMARY KEY, built_at timestamptz NOT NULL DEFAULT now(), rows int, trade_rows int, has_trades boolean, path text)""", + """CREATE TABLE IF NOT EXISTS replay_hours ( + hour timestamptz PRIMARY KEY, status text NOT NULL DEFAULT 'done', trades int, events int, bytes bigint, took_s real, + last_error text, done_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS replay_days ( + day date PRIMARY KEY, graduations int, tokens_written int, took_s real, assembled_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS corpus_meta ( + mint text PRIMARY KEY, create_ts timestamptz, creator text, dev_sol double precision, dev_tokens double precision, dev_share real, + supply double precision, mayhem boolean, uri text, name text, symbol text, ttg_min double precision, rq0 double precision, pool_id text, + prior_launches int, prior_grads int, prior_known int, prior_rug_share real, prior_moon_share real, + own_dd60 real, own_max60 real, own_alive6h boolean, updated_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS corpus_meta_creator_idx ON corpus_meta (creator)", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS graduated_at timestamptz", + "CREATE INDEX IF NOT EXISTS corpus_meta_creator_g_idx ON corpus_meta (creator, graduated_at)", + """CREATE TABLE IF NOT EXISTS mature_days ( + day date PRIMARY KEY, mints int, rows int, took_s real, built_at timestamptz NOT NULL DEFAULT now())""", + """CREATE TABLE IF NOT EXISTS pump_minutes ( + mint text NOT NULL, ts timestamptz NOT NULL, pool_id text, open double precision, high double precision, low double precision, close double precision, + buy_sol double precision, sell_sol double precision, n_buys int, n_sells int, n_traders int, resq_sol double precision, fee_rate double precision, + PRIMARY KEY (mint, ts))""", + "CREATE INDEX IF NOT EXISTS pump_minutes_ts_idx ON pump_minutes (ts)", + """CREATE TABLE IF NOT EXISTS pump_events ( + sig text PRIMARY KEY, ts timestamptz NOT NULL, action text NOT NULL, pool text, mint text, pool_id text, signer text, dev_sol double precision, + dev_tokens double precision, supply double precision, mayhem boolean, quote_in_pool double precision, name text, symbol text, uri text)""", + "CREATE INDEX IF NOT EXISTS pump_events_mint_idx ON pump_events (mint)", + "CREATE INDEX IF NOT EXISTS pump_events_creates_idx ON pump_events (signer, ts) WHERE action = 'create'", + # PumpSwap pools NOT created by a pump.fun migration (createPool by anyone): unburned liquidity, blocked from the universe + """CREATE TABLE IF NOT EXISTS pump_pools ( + pool_id text PRIMARY KEY, mint text, created_by text, ts timestamptz, source text NOT NULL, added_at timestamptz NOT NULL DEFAULT now())""", + "CREATE INDEX IF NOT EXISTS pump_pools_mint_idx ON pump_pools (mint)", + """CREATE TABLE IF NOT EXISTS ui_settings ( + key text PRIMARY KEY, value jsonb, updated_at timestamptz NOT NULL DEFAULT now())""", + # the plastic fly (agent/fly_session.py): every scored minute (pending rows are its synaptic tags; x is dropped once + # the label resolves), daily calibrations per arm (plastic / frozen), hourly learning statistics, rollbacks + """CREATE TABLE IF NOT EXISTS fly_scored ( + ts timestamptz NOT NULL, mint text NOT NULL, x real[], score real, frozen_score real, line real, frozen_line real, + label real, state text NOT NULL DEFAULT 'pending', resolved_at timestamptz, bootstrap_id bigint, + PRIMARY KEY (ts, mint))""", + "CREATE INDEX IF NOT EXISTS fly_scored_state_ts_idx ON fly_scored (state, ts)", + """CREATE TABLE IF NOT EXISTS fly_calibrations ( + day date NOT NULL, arm text NOT NULL, line real, sizing jsonb, trades int, total real, mean real, window_days int, + created_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (day, arm))""", + """CREATE TABLE IF NOT EXISTS fly_updates ( + hour timestamptz PRIMARY KEY, n int, mean_delta real, mean_abs_delta real, step real, capped int, drift real, ic real, detail jsonb)""", + """CREATE TABLE IF NOT EXISTS fly_rollbacks ( + id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), reason text, checks jsonb, from_snapshot bigint, to_snapshot bigint)""", + # per-book entry halts of the paper race books (agent/rails.check_book_drawdown); the live book uses circuit_state + """CREATE TABLE IF NOT EXISTS book_state ( + book text PRIMARY KEY, halted boolean NOT NULL DEFAULT false, reason text, peak double precision, + updated_at timestamptz NOT NULL DEFAULT now())""", +] + +# Version-gated migrations: {version: [statements]} applied when stored version < version. +MIGRATIONS: dict[int, list[str]] = { + 2: ["ALTER TABLE beat_slots ALTER COLUMN feature_mask TYPE bigint"], + 3: ["ALTER TABLE positions ADD COLUMN IF NOT EXISTS satiety double precision NOT NULL DEFAULT 0"], + 4: ["ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS fee_rate double precision"], # the pool fee the stream reports as charged + # 6: the selector's strategy stack — per-minute wallet flow (train/flow.py), graduation-time curve facts and insiders + # (train/corpus_meta.py), per-position hold and strategy, the fly's per-strategy scored rows + 6: ["ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS n_buyers int", + "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS wash_sol double precision", + "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS wash_buy_sol double precision", + "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS top_sell_sol double precision", + "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS insider_sell_sol double precision", + "ALTER TABLE pump_minutes ADD COLUMN IF NOT EXISTS skill_buy real[]", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS bundle_share real", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS dev_hold_share real", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS dev_sold_frac real", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS grad_hhi real", + "ALTER TABLE corpus_meta ADD COLUMN IF NOT EXISTS curve_known boolean", + "CREATE TABLE IF NOT EXISTS token_insiders (mint text NOT NULL, wallet text NOT NULL, kind text NOT NULL, PRIMARY KEY (mint, wallet))", + "ALTER TABLE positions ADD COLUMN IF NOT EXISTS hold_s double precision", + "ALTER TABLE positions ADD COLUMN IF NOT EXISTS strategy text", + "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS strategy text NOT NULL DEFAULT 'ev'", + "ALTER TABLE fly_scored ADD COLUMN IF NOT EXISTS hold_min int", + "ALTER TABLE fly_scored DROP CONSTRAINT IF EXISTS fly_scored_pkey", + "ALTER TABLE fly_scored ADD PRIMARY KEY (ts, mint, strategy)"], +} + + +def ensure_database(url: str | None = None) -> bool: + """Create the database named in DATABASE_URL if it does not exist. Returns True if created.""" + url = url or database_url() + name = database_name(url) + admin_url = psycopg.conninfo.make_conninfo(url, dbname="postgres") # same user/host/password, URL or key=value DSN + with psycopg.connect(admin_url, autocommit=True) as conn: + row = conn.execute("SELECT 1 FROM pg_database WHERE datname = %s", (name,)).fetchone() + if row: + return False + conn.execute(f'CREATE DATABASE "{name}"') + log.info("created database %s", name) + return True + + +def apply_schema(url: str | None = None) -> int: + """Apply the ladder under an advisory lock. Returns the resulting version.""" + with connect(url) as conn: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_lock(%s)", (_LOCK_KEY,)) + try: + for stmt in BASE_DDL: + cur.execute(stmt) + cur.execute("SELECT version FROM schema_version WHERE singleton") + row = cur.fetchone() + current = int(row["version"]) if row else 0 + for v in sorted(MIGRATIONS): + if current < v: + for stmt in MIGRATIONS[v]: + cur.execute(stmt) + current = v + target = max(current, SCHEMA_VERSION) + cur.execute( + "INSERT INTO schema_version (singleton, version) VALUES (true, %s) " + "ON CONFLICT (singleton) DO UPDATE SET version = EXCLUDED.version, updated_at = now()", + (target,), + ) + conn.commit() + ensure_partitions(conn) + conn.commit() + return target + except Exception: + conn.rollback() + raise + finally: + cur.execute("SELECT pg_advisory_unlock(%s)", (_LOCK_KEY,)) + conn.commit() + + +def _partition_name(base: str, start: datetime, granularity: str) -> str: + fmt = "%Y%m%d%H" if granularity == "hour" else "%Y%m%d" + return f"{base}_{start.strftime(fmt)}" + + +def ensure_partition(conn: psycopg.Connection, base: str, start: datetime, granularity: str) -> None: + start = start.astimezone(timezone.utc) + if granularity == "hour": + start = start.replace(minute=0, second=0, microsecond=0) + end = start + timedelta(hours=1) + else: + start = start.replace(hour=0, minute=0, second=0, microsecond=0) + end = start + timedelta(days=1) + name = _partition_name(base, start, granularity) + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM pg_class WHERE relname = %s", (name,)) + if cur.fetchone(): + return + cur.execute( + sql.SQL("CREATE TABLE IF NOT EXISTS {} PARTITION OF {} FOR VALUES FROM ({}) TO ({})").format( + sql.Identifier(name), sql.Identifier(base), sql.Literal(start), sql.Literal(end) + ) + ) + + +def ensure_partitions(conn: psycopg.Connection, now: datetime | None = None) -> None: + """Partitions for the near past and future so writers never hit a missing partition.""" + now = now or datetime.now(timezone.utc) + for h in range(-2, 4): + ensure_partition(conn, "swap_tape", now + timedelta(hours=h), "hour") + for d in range(-1, 3): + ensure_partition(conn, "beat_slots", now + timedelta(days=d), "day") + + +def ensure_partitions_for(conn: psycopg.Connection, ts: datetime) -> None: + """Partitions around an arbitrary (e.g. simulated) timestamp.""" + ensure_partition(conn, "swap_tape", ts, "hour") + ensure_partition(conn, "swap_tape", ts + timedelta(hours=1), "hour") + ensure_partition(conn, "beat_slots", ts, "day") + ensure_partition(conn, "beat_slots", ts + timedelta(days=1), "day") + + +def table_names(url: str | None = None) -> list[str]: + with connect(url) as conn: + rows = conn.execute( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' " + "AND tablename NOT LIKE 'swap_tape_%%' AND tablename NOT LIKE 'beat_slots_%%' ORDER BY 1" + ).fetchall() + return [r["tablename"] for r in rows] diff --git a/fly_trader/execution/__init__.py b/fly_trader/execution/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/execution/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/execution/broker_live.py b/fly_trader/execution/broker_live.py new file mode 100644 index 0000000000000000000000000000000000000000..c830152e2ec9da4c493da1107f714e73ac9a578a --- /dev/null +++ b/fly_trader/execution/broker_live.py @@ -0,0 +1,455 @@ +"""Live execution through Jupiter Swap API v2 with balance-verified fills (plan §9). + +Per attempt (one in-flight order per process, guarded by a module lock): + assert_signing_allowed() -> pre = snapshot_balances() -> GET /order (taker = bot, excludeRouters= + jupiterz) -> assert signatureFeePayer == taker -> orders row (status 'ordered') -> sign only our + signer slot -> POST /execute -> orders row updated -> poll getSignatureStatuses every 1 s until + confirmed or getBlockHeight() > lastValidBlockHeight (no other timeout) -> post = snapshot -> + fill iff the input balance decreased AND the output balance increased (a confirmed sell whose token + balance fell is a fill even when the SOL delta is <= 0: dust proceeds below fees) -> fills row. + +Jupiter's execute status is recorded but never trusted alone: 'Failed' with moved balances is a +fill (verified_by='balance_delta_despite_failed'); 'Success' without movement is a fills row with +verified_by='unverified' plus an events row after a getTransaction check. On execute code -1000 +(failed to land) or a slippage error with NO balance movement the order is retried with +slippage + step_bps up to max_slippage_bps; each attempt is its own orders row. + +orders.status words: ordered | order_error | sign_error | executed | execute_error | confirmed | +failed_on_chain | expired | unsent. fills.verified_by: balance_delta | +balance_delta_despite_failed | unverified. price_sol = |sol delta| / 1e9 / (|token delta| / +10^decimals) — an all-in effective price (transaction fee, priority fee and any ATA rent are inside +the SOL delta; fee_lamports carries the estimate so the ledger must not add it again). +Every write is committed immediately so a crash mid-poll leaves an auditable row for verify-fills. +""" +from __future__ import annotations + +import base64 +import logging +import re +import threading +import time +from dataclasses import dataclass + +from psycopg.types.json import Jsonb +from solders.hash import Hash +from solders.instruction import AccountMeta, Instruction +from solders.message import MessageV0 +from solders.pubkey import Pubkey +from solders.transaction import VersionedTransaction + +from .. import config +from ..chain.balances import Snapshot, compute_delta, snapshot_balances +from ..chain.cluster_guard import assert_signing_allowed +from ..chain.jupiter_swap import JupiterError, JupiterSwap +from ..db.connection import transaction +from ..chain.keys import load_keypair +from ..chain.rpc import TOKEN_2022_PROGRAM_ID, TOKEN_PROGRAM_ID, HttpSolanaRpc +from ..chain.signing import NotASigner, n_lookup_programs, sign_transaction_b64, static_program_ids, transaction_id +from ..db.apilog import record_event +from ..db.connection import connect, transaction +from ..logging_setup import scrub + +log = logging.getLogger(__name__) + +_INFLIGHT = threading.Lock() # one in-flight order at a time (plan §9) + +CONFIRM_POLL_S = 1.0 +BLOCKHASH_VALIDITY_BLOCKS = 150 # fallback horizon when /order carries no lastValidBlockHeight +CLOSE_ATA_BATCH = 10 +SYSTEM_PROGRAM_ID = "11111111111111111111111111111111" +COMPUTE_BUDGET_PROGRAM_ID = "ComputeBudget111111111111111111111111111111" +ASSOCIATED_TOKEN_PROGRAM_ID = "ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL" +JUPITER_PROGRAM_IDS = ("JUP6LkbZbjS1jKKwapdHNy74zcZ3tLUZoi5QNyVTaV4",) +KNOWN_PROGRAM_IDS = frozenset({SYSTEM_PROGRAM_ID, COMPUTE_BUDGET_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID, + TOKEN_PROGRAM_ID, TOKEN_2022_PROGRAM_ID, *JUPITER_PROGRAM_IDS}) +RETRY_CODES = frozenset({-1000}) # aggregator: failed to land +_SLIPPAGE_RE = re.compile(r"slippage|0x1771|\b6001\b", re.IGNORECASE) # Jupiter SlippageToleranceExceeded = 6001 + + +@dataclass +class FillResult: + ok: bool + order_id: int | None + fill_id: int | None + signature: str | None + status: str | None + code: int | None + error: str | None + token_delta: int + lamports_delta: int + price_sol: float | None + attempts: int + + +def _int_or_none(v) -> int | None: + try: + return int(v) if v is not None else None + except (TypeError, ValueError): + return None + + +def await_confirmation(rpc, signature: str | None, last_valid_block_height: int | None) -> tuple[str, dict | None]: + """Poll until the signature is confirmed/finalized, fails on chain, or the chain's block height + passes ``last_valid_block_height`` with no status for the signature (it can then never land; a + signature already seen as processed keeps being polled until it confirms or its status disappears). + Sleeps 1 s between polls; there is deliberately no other timeout.""" + if not signature: + return "unsent", None + if last_valid_block_height is None: + last_valid_block_height = rpc.get_block_height() + BLOCKHASH_VALIDITY_BLOCKS + while True: + st = (rpc.get_signature_statuses([signature]) or [None])[0] + if st: + if st.get("err") is not None: + return "failed_on_chain", st + if st.get("confirmationStatus") in ("confirmed", "finalized"): + return "confirmed", st + if not st and rpc.get_block_height() > last_valid_block_height: + return "expired", st + time.sleep(CONFIRM_POLL_S) + + +class LiveBroker: + def __init__(self, rpc=None, jup=None, keypair=None): + self._rpc = rpc + self._jup = jup + self._keypair = keypair + self._labels: dict[str, str] | None = None + + @property + def rpc(self): + if self._rpc is None: + self._rpc = HttpSolanaRpc() + return self._rpc + + @property + def jup(self): + if self._jup is None: + self._jup = JupiterSwap() + return self._jup + + @property + def keypair(self): + if self._keypair is None: + self._keypair = load_keypair() + return self._keypair + + @property + def pubkey(self) -> str: + return str(self.keypair.pubkey()) + + @property + def labels(self) -> dict[str, str]: + if self._labels is None: + try: + self._labels = dict(self.jup.program_id_to_label()) + except Exception as e: + log.warning("program-id-to-label unavailable: %s", type(e).__name__) + self._labels = {} + return self._labels + + # ---- public ---- + def swap(self, conn, *, decision_id: int | None, book: str, side: str, mint: str, amount_in: int, + slippage_bps: int, max_slippage_bps: int, step_bps: int) -> FillResult: + """Buy (lamports in -> ``mint``) or sell (raw token units in -> SOL) with balance verification.""" + assert_signing_allowed() + if side not in ("buy", "sell"): + raise ValueError(f"side must be 'buy' or 'sell', got {side!r}") + amount_in = int(amount_in) + if amount_in <= 0: + raise ValueError("amount_in must be positive") + if side == "buy": + input_mint, output_mint = config.WSOL_MINT, mint + else: + input_mint, output_mint = mint, config.WSOL_MINT + with _INFLIGHT: + slip = int(slippage_bps) + attempt = 0 + while True: + attempt += 1 + result, retryable = self._attempt( + conn, decision_id=decision_id, book=book, side=side, mint=mint, input_mint=input_mint, + output_mint=output_mint, amount_in=amount_in, slippage_bps=slip, attempt=attempt) + result.attempts = attempt + if result.ok or not retryable or step_bps <= 0 or slip + step_bps > max_slippage_bps: + if retryable and not result.ok: + log.warning("order %s not retried: slippage ladder exhausted at %d bps", result.order_id, slip) + return result + slip += int(step_bps) + log.warning("retrying %s %s after %s (code %s) with slippage %d bps", + side, mint, result.status, result.code, slip) + + # ---- one attempt ---- + def _attempt(self, conn, *, decision_id, book, side, mint, input_mint, output_mint, amount_in, + slippage_bps, attempt) -> tuple[FillResult, bool]: + taker = self.pubkey + pre = snapshot_balances(self.rpc, taker) + request = {"inputMint": input_mint, "outputMint": output_mint, "amount": str(amount_in), "taker": taker, + "slippageBps": slippage_bps, "excludeRouters": "jupiterz"} + base = dict(decision_id=decision_id, book=book, attempt=attempt, side=side, input_mint=input_mint, + output_mint=output_mint, amount_in=amount_in, slippage_bps=slippage_bps, request=request) + + def fail(status, error, code=None, order_response=None, order_id=None): + if order_id is None: + order_id = self._insert_order(conn, **base, order_response=order_response, status=status, + error_code=code, error=error) + else: + self._update_order(conn, order_id, status=status, error_code=code, error=error) + log.error("%s %s attempt %d: %s: %s", side, mint, attempt, status, error) + return FillResult(False, order_id, None, None, status, code, error, 0, 0, None, attempt), False + + # 1. order + try: + order = self.jup.order(input_mint, output_mint, amount_in, taker=taker, slippage_bps=slippage_bps) + except JupiterError as e: + return fail("order_error", scrub(str(e))) + err_code = _int_or_none(order.get("errorCode")) + tx_b64 = order.get("transaction") + if err_code is not None or not tx_b64: + msg = order.get("errorMessage") or order.get("error") or "no transaction in /order response" + return fail("order_error", scrub(str(msg)), err_code, order) + fee_payer = order.get("signatureFeePayer") + if fee_payer and fee_payer != taker: + return fail("order_error", f"signatureFeePayer {fee_payer} != taker", None, order) + lvbh = _int_or_none(order.get("lastValidBlockHeight")) + program_ids = static_program_ids(tx_b64) + unknown = [p for p in program_ids if p not in KNOWN_PROGRAM_IDS and p not in self.labels] + if unknown: + log.warning("order references unknown static program ids: %s", unknown) + n_lookup = n_lookup_programs(tx_b64) + if n_lookup: + log.warning("%d instruction(s) use program ids from lookup tables (unresolved in v1)", n_lookup) + order_id = self._insert_order( + conn, **base, order_response=order, request_id=order.get("requestId"), router=order.get("router"), + fee_bps=_int_or_none(order.get("feeBps")), program_ids=program_ids, lvbh=lvbh, status="ordered") + + # 2. sign (our slot only) + try: + signed_b64, _idx = sign_transaction_b64(tx_b64, self.keypair) + except (NotASigner, ValueError) as e: + return fail("sign_error", f"{type(e).__name__}: {scrub(str(e))}", None, order_id=order_id) + if _idx != 0 and not order.get("gasless"): + return fail("sign_error", f"our key is signer {_idx}, but a non-gasless order must have the taker as fee payer", None, order_id=order_id) + tx_sig = transaction_id(signed_b64) + + # 3. execute + exec_req = {"requestId": order.get("requestId"), "lastValidBlockHeight": lvbh, "signedTransaction": signed_b64} + ex: dict | None + try: + ex = self.jup.execute(signed_b64, order.get("requestId"), lvbh) + except JupiterError as e: + ex = None + code = None + error = scrub(str(e)) + signature = tx_sig # the transaction may still have been broadcast: poll it anyway + self._update_order(conn, order_id, execute_request=exec_req, execute_response=None, signature=signature, + status="execute_error", error_code=None, error=error) + else: + code = _int_or_none(ex.get("code")) + error = ex.get("error") + signature = ex.get("signature") or tx_sig + if ex.get("signature") and ex["signature"] != tx_sig: + log.warning("execute signature %s differs from our transaction id %s", ex["signature"], tx_sig) + self._update_order(conn, order_id, execute_request=exec_req, execute_response=ex, signature=signature, + status="executed", error_code=code, error=error, latency_ms=ex.get("_latency_ms")) + + # 4. confirm (or expire), 5. verify by balance deltas — retried: a transport error here must not leave a landed swap unrecorded + chain_status = st = post = None; last_exc = None + for k in range(4): + try: + if chain_status is None: + chain_status, st = await_confirmation(self.rpc, signature, lvbh) + self._update_order(conn, order_id, status=chain_status) + post = snapshot_balances(self.rpc, taker); break + except Exception as e: # RPC transport / RPC errors + last_exc = e; log.warning("confirm/verify attempt %d for %s failed: %s", k + 1, signature, scrub(str(e))) + time.sleep(2.0 * (k + 1)) + if post is None: + fill_id = self._insert_fill(conn, order_id, book, signature, None, mint, side, 0, 0, None, None, order.get("platformFee"), "unknown", pre, pre) + record_event("error", "broker_live", "fill unknown: could not confirm or verify after retries; entries paused until reconciled (verify-fills)", + {"order_id": order_id, "signature": signature, "side": side, "mint": mint, "error": scrub(str(last_exc))[:200]}) + try: # a landed but unrecorded swap must not be bought again or leave a stale position + with transaction() as c2: + c2.execute("UPDATE circuit_state SET entries_paused = true, updated_at = now() WHERE id = 1") + c2.execute("INSERT INTO circuit_events (kind, detail) VALUES ('unknown_fill_pause', %s)", (Jsonb({"order_id": order_id, "signature": signature, "mint": mint, "side": side}),)) + except Exception: + log.exception("could not pause entries after an unknown fill") + raise RuntimeError(f"confirm/verify failed for {signature}: {type(last_exc).__name__}") from last_exc + delta = compute_delta(pre, post) + lam = int(delta["lamports_delta"]) + tok = int(delta["token_deltas"].get(mint, 0)) + # a confirmed sell that moved the tokens is a fill even at lam <= 0 (a rugged token's dust proceeds < fees) + verified = (lam < 0 and tok > 0) if side == "buy" else (tok < 0 and (lam > 0 or chain_status == "confirmed")) + jup_status = (ex or {}).get("status") + slot = _int_or_none((ex or {}).get("slot")) or _int_or_none((st or {}).get("slot")) + if verified: + verified_by = "balance_delta" if jup_status == "Success" else "balance_delta_despite_failed" + decimals = self._decimals(conn, mint, pre, post) + sol = abs(lam) if side == "buy" else max(lam, 0) + price = (sol / config.LAMPORTS_PER_SOL) / (abs(tok) / 10 ** decimals) if decimals is not None else None + fee = self._fee_lamports(side, lam, amount_in, ex, order) + fill_id = self._insert_fill(conn, order_id, book, signature, slot, mint, side, tok, lam, price, fee, + order.get("platformFee"), verified_by, pre, post) + log.info("fill %s: %s %s token_delta=%d lamports_delta=%d price_sol=%s (%s)", + fill_id, side, mint, tok, lam, price, verified_by) + return FillResult(True, order_id, fill_id, signature, chain_status, code, error, tok, lam, price, attempt), False + if jup_status == "Success" or chain_status == "confirmed": + tx_info = None + try: + tx_info = self.rpc.get_transaction(signature) + except Exception as e: + log.warning("getTransaction failed for %s: %s", signature, type(e).__name__) + meta = (tx_info or {}).get("meta") or {} + record_event("warning", "broker_live", + "fill unverified: execute reported success but balances did not move as expected", + {"order_id": order_id, "signature": signature, "side": side, "mint": mint, + "lamports_delta": lam, "token_delta": tok, "chain_status": chain_status, + "jupiter_status": jup_status, "tx_found": tx_info is not None, + "tx_err": meta.get("err"), "tx_fee": meta.get("fee"), "tx_slot": (tx_info or {}).get("slot")}) + fill_id = self._insert_fill(conn, order_id, book, signature, slot, mint, side, tok, lam, None, None, + order.get("platformFee"), "unverified", pre, post) + return FillResult(False, order_id, fill_id, signature, chain_status, code, error, tok, lam, None, attempt), False + retryable = (code in RETRY_CODES) or bool(error and _SLIPPAGE_RE.search(str(error))) + log.error("%s %s attempt %d not filled: chain=%s jupiter=%s code=%s error=%s lamports_delta=%d token_delta=%d", + side, mint, attempt, chain_status, jup_status, code, error, lam, tok) + return FillResult(False, order_id, None, signature, chain_status, code, error, tok, lam, None, attempt), retryable + + # ---- helpers ---- + def _decimals(self, conn, mint: str, pre: Snapshot, post: Snapshot) -> int | None: + d = post.decimals.get(mint, pre.decimals.get(mint)) + if d is not None: + return int(d) + try: + row = conn.execute("SELECT decimals FROM tokens WHERE mint = %s", (mint,)).fetchone() + if row and row["decimals"] is not None: + return int(row["decimals"]) + except Exception: + conn.rollback() + try: + return self.rpc.get_mint_decimals(mint) + except Exception: + return None + + @staticmethod + def _fee_lamports(side: str, lam: int, amount_in: int, ex: dict | None, order: dict) -> int | None: + """Buy: lamports beyond the swap input (tx fee + priority fee + ATA rent). Sell: SOL the route + produced that did not reach the wallet. Approximations from execute amounts, else the order's + fee fields.""" + ex = ex or {} + if side == "buy": + total_in = _int_or_none(ex.get("totalInputAmount")) or amount_in + return abs(lam) - total_in + total_out = _int_or_none(ex.get("totalOutputAmount")) + if total_out is not None: + return total_out - lam + sig_fee = _int_or_none(order.get("signatureFeeLamports")) or 0 + prio = _int_or_none(order.get("prioritizationFeeLamports")) or 0 + return sig_fee + prio + + def _insert_order(self, conn, *, decision_id, book, attempt, side, input_mint, output_mint, amount_in, + slippage_bps, request, order_response=None, request_id=None, router=None, fee_bps=None, + program_ids=None, lvbh=None, status, error_code=None, error=None) -> int: + row = conn.execute( + "INSERT INTO orders (decision_id, book, attempt, side, input_mint, output_mint, amount_in, slippage_bps, " + "request, order_response, request_id, router, fee_bps, program_ids, last_valid_block_height, status, " + "error_code, error) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id", + (decision_id, book, attempt, side, input_mint, output_mint, amount_in, slippage_bps, Jsonb(request), + Jsonb(order_response) if order_response is not None else None, request_id, router, fee_bps, + list(program_ids) if program_ids is not None else None, lvbh, status, error_code, error), + ).fetchone() + conn.commit() + return int(row["id"]) + + def _update_order(self, conn, order_id: int, **fields) -> None: + json_cols = {"execute_request", "execute_response", "order_response"} + cols, vals = [], [] + for k, v in fields.items(): + cols.append(f"{k} = %s") + vals.append(Jsonb(v) if k in json_cols and v is not None else v) + conn.execute(f"UPDATE orders SET {', '.join(cols)} WHERE id = %s", (*vals, order_id)) + conn.commit() + + def _insert_fill(self, conn, order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports, + price_sol, fee_lamports, platform_fee, verified_by, pre: Snapshot, post: Snapshot) -> int: + row = conn.execute( + "INSERT INTO fills (order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports, price_sol, " + "fee_lamports, platform_fee, verified_by, pre_snapshot, post_snapshot) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id", + (order_id, book, signature, slot, mint, side, token_delta, sol_delta_lamports, price_sol, fee_lamports, + Jsonb(platform_fee) if platform_fee is not None else None, verified_by, + Jsonb(pre.to_json()), Jsonb(post.to_json())), + ).fetchone() + conn.commit() + return int(row["id"]) + + +def swap_smoke(sol: float) -> None: + """Operator smoke test (plan phase 3.12): SOL -> USDC then USDC -> SOL through the live broker. + Requires the full live gate (LIVE_ENABLED=1 etc.). Prints both FillResults and the net lamports.""" + assert_signing_allowed() + lamports = int(round(float(sol) * config.LAMPORTS_PER_SOL)) + broker = LiveBroker() + print(f"swap-smoke: wallet {broker.pubkey}: {sol} SOL -> USDC -> SOL") + with connect() as conn: + r1 = broker.swap(conn, decision_id=None, book="live", side="buy", mint=config.USDC_MINT, amount_in=lamports, + slippage_bps=config.SLIPPAGE_ENTRY_BPS, max_slippage_bps=config.MAX_SLIPPAGE_ENTRY_BPS, + step_bps=config.SLIPPAGE_STEP_BPS) + print(f"leg 1 (buy USDC): {r1}") + net = r1.lamports_delta + if r1.ok and r1.token_delta > 0: + r2 = broker.swap(conn, decision_id=None, book="live", side="sell", mint=config.USDC_MINT, + amount_in=r1.token_delta, slippage_bps=config.SLIPPAGE_EXIT_BPS, + max_slippage_bps=config.SLIPPAGE_FORCED_BPS, step_bps=config.SLIPPAGE_STEP_BPS) + print(f"leg 2 (sell USDC): {r2}") + net += r2.lamports_delta + else: + print("leg 2 skipped: leg 1 was not a verified fill") + print(f"net lamports delta: {net} ({net / config.LAMPORTS_PER_SOL:.9f} SOL)") + + +def close_empty_atas(rpc=None, keypair=None, url: str | None = None, batch: int = CLOSE_ATA_BATCH) -> dict: + """Close every zero-balance token account owned by the bot (rent back to the wallet). + SPL Token / Token-2022 CloseAccount = instruction index 9, accounts [account, destination, owner]. + One v0 transaction per ``batch`` accounts; each is recorded as wallet_events(kind='ata_closed').""" + assert_signing_allowed() + kp = keypair or load_keypair() + owner = kp.pubkey() + owner_s = str(owner) + rpc = rpc or HttpSolanaRpc() + empties = [a for a in rpc.get_token_accounts_by_owner(owner_s) if int(a["amount"]) == 0] + result = {"pubkey": owner_s, "candidates": len(empties), "closed": 0, "signatures": [], "failed": []} + if not empties: + return result + for i in range(0, len(empties), batch): + chunk = empties[i:i + batch] + accounts = [a["address"] for a in chunk] + ixs = [Instruction(Pubkey.from_string(a["program"]), bytes([9]), + [AccountMeta(Pubkey.from_string(a["address"]), False, True), + AccountMeta(owner, False, True), + AccountMeta(owner, True, False)]) for a in chunk] + bh = rpc.get_latest_blockhash() + msg = MessageV0.try_compile(owner, ixs, [], Hash.from_string(bh["blockhash"])) + tx = VersionedTransaction(msg, [kp]) + tx_b64 = base64.b64encode(bytes(tx)).decode() + try: + sig = rpc.send_transaction(tx_b64) + except Exception as e: + err = f"{type(e).__name__}: {scrub(str(e))}" + result["failed"].append({"accounts": accounts, "error": err}) + record_event("error", "close_empty_atas", "close transaction rejected", {"accounts": accounts, "error": err}) + continue + status, _st = await_confirmation(rpc, sig, bh["last_valid_block_height"]) + detail = {"accounts": accounts, "signature": sig, "status": status} + try: + with transaction(url) as conn: + conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('ata_closed', %s, %s)", + (owner_s, Jsonb(detail))) + except Exception as e: + log.warning("wallet_events insert failed: %s", type(e).__name__) + if status == "confirmed": + result["closed"] += len(chunk) + result["signatures"].append(sig) + else: + result["failed"].append(detail) + log.info("close_empty_atas: closed %d of %d (%d tx)", result["closed"], len(empties), len(result["signatures"])) + return result diff --git a/fly_trader/execution/broker_paper.py b/fly_trader/execution/broker_paper.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c73969c9e4fe8f782e1fcd8c5a93a924733a6c --- /dev/null +++ b/fly_trader/execution/broker_paper.py @@ -0,0 +1,97 @@ +"""PaperBroker: the same decisions executed at model prices. + +Fill price = last tape price; cost = the real fees (pool fee by market cap, Jupiter 10 bps, network fee) + constant-product impact +of the position size against the pool's quote reserve (market/exit_cost.py). Deviation from the plan's +"first tape price ≥ 2 s after the decision" rule: fills are immediate at the last price; the +live-vs-mirror gap measures the difference. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone + +from .. import config +from ..market.exit_cost import exit_cost_fraction, fee_fraction, impact_fraction +from . import ledger + +log = logging.getLogger(__name__) + + +@dataclass +class PaperFill: + ok: bool + position_id: int | None + fill_id: int | None + price: float + sol_delta: float + token_delta: int + fee_sol: float + reason: str = "" + + +class PaperBroker: + def __init__(self, book: str): + assert book in ("paper_selector", "paper_fly") or book.startswith("replay_"), book # the race books; replay_* for tests + self.book = book + + def buy(self, conn, *, decision_id: int | None, mint: str, pool: str | None, size_sol: float, price: float | None, + res_quote_sol: float | None, mcap_sol: float | None, decimals: int, program_label: str | None, pool_fee: float | None = None, + ts: datetime | None = None, hold_s: float | None = None, strategy: str | None = None) -> PaperFill: + """Fill at ``price`` pushed up by the constant-product impact, paying the real fees (market/exit_cost.py): the pool + fee the stream reports (``pool_fee``) or the schedule's rate for ``mcap_sol``, Jupiter's platform fee, the network fee.""" + ts = ts or datetime.now(timezone.utc) + if not price or price <= 0: + return PaperFill(False, None, None, 0.0, 0.0, 0, 0.0, "no price") + if res_quote_sol is None or res_quote_sol <= 0: + return PaperFill(False, None, None, price, 0.0, 0, 0.0, "no liquidity estimate") + fee_frac = fee_fraction(size_sol, mcap_sol, pool_fee) + impact = impact_fraction(size_sol, res_quote_sol, program_label) + if impact > config.EXECUTABILITY_MAX_IMPACT: + # a router would not fill this either; refuse like the live executability check + return PaperFill(False, None, None, price, 0.0, 0, 0.0, f"impact {impact:.2f} > {config.EXECUTABILITY_MAX_IMPACT}") + eff_price = price * (1.0 + impact) # buying pushes the price up against us + tokens = size_sol * (1.0 - fee_frac) / eff_price + qty_raw = int(tokens * (10 ** decimals)) + if qty_raw <= 0: + return PaperFill(False, None, None, price, 0.0, 0, 0.0, "zero quantity") + fee_sol = size_sol * fee_frac + pid = ledger.open_position(conn, book=self.book, mint=mint, pool=pool, qty_raw=qty_raw, cost_sol=size_sol, + entry_price=eff_price, decision_id=decision_id, fees_sol=fee_sol, ts=ts, hold_s=hold_s, strategy=strategy) + fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=mint, side="buy", token_delta=qty_raw, + sol_delta_lamports=-int(size_sol * config.LAMPORTS_PER_SOL), price_sol=eff_price, + fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts) + return PaperFill(True, pid, fid, eff_price, -size_sol, qty_raw, fee_sol) + + def sell(self, conn, *, position: dict, decision_id: int | None, price: float | None, res_quote_sol: float | None, + mcap_sol: float | None, program_label: str | None, forced_kind: str | None, pool_fee: float | None = None, + ts: datetime | None = None, fraction: float = 1.0) -> PaperFill: + ts = ts or datetime.now(timezone.utc) + price = price or float(position.get("last_mark_price") or position.get("entry_price") or 0.0) + fraction = float(min(1.0, max(0.0, fraction))) + qty_raw = int(int(position["qty"]) * fraction) if fraction < 0.999 else int(position["qty"]) + decimals = int(position.get("decimals") or 6) + gross = qty_raw / (10 ** decimals) * price + c = exit_cost_fraction(gross, res_quote_sol, mcap_sol, program_label, pool_fee) + proceeds = gross * (1.0 - c) + fee_sol = gross * fee_fraction(gross, mcap_sol, pool_fee) + if fraction < 0.999: # partial: shrink the position, realise the sold part + realized = ledger.realize_partial(conn, position_id=int(position["id"]), qty_raw=qty_raw, + cost_part=float(position["cost_sol"]) * fraction, proceeds_sol=proceeds, + fees_sol=fee_sol, price=price, ts=ts) + if realized is None: + return PaperFill(False, int(position["id"]), None, price, 0.0, 0, 0.0, "position not open") + fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=position["mint"], side="sell", token_delta=-qty_raw, + sol_delta_lamports=int(proceeds * config.LAMPORTS_PER_SOL), price_sol=price * (1.0 - c), + fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts) + return PaperFill(True, int(position["id"]), fid, price * (1.0 - c), proceeds, -qty_raw, fee_sol, reason=f"partial {fraction:.2f} realized {realized:+.5f} SOL") + realized = ledger.close_position(conn, position_id=int(position["id"]), exit_price=price * (1.0 - c), + proceeds_sol=proceeds, fees_sol=fee_sol, decision_id=decision_id, + forced_kind=forced_kind, ts=ts) + if realized is None: + return PaperFill(False, int(position["id"]), None, price, 0.0, 0, 0.0, "position not open") + fid = ledger.record_fill(conn, order_id=None, book=self.book, mint=position["mint"], side="sell", token_delta=-qty_raw, + sol_delta_lamports=int(proceeds * config.LAMPORTS_PER_SOL), price_sol=price * (1.0 - c), + fee_lamports=int(fee_sol * config.LAMPORTS_PER_SOL), verified_by="model", ts=ts) + return PaperFill(True, int(position["id"]), fid, price * (1.0 - c), proceeds, -qty_raw, fee_sol, + reason=f"realized {realized:+.5f} SOL") diff --git a/fly_trader/execution/ledger.py b/fly_trader/execution/ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..8d4a5b15f30491a977aca6ad0a7409bf0f33cb68 --- /dev/null +++ b/fly_trader/execution/ledger.py @@ -0,0 +1,147 @@ +"""Positions and fills bookkeeping per book (live, paper_selector). + +Paper cash is derived, never stored: cash = CAPITAL_SOL + Σ realized_sol(all) − Σ cost_sol(open). +realized_sol accumulates per position: each partial sale books proceeds − the cost it retires (cost_sol +shrinks by that part) and the close adds proceeds − the remaining cost, so an open position may carry +realized P&L from partial sales. +Live cash is the on-chain SOL balance. Every open/close writes positions rows; every fill writes a +fills row (verified_by = 'balance_delta' for live, 'model' for paper). +""" +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone + +from .. import config + +log = logging.getLogger(__name__) + + +def open_positions(conn, book: str) -> list[dict]: + rows = conn.execute( + """SELECT p.*, t.decimals, t.graduated_at, wp.program_label, + COALESCE(p.hold_s, (d.detail->>'hold_s')::float) AS hold_s, COALESCE(p.strategy, d.detail->>'strategy') AS strategy + FROM positions p LEFT JOIN tokens t ON t.mint = p.mint + LEFT JOIN watch_pools wp ON wp.pool = p.pool + LEFT JOIN decisions d ON d.id = p.entry_decision_id + WHERE p.book = %s AND p.status = 'open' ORDER BY p.opened_at""", + (book,), + ).fetchall() + return [dict(r) for r in rows] + + +def paper_cash(conn, book: str) -> float: + r = conn.execute( + """SELECT COALESCE(sum(realized_sol), 0) AS realized, + COALESCE(sum(cost_sol) FILTER (WHERE status = 'open'), 0) AS open_cost + FROM positions WHERE book = %s""", + (book,), + ).fetchone() + return config.CAPITAL_SOL + float(r["realized"]) - float(r["open_cost"]) + + +def record_fill(conn, *, order_id: int | None, book: str, mint: str, side: str, token_delta: int, + sol_delta_lamports: int, price_sol: float, fee_lamports: int, verified_by: str, + signature: str | None = None, slot: int | None = None, ts: datetime | None = None, + pre: dict | None = None, post: dict | None = None) -> int: + row = conn.execute( + """INSERT INTO fills (order_id, book, ts, signature, slot, mint, side, token_delta, sol_delta_lamports, + price_sol, fee_lamports, verified_by, pre_snapshot, post_snapshot) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING id""", + (order_id, book, ts or datetime.now(timezone.utc), signature, slot, mint, side, token_delta, sol_delta_lamports, + price_sol, fee_lamports, verified_by, json.dumps(pre) if pre else None, json.dumps(post) if post else None), + ).fetchone() + return int(row["id"]) + + +def open_position(conn, *, book: str, mint: str, pool: str | None, qty_raw: int, cost_sol: float, + entry_price: float, decision_id: int | None, fees_sol: float, ts: datetime | None = None, + hold_s: float | None = None, strategy: str | None = None) -> int: + ts = ts or datetime.now(timezone.utc) + existing = conn.execute("SELECT id, qty, cost_sol, entry_price FROM positions WHERE book=%s AND mint=%s AND status='open'", + (book, mint)).fetchone() + if existing: # add to an open position: average the entry + q0, c0 = int(existing["qty"]), float(existing["cost_sol"]) + q1, c1 = q0 + qty_raw, c0 + cost_sol + p1 = (float(existing["entry_price"] or entry_price) * q0 + entry_price * qty_raw) / max(q1, 1) + conn.execute("UPDATE positions SET qty=%s, cost_sol=%s, entry_price=%s, fees_sol=fees_sol+%s, last_mark_price=%s, last_mark_ts=%s WHERE id=%s", + (q1, c1, p1, fees_sol, entry_price, ts, existing["id"])) + return int(existing["id"]) + row = conn.execute( + """INSERT INTO positions (book, mint, pool, opened_at, entry_decision_id, qty, cost_sol, entry_price, peak_price, + last_mark_price, last_mark_ts, fees_sol, status, last_swap_ts, hold_s, strategy) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'open',%s,%s,%s) RETURNING id""", + (book, mint, pool, ts, decision_id, qty_raw, cost_sol, entry_price, entry_price, entry_price, ts, fees_sol, ts, hold_s, strategy), + ).fetchone() + return int(row["id"]) + + +def realize_partial(conn, *, position_id: int, qty_raw: int, cost_part: float, proceeds_sol: float, fees_sol: float, + price: float, ts: datetime | None = None) -> float | None: + """Shrink an open position by qty_raw / cost_part and book proceeds − cost_part into realized_sol. + Returns this sale's realized P&L, or None when the position is no longer open (nothing booked).""" + ts = ts or datetime.now(timezone.utc) + realized = proceeds_sol - cost_part + r = conn.execute( + """UPDATE positions SET qty = qty - %s, cost_sol = cost_sol - %s, realized_sol = COALESCE(realized_sol, 0) + %s, + fees_sol = fees_sol + %s, last_mark_price=%s, last_mark_ts=%s WHERE id=%s AND status='open' RETURNING id""", + (qty_raw, cost_part, realized, fees_sol, price, ts, position_id)).fetchone() + return realized if r else None + + +def close_position(conn, *, position_id: int, exit_price: float, proceeds_sol: float, fees_sol: float, + decision_id: int | None, forced_kind: str | None, ts: datetime | None = None) -> float | None: + """Close an open position; realized_sol accumulates (earlier partial sales + proceeds − remaining cost). + Returns this sale's realized P&L, or None when the position was not open (already closed: nothing booked).""" + ts = ts or datetime.now(timezone.utc) + r = conn.execute( + """UPDATE positions SET status='closed', closed_at=%s, exit_decision_id=%s, exit_price=%s, + realized_sol=COALESCE(realized_sol, 0) + %s - cost_sol, fees_sol=fees_sol+%s, forced_exit_kind=%s, + last_mark_price=%s, last_mark_ts=%s WHERE id=%s AND status='open' RETURNING %s - cost_sol AS realized""", + (ts, decision_id, exit_price, proceeds_sol, fees_sol, forced_kind, exit_price, ts, position_id, proceeds_sol), + ).fetchone() + if r is None: + log.warning("close_position: position %s is not open; nothing booked", position_id) + return None + return float(r["realized"]) + + +def mark_positions(conn, book: str, prices: dict[str, float], last_swaps: dict[str, float], ts: datetime, + dt_s: float = 0.0) -> None: + """Mark to the last price; track the peak; accrue satiety = Σ max(unrealized return, 0) × dt while in profit.""" + rows = conn.execute("SELECT id, mint, peak_price, entry_price, satiety FROM positions WHERE book=%s AND status='open'", (book,)).fetchall() + for r in rows: + p = prices.get(r["mint"]) + if p is None: + continue + peak = max(float(r["peak_price"] or 0.0), p) + entry = float(r["entry_price"] or 0.0) + u = (p / entry - 1.0) if entry > 0 else 0.0 + sat = float(r["satiety"] or 0.0) + max(u, 0.0) * dt_s + ls = last_swaps.get(r["mint"]) + conn.execute("UPDATE positions SET last_mark_price=%s, last_mark_ts=%s, peak_price=%s, satiety=%s, " + "last_swap_ts=COALESCE(%s, last_swap_ts) WHERE id=%s", + (p, ts, peak, sat, datetime.fromtimestamp(ls, tz=timezone.utc) if ls else None, r["id"])) + + +def verify_fills() -> None: + """Reconcile live positions against on-chain token balances; prints mismatches, writes wallet_events.""" + from ..chain import keys + from ..chain.rpc import HttpSolanaRpc + from ..db.connection import transaction + rpc = HttpSolanaRpc() + pub = keys.bot_pubkey() + accounts = rpc.get_token_accounts_by_owner(pub) + onchain = {a["mint"]: int(a["amount"]) for a in accounts} + with transaction() as conn: + pos = open_positions(conn, "live") + mismatches = [] + for p in pos: + have = onchain.get(p["mint"], 0) + if abs(have - int(p["qty"])) > max(1, int(p["qty"]) // 1000): + mismatches.append({"mint": p["mint"], "db_qty": int(p["qty"]), "onchain": have}) + stray = [m for m, a in onchain.items() if a > 0 and m not in {p["mint"] for p in pos} and m != config.WSOL_MINT] + conn.execute("INSERT INTO wallet_events (kind, pubkey, detail) VALUES ('verify_fills', %s, %s)", + (pub, json.dumps({"mismatches": mismatches, "stray_mints": stray, "n_positions": len(pos)}))) + print(json.dumps({"positions": len(pos), "mismatches": mismatches, "stray_mints": stray}, indent=1)) diff --git a/fly_trader/execution/worker.py b/fly_trader/execution/worker.py new file mode 100644 index 0000000000000000000000000000000000000000..204932315b59f49f221f3551fc47dfe35b4093d7 --- /dev/null +++ b/fly_trader/execution/worker.py @@ -0,0 +1,143 @@ +"""Execution queue for the live book: the caller never blocks on Jupiter. + +The caller enqueues ExecRequest objects; one background thread executes them sequentially through +LiveBroker (one in-flight order at a time, balance-verified); results are drained by the caller, which +opens/closes positions accordingly. Mints with an in-flight request are skipped. +""" +from __future__ import annotations + +import logging +import queue +import threading +import traceback +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from .. import config +from ..db.connection import transaction +from ..agent import rails +from . import ledger + +log = logging.getLogger(__name__) + + +@dataclass +class ExecRequest: + decision_id: int | None + mint: str + pool: str | None + side: str # buy | sell + amount_in: int # lamports (buy) or raw token units (sell) + slippage_bps: int + max_slippage_bps: int + decimals: int + forced_kind: str | None = None + position_id: int | None = None + size_sol: float = 0.0 + enqueued_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + +@dataclass +class ExecResult: + request: ExecRequest + ok: bool + fill_id: int | None = None + signature: str | None = None + token_delta: int = 0 + lamports_delta: int = 0 + price_sol: float | None = None + error: str | None = None + position_id: int | None = None + realized_sol: float | None = None + + +class ExecutionWorker: + def __init__(self, broker): + self.broker = broker + self.q: queue.Queue[ExecRequest] = queue.Queue() + self.results: queue.Queue[ExecResult] = queue.Queue() + self.inflight: set[str] = set() + self.lock = threading.Lock() + self.thread = threading.Thread(target=self._loop, name="runner-exec", daemon=True) # routed to the runner's log + self.stop_flag = False + self.thread.start() + + def submit(self, req: ExecRequest) -> bool: + with self.lock: + if req.mint in self.inflight: + return False + self.inflight.add(req.mint) + self.q.put(req) + return True + + def pending(self) -> set[str]: + with self.lock: + return set(self.inflight) + + def drain(self) -> list[ExecResult]: + out = [] + while True: + try: + out.append(self.results.get_nowait()) + except queue.Empty: + break + return out + + def stop(self) -> None: + self.stop_flag = True + + def _loop(self) -> None: + while not self.stop_flag: + try: + req = self.q.get(timeout=1.0) + except queue.Empty: + continue + res = self._execute(req) + with self.lock: + self.inflight.discard(req.mint) + self.results.put(res) + + def _execute(self, req: ExecRequest) -> ExecResult: + try: + with transaction() as conn: + fr = self.broker.swap(conn, decision_id=req.decision_id, book="live", side=req.side, mint=req.mint, + amount_in=req.amount_in, slippage_bps=req.slippage_bps, + max_slippage_bps=req.max_slippage_bps, step_bps=config.SLIPPAGE_STEP_BPS) + if not fr.ok: + rails.record_failure(conn, fr.error or fr.status or "swap failed") + return ExecResult(req, False, error=fr.error or fr.status) + rails.record_success(conn) + ts = datetime.now(timezone.utc) + if req.side == "buy": + cost = -fr.lamports_delta / config.LAMPORTS_PER_SOL + price = fr.price_sol or (cost / max(fr.token_delta / (10 ** req.decimals), 1e-12)) + pid = ledger.open_position(conn, book="live", mint=req.mint, pool=req.pool, qty_raw=fr.token_delta, + cost_sol=cost, entry_price=price, decision_id=req.decision_id, + fees_sol=0.0, ts=ts) + rails.record_notional(conn, cost, "entry") + return ExecResult(req, True, fr.fill_id, fr.signature, fr.token_delta, fr.lamports_delta, price, position_id=pid) + else: + proceeds = fr.lamports_delta / config.LAMPORTS_PER_SOL + pos = conn.execute("SELECT * FROM positions WHERE id=%s FOR UPDATE", (req.position_id,)).fetchone() + price = fr.price_sol if fr.price_sol is not None else (proceeds / max(abs(fr.token_delta) / (10 ** req.decimals), 1e-12)) + remaining = int(pos["qty"]) + fr.token_delta if pos else 0 # token_delta negative + if pos is None or pos["status"] != "open": # closed elsewhere: the fill row stands, nothing is booked twice + log.warning("sell of %s filled but position %s is not open; not booked", req.mint, req.position_id) + realized = None + elif remaining > max(1, int(pos["qty"]) // 200): # partial fill: keep the remainder open + realized = ledger.realize_partial(conn, position_id=req.position_id, qty_raw=-fr.token_delta, + cost_part=float(pos["cost_sol"]) * (1 - remaining / max(int(pos["qty"]), 1)), + proceeds_sol=proceeds, fees_sol=0.0, price=price, ts=ts) + else: + realized = ledger.close_position(conn, position_id=req.position_id, exit_price=price, proceeds_sol=proceeds, + fees_sol=0.0, decision_id=req.decision_id, forced_kind=req.forced_kind, ts=ts) + return ExecResult(req, True, fr.fill_id, fr.signature, fr.token_delta, fr.lamports_delta, price, + position_id=req.position_id, realized_sol=realized) + except Exception as e: + log.error("execution error for %s: %s\n%s", req.mint, e, traceback.format_exc()) + try: + with transaction() as conn: + rails.record_failure(conn, f"{type(e).__name__}: {e}") + except Exception: + pass + return ExecResult(req, False, error=f"{type(e).__name__}: {e}") diff --git a/fly_trader/ingest/__init__.py b/fly_trader/ingest/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/ingest/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/ingest/corpus_pull.py b/fly_trader/ingest/corpus_pull.py new file mode 100644 index 0000000000000000000000000000000000000000..bbc105296308d949bdd1c460f2b926db7b0d40b5 --- /dev/null +++ b/fly_trader/ingest/corpus_pull.py @@ -0,0 +1,439 @@ +"""Historical corpus puller (Supervisor thread ``corpus``). + +Enumerates every pump.fun graduation by paging the migration authority's succeeded transactions through +Helius ``getTransactionsForAddress`` (newest first, back to ``CORPUS_DAYS``), then, per token and newest +first, pulls from pump.fun's own swap-api: + +* candles in SOL: 1-minute for the first ``CORPUS_CANDLE_1M_H`` hours after graduation (plus the last 30 + minutes of the bonding curve), then 5-minute until ``CORPUS_CANDLE_5M_D`` days (2–4 requests per token); +* tokens are pulled only once they are ``CORPUS_MIN_AGE_H`` old, so the first hours are complete; +* per-trade rows (wallet, side, SOL, tokens, price) for the first ``CORPUS_TRADES_H`` hours, for a deterministic + ``CORPUS_TRADES_SAMPLE`` share of the tokens that kept trading at least ``CORPUS_MIN_LIFE_H`` and graduated within + ``CORPUS_TRADES_DAYS`` (trades cost up to 100 pages per token; candles cost one to three). + +Registry: ``corpus_tokens``. Data: ``data/corpus/candles/.parquet`` and ``data/corpus/trades/.parquet`` +(append-only: a token already on disk is never rewritten). Progress: ``ui_settings['corpus_status']``. +The swap-api throttles bursts (Cloudflare 1015 after ~20 requests at 1/s), so requests are paced adaptively. +Measured 2026-09-13: ~36 graduations/hour; candle volume in SOL matches the tape within 0.5 %. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import httpx +import psycopg +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup + +log = logging.getLogger(__name__) +WSOL = "So11111111111111111111111111111111111111112" +PUMP_PROGRAM = "6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P" +SLOTS_PER_S = 2.5 +RETRY_ERROR_H = 6.0 # a token that failed transiently (retries exhausted on 5xx/429, network, disk) is pulled again after this +CANDLE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("interval", pa.string()), ("open", pa.float64()), + ("high", pa.float64()), ("low", pa.float64()), ("close", pa.float64()), ("volume_sol", pa.float64())]) +TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot_index", pa.string()), ("tx", pa.string()), ("wallet", pa.string()), + ("side", pa.int8()), ("program", pa.string()), ("price_sol", pa.float64()), ("sol", pa.float64()), ("tokens", pa.float64())]) + + +class Throttled(Exception): + pass + + +class Permanent(RuntimeError): + """A 4xx from the swap-api: the token's error row is never requeued.""" + + +class Status: + """ui_settings['corpus_status'] writer (throttled to one write per 3 s unless forced).""" + + def __init__(self): + self.s: dict = {"started_at": datetime.now(timezone.utc).isoformat(), "req_ok": 0, "req_429": 0, "candles_rows": 0, "trades_rows": 0, + "tokens_pulled": 0, "with_trades": 0} + self.lock = threading.Lock(); self.last = 0.0; self.t0 = time.time() + + def update(self, force: bool = False, **kv) -> None: + with self.lock: + self.s.update(kv); self.s["updated_at"] = datetime.now(timezone.utc).isoformat() + hours = (time.time() - self.t0) / 3600 + self.s["tokens_per_h"] = self.s["tokens_pulled"] / hours if hours > 0.01 else 0.0 + if not force and time.time() - self.last < 3.0: + return + self.last = time.time(); payload = dict(self.s) + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('corpus_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(payload, default=str),)) + except Exception: + log.debug("corpus status write failed", exc_info=True) + + def bump(self, key: str, n: int = 1) -> None: + with self.lock: + self.s[key] = self.s.get(key, 0) + n + + +class Pacer: + """Adaptive spacing between swap-api requests: back off 0.5 s per throttle, creep back after 200 clean calls.""" + + def __init__(self, pace_s: float, floor_s: float): + self.pace = pace_s; self.floor = floor_s; self.last = 0.0; self.clean = 0 + + def wait(self, stop: threading.Event | None) -> None: + dt = time.time() - self.last + if dt < self.pace: + _sleep(self.pace - dt, stop) + self.last = time.time() + + def ok(self) -> None: + self.clean += 1 + if self.clean >= 200: + self.clean = 0; self.pace = max(self.floor, self.pace - 0.1) + + def throttled(self, stop: threading.Event | None, retry_after: float | None) -> None: + self.clean = 0; self.pace = min(10.0, self.pace + 0.5) + _sleep(max(60.0, retry_after or 0.0), stop) + + +def _sleep(seconds: float, stop: threading.Event | None) -> None: + end = time.time() + seconds + while time.time() < end: + if stop is not None and stop.is_set(): + return + time.sleep(min(1.0, end - time.time())) + + +def _db_retry(stop: threading.Event | None, what: str, fn, *args): + """``fn(*args)``, riding out Postgres outages (log, back off 5 s → 60 s, retry); None once ``stop`` is set.""" + delay = 5.0 + while not (stop is not None and stop.is_set()): + try: + return fn(*args) + except psycopg.Error as e: + log.warning("%s failed (%s: %s); retrying in %.0fs", what, type(e).__name__, str(e)[:160], delay) + _sleep(delay, stop); delay = min(60.0, delay * 2) + return None + + +class SwapApi: + def __init__(self, pacer: Pacer, status: Status, stop: threading.Event | None): + self.c = httpx.Client(base_url=config.CORPUS_SWAP_API, headers={"User-Agent": "Mozilla/5.0", "Accept": "application/json"}, timeout=30) + self.pacer, self.status, self.stop = pacer, status, stop + + def get(self, path: str, params: dict): + for attempt in range(6): + if self.stop is not None and self.stop.is_set(): + raise InterruptedError + self.pacer.wait(self.stop) + try: + r = self.c.get(path, params=params) + except httpx.HTTPError as e: + log.warning("swap-api %s: %s", path, e); _sleep(5.0 * (attempt + 1), self.stop); continue + if r.status_code == 200: + self.pacer.ok(); self.status.bump("req_ok"); return r.json() + if r.status_code == 429: + self.status.bump("req_429"); ra = r.headers.get("retry-after") + log.info("swap-api throttled; pace -> %.1f s", self.pacer.pace + 0.5) + self.pacer.throttled(self.stop, float(ra) if ra and ra.isdigit() else None); continue + if r.status_code >= 500: + _sleep(5.0 * (attempt + 1), self.stop); continue + raise (Permanent if 400 <= r.status_code < 500 else RuntimeError)(f"swap-api {path} HTTP {r.status_code}: {r.text[:160]}") + raise RuntimeError(f"swap-api {path}: gave up after retries") + + def candles(self, mint: str, interval: str, created_ms: int, before_ms: int | None = None) -> list[dict]: + p = {"interval": interval, "limit": 1000, "createdTs": created_ms, "currency": "SOL"} + if before_ms is not None: + p["beforeTs"] = before_ms + j = self.get(f"/v2/coins/{mint}/candles", p) + return j if isinstance(j, list) else (j.get("candles") or []) + + def trades(self, mint: str, cursor: str | None) -> tuple[list[dict], str | None]: + p = {"limit": 100} + if cursor: + p["cursor"] = cursor + j = self.get(f"/v2/coins/{mint}/trades", p) + return j.get("trades") or [], (j.get("pagination") or {}).get("nextCursor") + + +class Helius: + def __init__(self, stop: threading.Event | None): + if not config.HELIUS_API_KEY: + raise RuntimeError("HELIUS_API_KEY missing") + self.c = httpx.Client(timeout=120); self.stop = stop; self.last = 0.0 + + def gtfa(self, address: str, opts: dict) -> dict: + for attempt in range(5): + dt = time.time() - self.last + if dt < 0.3: + time.sleep(0.3 - dt) + self.last = time.time() + try: + r = self.c.post(f"{config.HELIUS_HTTP}?api-key={config.HELIUS_API_KEY}", + json={"jsonrpc": "2.0", "id": 1, "method": "getTransactionsForAddress", "params": [address, opts]}) + j = r.json() + except (httpx.HTTPError, ValueError) as e: + log.warning("helius gTFA: %s", e); _sleep(3.0 * (attempt + 1), self.stop); continue + if "result" in j and j["result"] is not None: + return j["result"] + err = (j.get("error") or {}).get("message", "unknown error") + if "rate" in err.lower() or r.status_code == 429: + _sleep(5.0 * (attempt + 1), self.stop); continue + raise RuntimeError(f"helius gTFA: {err}") + raise RuntimeError("helius gTFA: gave up after retries") + + +# ---------------------------------------------------------------- enumeration +def _graduations_from_page(data: list[dict]) -> list[tuple[str, int, str, int]]: + """(mint, blockTime, signature, slot) for every succeeded migration transaction on the page.""" + out = [] + for tx in data: + msg = tx["transaction"]["message"]; keys = msg.get("accountKeys") or [] + progs = set() + for ins in msg.get("instructions") or []: + pid = ins.get("programId") + if pid is None and "programIdIndex" in ins and ins["programIdIndex"] < len(keys): + k = keys[ins["programIdIndex"]]; pid = k["pubkey"] if isinstance(k, dict) else k + progs.add(pid) + if PUMP_PROGRAM not in progs: + continue + mints = {b.get("mint") for b in ((tx.get("meta") or {}).get("postTokenBalances") or [])} - {WSOL, None} + if len(mints) != 1: + continue + sig = tx["transaction"]["signatures"][0] if tx["transaction"].get("signatures") else None + out.append((mints.pop(), int(tx["blockTime"]), sig, int(tx["slot"]))) + return out + + +def _upsert_graduations(rows: list[tuple[str, int, str, int]]) -> int: + if not rows: + return 0 + with transaction() as conn: + before = conn.execute("SELECT count(*) AS n FROM corpus_tokens").fetchone()["n"] + conn.cursor().executemany( + "INSERT INTO corpus_tokens (mint, graduated_at, grad_sig, grad_slot) VALUES (%s, to_timestamp(%s), %s, %s) " + "ON CONFLICT (mint) DO UPDATE SET graduated_at = LEAST(corpus_tokens.graduated_at, EXCLUDED.graduated_at), " + "grad_slot = LEAST(corpus_tokens.grad_slot, EXCLUDED.grad_slot), updated_at = now()", rows) + after = conn.execute("SELECT count(*) AS n FROM corpus_tokens").fetchone()["n"] + return int(after - before) + + +def _enum_state() -> dict: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'corpus_enum'").fetchone() + v = r["value"] if r else {} + return v if isinstance(v, dict) else json.loads(v or "{}") + + +def _save_enum_state(state: dict) -> None: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('corpus_enum', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(state, default=str),)) + + +def enumerate_graduations(hel: Helius, status: Status, stop: threading.Event | None, max_pages: int = 20) -> int: + """One enumeration round: (1) newest pages until a page adds nothing new, (2) up to ``max_pages`` of backfill + from the persisted pagination token until ``CORPUS_DAYS`` is reached. Returns rows added.""" + addr = config.CORPUS_MIGRATION_AUTHORITY + pull_before = datetime.fromisoformat(config.CORPUS_PULL_BEFORE).replace(tzinfo=timezone.utc) + horizon = int(min(datetime.now(timezone.utc) - timedelta(days=config.CORPUS_DAYS), pull_before - timedelta(days=30)).timestamp()) + state = _enum_state(); added = 0 + # (1) incremental: newest first, stop when a full page adds nothing + token = None + for _ in range(500): # keeps walking after an outage until a page adds nothing + res = hel.gtfa(addr, {"limit": 100, "transactionDetails": "full", "sortOrder": "desc", "filters": {"status": "succeeded"}, + **({"paginationToken": token} if token else {})}) + rows = _graduations_from_page(res.get("data") or []); n = _upsert_graduations(rows); added += n + token = res.get("paginationToken") + status.update(stage="enumerating (new graduations)", enum_added=added) + if n == 0 or not token or (stop is not None and stop.is_set()): + break + # (2) backfill toward the horizon from where the last round stopped + if not state.get("backfill_done"): + token = state.get("token"); oldest = state.get("oldest_bt") + if token is None and oldest is None: + # first round: start a fresh newest-first walk that will proceed past the incremental pages + token = None + for _ in range(max_pages): + if stop is not None and stop.is_set(): + break + res = hel.gtfa(addr, {"limit": 100, "transactionDetails": "full", "sortOrder": "desc", "filters": {"status": "succeeded"}, + **({"paginationToken": token} if token else {})}) + data = res.get("data") or [] + rows = _graduations_from_page(data); added += _upsert_graduations(rows) + token = res.get("paginationToken") + bts = [int(t["blockTime"]) for t in data if t.get("blockTime")] + if bts: + oldest = min(bts) + state.update({"token": token, "oldest_bt": oldest}) + status.update(stage="enumerating (backfill)", backfill_through=datetime.fromtimestamp(oldest, timezone.utc).isoformat() if oldest else None, + backfill_target=datetime.fromtimestamp(horizon, timezone.utc).date().isoformat(), enum_added=added) + if not token or (oldest is not None and oldest < horizon): + state["backfill_done"] = True; break + _save_enum_state(state) + status.update(backfill_done=bool(state.get("backfill_done")), force=True) + return added + + +# ---------------------------------------------------------------- per-token pull +def _write_atomic(table: pa.Table, path: Path) -> None: + tmp = path.with_name(path.name + ".tmp") + pq.write_table(table, tmp, compression="zstd"); tmp.replace(path) + + +def _candle_rows(cands: list[dict], interval: str) -> list[dict]: + return [{"ts": datetime.fromtimestamp(int(c["timestamp"]) / 1000, timezone.utc), "interval": interval, "open": float(c["open"]), + "high": float(c["high"]), "low": float(c["low"]), "close": float(c["close"]), "volume_sol": float(c["volume"])} for c in cands] + + +def _page_back(api: SwapApi, mint: str, interval: str, created_ms: int, before_ms: int, until_ms: int, max_pages: int) -> list[dict]: + """Candles with timestamp in [until_ms, before_ms), paging backwards.""" + out, before = [], before_ms + for _ in range(max_pages): + page = api.candles(mint, interval, created_ms, before) + if not page: + break + out.extend(c for c in page if until_ms <= int(c["timestamp"]) < before_ms) + lo = min(int(c["timestamp"]) for c in page) + if lo <= until_ms or len(page) < 1000: + break + before = lo + return out + + +def pull_token(api: SwapApi, row: dict, status: Status) -> dict: + """Two to four swap-api requests per token: 1-minute candles for the first CORPUS_CANDLE_1M_H hours (plus the last + 30 curve minutes), then the newest 5-minute page (which also tells how long the token lived), paged back to the + 1-minute window and capped at CORPUS_CANDLE_5M_D days. Trades (first CORPUS_TRADES_H hours) for a deterministic + CORPUS_TRADES_SAMPLE share of tokens that lived ≥ CORPUS_MIN_LIFE_H and graduated within CORPUS_TRADES_DAYS.""" + mint: str = row["mint"]; grad: datetime = row["graduated_at"] + g_ms = int(grad.timestamp() * 1000); created_ms = g_ms - 86_400_000 + cdir = config.CORPUS_DIR / "candles"; tdir = config.CORPUS_DIR / "trades" + cpath, tpath = cdir / f"{mint}.parquet", tdir / f"{mint}.parquet" + upd: dict = {"status": "done", "candles_1m": 0, "candles_5m": 0, "trades": 0, "trades_through": None, "candle_path": None, "trade_path": None} + start_ms = g_ms - 30 * 60_000; win1_end = g_ms + int(config.CORPUS_CANDLE_1M_H * 3.6e6); win2_end = g_ms + int(config.CORPUS_CANDLE_5M_D * 86_400_000) + one_m = _page_back(api, mint, "1m", created_ms, win1_end, start_ms, max_pages=3) + newest5 = api.candles(mint, "5m", created_ms) + stamps = [int(c["timestamp"]) for c in one_m] + [int(c["timestamp"]) for c in newest5] + if not stamps: + upd["status"] = "empty"; return upd + last_ms = max(stamps); life_h = (last_ms - g_ms) / 3.6e6; upd["life_h"] = life_h + five_m = [c for c in newest5 if win1_end <= int(c["timestamp"]) < win2_end] + if newest5 and len(newest5) >= 1000 and min(int(c["timestamp"]) for c in newest5) > win1_end: + lo5 = min(int(c["timestamp"]) for c in newest5) + pages = min(20, int((min(lo5, win2_end) - win1_end) / (1000 * 300_000)) + 2) # enough pages to reach the 1-minute window's end + five_m += _page_back(api, mint, "5m", created_ms, min(lo5, win2_end), win1_end, max_pages=pages) + five_m = [c for c in five_m if int(c["timestamp"]) < win2_end] + rows = _candle_rows(one_m, "1m") + _candle_rows(five_m, "5m"); upd["candles_1m"] = len(one_m); upd["candles_5m"] = len(five_m) + if rows and not cpath.exists(): + _write_atomic(pa.Table.from_pylist(rows, schema=CANDLE_SCHEMA), cpath) + upd["candle_path"] = str(cpath) if cpath.exists() else None + status.bump("candles_rows", len(rows)) + recent = grad >= datetime.now(timezone.utc) - timedelta(days=config.CORPUS_TRADES_DAYS) + sampled = (int.from_bytes(hashlib.sha1(mint.encode()).digest()[:4], "big") / 2**32) < config.CORPUS_TRADES_SAMPLE + if life_h >= config.CORPUS_MIN_LIFE_H and recent and sampled and not tpath.exists(): + end_ms = min(g_ms + int(config.CORPUS_TRADES_H * 3.6e6), last_ms + 60_000) + slot_t = int(row["grad_slot"] or 0) + int((end_ms - g_ms) / 1000 * SLOTS_PER_S) + cursor: str | None = f"{slot_t:010d}000000000000-{end_ms}" + trows: list[dict] = []; pages = 0; oldest_seen = end_ms + while cursor and pages < config.CORPUS_TRADES_MAX_PAGES: + trades, cursor = api.trades(mint, cursor); pages += 1 + if not trades: + break + for t in trades: + ts = datetime.fromisoformat(t["timestamp"].replace("Z", "+00:00")); ms = int(ts.timestamp() * 1000) + oldest_seen = min(oldest_seen, ms) + if ms < start_ms: + continue + trows.append({"ts": ts, "slot_index": t.get("slotIndexId"), "tx": t.get("tx"), "wallet": t.get("userAddress"), + "side": 1 if t.get("type") == "buy" else -1, "program": t.get("program"), + "price_sol": float(t.get("priceSol") or 0.0), "sol": float(t.get("amountSol") or 0.0), "tokens": float(t.get("baseAmount") or 0.0)}) + if oldest_seen < start_ms: + break + if trows: + _write_atomic(pa.Table.from_pylist(trows, schema=TRADE_SCHEMA), tpath) + upd.update(trades=len(trows), trades_through=datetime.fromtimestamp(end_ms / 1000, timezone.utc), trade_path=str(tpath)) + status.bump("trades_rows", len(trows)); status.bump("with_trades") + return upd + + +def _next_pending(limit: int = 50) -> list[dict]: + """Pending tokens plus transient errors (last_error not 'permanent: …') that have cooled down RETRY_ERROR_H.""" + with transaction() as conn: + return conn.execute("SELECT mint, graduated_at, grad_slot FROM corpus_tokens WHERE (status = 'pending' OR (status = 'error' " + "AND updated_at < now() - %s * interval '1 hour' AND COALESCE(last_error, '') NOT LIKE 'permanent:%%')) " + "AND graduated_at < now() - %s * interval '1 hour' AND graduated_at < %s::timestamptz ORDER BY graduated_at DESC LIMIT %s", + (RETRY_ERROR_H, config.CORPUS_MIN_AGE_H, config.CORPUS_PULL_BEFORE, limit)).fetchall() + + +def _mark(mint: str, upd: dict) -> None: + with transaction() as conn: + conn.execute("UPDATE corpus_tokens SET status = %s, life_h = %s, candles_1m = %s, candles_5m = %s, trades = %s, trades_through = %s, " + "candle_path = %s, trade_path = %s, last_error = %s, updated_at = now() WHERE mint = %s", + (upd.get("status", "done"), upd.get("life_h"), upd.get("candles_1m"), upd.get("candles_5m"), upd.get("trades"), upd.get("trades_through"), + upd.get("candle_path"), upd.get("trade_path"), upd.get("last_error"), mint)) + + +def counts() -> dict: + with transaction() as conn: + rows = conn.execute("SELECT status, count(*) AS n FROM corpus_tokens GROUP BY status").fetchall() + return {r["status"]: int(r["n"]) for r in rows} + + +# ---------------------------------------------------------------- worker entry +def main(stop_event: threading.Event | None = None) -> None: + setup("corpus") + status = Status(); stop = stop_event + (config.CORPUS_DIR / "candles").mkdir(parents=True, exist_ok=True); (config.CORPUS_DIR / "trades").mkdir(parents=True, exist_ok=True) + pacer = Pacer(config.CORPUS_PACE_S, config.CORPUS_PACE_MIN_S); api = SwapApi(pacer, status, stop); hel = Helius(stop) + record_event("info", "corpus", "corpus puller started", {"days": config.CORPUS_DAYS, "trades_h": config.CORPUS_TRADES_H, "pace_s": pacer.pace}) + log.info("corpus puller started (days=%d)", config.CORPUS_DAYS) + from ..train import corpus_features + builder = threading.Thread(target=corpus_features.build_loop, args=(stop,), name="corpus-features", daemon=True); builder.start() + try: + while not (stop is not None and stop.is_set()): + try: + added = enumerate_graduations(hel, status, stop) + if added: + log.info("enumerated %d new graduations", added) + except InterruptedError: + break + except Exception as e: + log.exception("enumeration failed"); record_event("error", "corpus", f"enumeration failed: {type(e).__name__}: {e}") + status.update(stage="enumeration error", last_error=str(e)[:200], force=True); _sleep(30, stop) + pending = _db_retry(stop, "corpus pending query", _next_pending) + if pending is None: + break + c = _db_retry(stop, "corpus counts", counts) or {}; status.update(stage="pulling" if pending else "idle", pending=c.get("pending", 0), done=c.get("done", 0), empty=c.get("empty", 0), + errors=c.get("error", 0), enumerated=sum(c.values()), pace_s=round(pacer.pace, 2), force=True) + if not pending: + _sleep(60, stop); continue + for row in pending: + if stop is not None and stop.is_set(): + break + try: + upd = pull_token(api, row, status) + except InterruptedError: + break + except Exception as e: # a 4xx is permanent; anything else is requeued by _next_pending after RETRY_ERROR_H + perm = "permanent: " if isinstance(e, Permanent) else "" + log.warning("pull %s failed: %s%s", row["mint"], perm, e); upd = {"status": "error", "last_error": f"{perm}{type(e).__name__}: {e}"[:300]} + _db_retry(stop, "corpus mark", _mark, row["mint"], upd); status.bump("tokens_pulled") + c = _db_retry(stop, "corpus counts", counts) or {} + status.update(stage="pulling", last_mint=row["mint"], last_graduated=row["graduated_at"].isoformat(), pending=c.get("pending", 0), done=c.get("done", 0), + empty=c.get("empty", 0), errors=c.get("error", 0), enumerated=sum(c.values()), pace_s=round(pacer.pace, 2), + eta_h=(c.get("pending", 0) / status.s["tokens_per_h"]) if status.s.get("tokens_per_h") else None) + finally: + builder.join(30.0) + status.update(stage="stopped", force=True) + record_event("info", "corpus", "corpus puller stopped", {k: status.s.get(k) for k in ("tokens_pulled", "candles_rows", "trades_rows", "req_ok", "req_429")}) + log.info("corpus puller stopped") diff --git a/fly_trader/ingest/discovery.py b/fly_trader/ingest/discovery.py new file mode 100644 index 0000000000000000000000000000000000000000..cc20bfccaf983df44dede07e2809271b85925db0 --- /dev/null +++ b/fly_trader/ingest/discovery.py @@ -0,0 +1,80 @@ +"""Jupiter token stats (the `discover` worker). + +Every STATS_REFRESH_S, token_stats via /search batches (100 mints per call) for every token of the selector's universe +(traded on the stream in the last hour, pool above its gate). Jupiter serves only current values, so this is how a +point-in-time history accrues for future model inputs. +""" +from __future__ import annotations + +import json +import logging +import signal +import time + +from .. import config +from ..chain.jupiter_tokens import JupiterTokens +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup + +log = logging.getLogger(__name__) + + +def insert_stats(conn, tok: dict) -> None: + audit = tok.get("audit") or {} + stats = {k: tok.get(k) for k in ("stats5m", "stats1h", "stats6h", "stats24h")} + conn.execute( + """INSERT INTO token_stats (mint, organic_score, organic_label, holder_count, liquidity_usd, usd_price, mcap, + top_holders_pct, dev_balance_pct, dev_mints, is_sus, mint_auth_disabled, freeze_auth_disabled, is_verified, stats) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""", + (tok["id"], tok.get("organicScore"), tok.get("organicScoreLabel"), tok.get("holderCount"), tok.get("liquidity"), + tok.get("usdPrice"), tok.get("mcap"), audit.get("topHoldersPercentage"), audit.get("devBalancePercentage"), + audit.get("devMints"), audit.get("isSus"), audit.get("mintAuthorityDisabled"), audit.get("freezeAuthorityDisabled"), + tok.get("isVerified"), json.dumps(stats, default=str)), + ) + + +def refresh_stats(conn, client: JupiterTokens) -> dict: + """token_stats for the selector's universe: every pump.fun PumpSwap token the stream saw trade in the last hour with a + pool above the selector's gate, so a point-in-time history accrues for the tokens actually traded.""" + from ..train.decisions import MIN_RESQ_SOL # lazy: keeps the training stack out of the worker's import + uni = conn.execute("SELECT DISTINCT mint FROM pump_minutes WHERE ts > now() - interval '1 hour' AND resq_sol >= %s", + (MIN_RESQ_SOL,)).fetchall() + mints = sorted(r["mint"] for r in uni) + toks = [t for t in client.search_many(mints) if t.get("id")] if mints else [] + for tok in toks: + insert_stats(conn, tok) + return {"universe": len(mints), "refreshed": len(toks)} + + +def run_once() -> None: + setup("discover") + with transaction() as conn: + print(json.dumps(refresh_stats(conn, JupiterTokens()), indent=1)) + + +def run_forever(stop_event=None) -> None: + """Loop until stop_event is set (console thread) or SIGTERM/SIGINT (CLI, main thread only).""" + import threading + setup("discover") + client = JupiterTokens(); stopped = {"v": False} + if stop_event is None and threading.current_thread() is threading.main_thread(): + def _stop(*_): + stopped["v"] = True + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + record_event("info", "discover", "token stats worker started", {"refresh_s": config.STATS_REFRESH_S}) + while not stopped["v"] and not (stop_event is not None and stop_event.is_set()): + t0 = time.monotonic() + try: + with transaction() as conn: + log.info("stats refresh %s", refresh_stats(conn, client)) + except Exception as e: + log.exception("token stats refresh error") + record_event("error", "discover", f"stats refresh error: {type(e).__name__}: {e}") + # sleep the remainder of the interval, waking early on stop + remaining = config.STATS_REFRESH_S - (time.monotonic() - t0) + while remaining > 0 and not stopped["v"] and not (stop_event is not None and stop_event.is_set()): + time.sleep(min(1.0, remaining)) + remaining -= 1.0 + record_event("info", "discover", "token stats worker stopped") diff --git a/fly_trader/ingest/pumpstream.py b/fly_trader/ingest/pumpstream.py new file mode 100644 index 0000000000000000000000000000000000000000..c4c9b7649a658d81a36951d0e9a0d16e21768f81 --- /dev/null +++ b/fly_trader/ingest/pumpstream.py @@ -0,0 +1,488 @@ +"""PumpAPI live stream → per-minute rich candles (Supervisor thread ``pumpstream``). + +``wss://stream.pumpapi.io`` is a free, keyless firehose (~400–800 events/s, one connection per IP) whose events are +identical to the replay archive. This worker keeps only what the selector needs: PumpSwap (``pump-amm``, SOL-quoted) +buys and sells of pump.fun-origin mints, aggregated per (mint, minute) into the same fields ``train/mature.py`` +derives from the archive, with the same filters: legs in a pool created directly rather than by a pump.fun migration +(``createPool`` with ``poolCreatedBy`` other than ``pump``; the owner can pull its unburned liquidity) are dropped — such pools +go to ``pump_pools`` as they are created, and the blocked set is reloaded from that table (archive backfill included) at +start and every 5 minutes —, the pool's quote reserve must lie in 0.001–100,000 SOL, a leg more +than 50× away from the median of the mint's previous 200 legs that UTC day is dropped, and each minute keeps the +pool with the most legs. Minutes are written to ``pump_minutes`` once complete: an event stamped ≥ 2 s after the +minute's end has arrived (event-time watermark) or, once no trade has arrived for 15 s, by wall clock; a leg for a minute +already written is dropped (never merged into it). ``pumpstream_status.flushed_through`` +publishes the newest complete minute for the selector. + +Lifecycle: ``create`` events go to ``pump_events`` as they arrive; at ``migrate`` a ``corpus_meta`` row is written +with the creation facts and the creator's point-in-time history (``train/corpus_meta.creator_history``, the training +definition); 61 minutes after a graduation its own 60-minute outcome is filled from the stream's minutes so later +graduations by the same creator see it, as in training. Minute rows older than ``PUMP_MINUTES_KEEP_DAYS`` are archived +to ``data/corpus/stream/.parquet`` before removal. Database failures keep the data in memory for the next attempt. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import math +import ssl +import statistics +import threading +import time +from collections import defaultdict, deque +from datetime import datetime, timedelta, timezone + +import certifi +import orjson +import pyarrow as pa +import pyarrow.parquet as pq +import websockets + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup +from ..train.corpus_meta import CURVE_COLS, blocked_pool_ids, creator_history, curve_facts +from ..train.flow import minute_wallet_cols + +log = logging.getLogger(__name__) +WSOL = "So11111111111111111111111111111111111111112" +RESQ_BAND = (0.001, 100000.0) # train/mature.py: quote_in_pool BETWEEN 0.001 AND 100000 +PRICE_BAND = 50.0 # train/mature.py: within 50x of the trailing median of the mint's previous 200 legs +REF_LEGS = 200 +FLUSH_GRACE_S = 2.0 +FLUSH_FALLBACK_S = 15.0 +MAX_PENDING_CREATES = 200_000 + + +def _txt(v): + """Postgres text cannot hold NUL bytes; one in a token name would otherwise wedge the create queue.""" + return v.replace("\x00", "") if isinstance(v, str) else v + + +class Minute: + __slots__ = ("open", "high", "low", "close", "buy", "sell", "nb", "ns", "traders", "wallets", "resq", "pool_id", "fee_rate") + + def __init__(self, pool_id: str | None = None): + self.open = None; self.high = -math.inf; self.low = math.inf; self.close = None; self.buy = 0.0; self.sell = 0.0 + self.nb = 0; self.ns = 0; self.traders = set(); self.resq = None; self.pool_id = pool_id; self.fee_rate = None # the pool fee charged (poolFeeRate) + self.wallets: dict[str, list[float]] = {} # trader → [bought SOL, sold SOL] (train/flow.py) + + +class Aggregator: + def __init__(self): + self.minutes: dict[int, dict[str, dict[str, Minute]]] = defaultdict(lambda: defaultdict(dict)) # minute → mint → pool → Minute + self.ref: dict[str, deque] = {}; self.ref_day: int | None = None # trailing legs per mint, reset each UTC day + self.max_event_s = 0.0; self.last_event_wall = 0.0 # newest event time; wall clock of the last trade received + self.creates: dict[str, dict] = {} # mint → create facts (24 h) + self.curve: dict[str, dict] = {} # mint → bonding-curve wallets since its create (corpus_meta.curve_facts) + self.pending_creates: list[tuple[str, dict]] = [] # creates not yet written to pump_events + self.blocked: set[str] = set() # pool ids of directly created (custom) PumpSwap pools + self.pending_pools: list[tuple] = [] # custom pools not yet written to pump_pools + self.flushed_through: datetime | None = None # start of the newest complete minute written + self.insiders: dict[str, frozenset] = {} # mint → creator/bundle/early wallets (token_insiders) + self.skill: dict | None = None; self.skill_day: str | None = None # wallet → skill decile for the current UTC day (train/wallet_skill.py) + self.stats = {"events": 0, "trades": 0, "dropped_band": 0, "dropped_late": 0, "dropped_custom_pool": 0, "flushed_minutes": 0, "flushed_rows": 0, "creates": 0, "migrates": 0, + "custom_pools": 0, "blocked_pools": 0, "reconnects": 0, "outcomes_filled": 0, "started_at": datetime.now(timezone.utc).isoformat()} + + def row(self, minute: int, mint: str) -> Minute | None: + """The minute's candle for a mint: the pool with the most legs (as in training).""" + pools = self.minutes.get(minute, {}).get(mint) + return max(pools.values(), key=lambda r: (r.nb + r.ns, r.buy + r.sell)) if pools else None + + def ingest(self, e: dict) -> None: + self.stats["events"] += 1 + a = e.get("action"); pool = e.get("pool"); mint = e.get("mint"); ts = e.get("timestamp") + if a == "createPool" and pool == "pump-amm" and (e.get("poolCreatedBy") or "custom") != "pump": + pid = e.get("poolId") # not a pump.fun migration: the owner can pull the liquidity unseen + if pid: + self.block([pid]); self.stats["custom_pools"] += 1 + self.pending_pools.append((pid, mint, e.get("poolCreatedBy"), datetime.fromtimestamp(ts / 1000, timezone.utc) if ts is not None else None)) + return + if ts is None or not mint: + return + if a in ("buy", "sell") and pool == "pump": + self._curve(e) + return + if a in ("buy", "sell"): + if pool != "pump-amm" or e.get("quoteMint") != WSOL or not str(mint).endswith("pump"): + return + if e.get("poolId") in self.blocked: + self.stats["dropped_custom_pool"] += 1 + return + price = e.get("price"); q = e.get("quoteInPool") + if not price or price <= 0 or q is None or not (RESQ_BAND[0] <= float(q) <= RESQ_BAND[1]): + return + ts_s = ts / 1000.0; self.max_event_s = max(self.max_event_s, ts_s); self.last_event_wall = time.time() + day = int(ts_s // 86400) + if self.ref_day is None or day > self.ref_day: # a late event from the previous day never wipes today's references + self.ref = {}; self.ref_day = day + p = float(price) + legs = e.get("breakdown") or [{"action": a, "trader": e.get("txSigner"), "quoteAmount": e.get("quoteAmount")}] + dq = self.ref.get(mint) + if dq is None: + dq = self.ref[mint] = deque(maxlen=REF_LEGS) + ok = True + if dq: + pref = statistics.median(dq) + ok = pref / PRICE_BAND <= p <= pref * PRICE_BAND + for _ in legs: + dq.append(p) + if not ok: + self.stats["dropped_band"] += 1 + return + m = int(ts_s // 60) * 60; pid = e.get("poolId") or "" + if self.flushed_through is not None and m <= self.flushed_through.timestamp(): + self.stats["dropped_late"] += 1 # its minute is written and consumed; a partial re-write would corrupt it + return + row = self.minutes[m][mint].get(pid) + if row is None: + row = self.minutes[m][mint][pid] = Minute(pid or None) + row.open = p if row.open is None else row.open; row.high = max(row.high, p); row.low = min(row.low, p); row.close = p + for b in legs: + sol = float(b.get("quoteAmount") or 0.0) + if b.get("action") == "buy": + row.buy += sol; row.nb += 1 + else: + row.sell += sol; row.ns += 1 + trader = b.get("trader") or e.get("txSigner") # as replay_pull writes the archive's trader column + if trader: + row.traders.add(trader) + row.wallets.setdefault(trader, [0.0, 0.0])[0 if b.get("action") == "buy" else 1] += sol + row.resq = float(q) + if e.get("poolFeeRate") is not None: + row.fee_rate = float(e["poolFeeRate"]) + self.stats["trades"] += 1 + elif a == "create" and pool == "pump": + c = {"ts": ts, "creator": e.get("txSigner"), "dev_sol": e.get("quoteAmount"), "dev_tokens": e.get("initialBuy"), "supply": e.get("supply"), + "mayhem": e.get("mayhemMode"), "name": _txt(e.get("name")), "symbol": _txt(e.get("symbol")), "uri": _txt(e.get("uri")), "sig": e.get("signature")} + self.creates[mint] = c; self.pending_creates.append((mint, c)) + self.curve[mint] = {"slot": int(e.get("block") or 0), "creator": e.get("txSigner"), "supply": e.get("supply"), "initial_buy": e.get("initialBuy"), "ts": ts, "wallets": {}} + self.stats["creates"] += 1 + if len(self.creates) > 200_000: + cutoff = ts - 86_400_000 + self.creates = {k: v for k, v in self.creates.items() if v["ts"] >= cutoff} + self.curve = {k: v for k, v in self.curve.items() if v["ts"] >= cutoff} + elif a == "migrate" and pool == "pump-amm": + self.stats["migrates"] += 1 + self.write_creates() # the creator's own earlier creates must be visible to the history query + self._graduation(e) + + def _curve(self, e: dict) -> None: + """A bonding-curve buy/sell of a token created while the stream ran: per wallet its first buy slot and tokens bought/sold.""" + st = self.curve.get(e.get("mint")) + if st is None: + return # created before this process started: the archive fills its facts (fail closed until then) + slot = int(e.get("block") or 0) + for b in e.get("breakdown") or [{"action": e.get("action"), "trader": e.get("txSigner"), "tokenAmount": e.get("tokenAmount")}]: + w = b.get("trader") or e.get("txSigner") + if not w: + continue + rec = st["wallets"].setdefault(w, [None, 0.0, 0.0]); tok = float(b.get("tokenAmount") or 0.0) + if (b.get("action") or e.get("action")) == "buy": + rec[1] += tok; rec[0] = slot if rec[0] is None else min(rec[0], slot) + else: + rec[2] += tok + + def write_creates(self) -> int: + """Persist creates as they arrive (keyed by signature); on failure they stay queued for the next attempt.""" + rows, self.pending_creates = self.pending_creates, [] + if not rows: + return 0 + try: + with transaction() as conn: + conn.cursor().executemany("INSERT INTO pump_events (sig, ts, action, pool, mint, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri) " + "VALUES (%s,%s,'create','pump',%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (sig) DO NOTHING", + [(c["sig"], datetime.fromtimestamp(c["ts"] / 1000, timezone.utc), m, c["creator"], c["dev_sol"], c["dev_tokens"], c["supply"], + c["mayhem"], c["name"], c["symbol"], c["uri"]) for m, c in rows if c.get("sig")]) + except Exception: + self.pending_creates = (rows + self.pending_creates)[-MAX_PENDING_CREATES:] + log.exception("create rows failed; %d queued for the next attempt", len(self.pending_creates)) + return 0 + return len(rows) + + def block(self, pool_ids) -> None: + self.blocked.update(pool_ids); self.stats["blocked_pools"] = len(self.blocked) + + def write_pools(self) -> int: + """Persist custom pools seen on the stream (training excludes the same pools); on failure they stay queued.""" + rows, self.pending_pools = self.pending_pools, [] + if not rows: + return 0 + try: + with transaction() as conn: + conn.cursor().executemany("INSERT INTO pump_pools (pool_id, mint, created_by, ts, source) VALUES (%s,%s,%s,%s,'stream') ON CONFLICT (pool_id) DO NOTHING", rows) + except Exception: + self.pending_pools = (rows + self.pending_pools)[-MAX_PENDING_CREATES:] + log.exception("custom pool rows failed; %d queued for the next attempt", len(self.pending_pools)) + return 0 + return len(rows) + + def _graduation(self, e: dict) -> None: + mint = e["mint"]; g = datetime.fromtimestamp(e["timestamp"] / 1000, timezone.utc); c = self.creates.get(mint) + try: + with transaction() as conn: + if c is None: # created before this process started: look it up + r = conn.execute("SELECT sig, ts, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri FROM pump_events WHERE mint = %s AND action = 'create' ORDER BY ts LIMIT 1", (mint,)).fetchone() + if r: + c = {"ts": r["ts"].timestamp() * 1000, "creator": r["signer"], "dev_sol": r["dev_sol"], "dev_tokens": r["dev_tokens"], "supply": r["supply"], + "mayhem": r["mayhem"], "name": r["name"], "symbol": r["symbol"], "uri": r["uri"], "sig": r["sig"]} + conn.execute("INSERT INTO pump_events (sig, ts, action, pool, mint, pool_id, signer, quote_in_pool) VALUES (%s,%s,'migrate','pump-amm',%s,%s,%s,%s) ON CONFLICT (sig) DO NOTHING", + (e.get("signature"), g, mint, e.get("poolId"), e.get("txSigner"), e.get("quoteInPool"))) + ct = datetime.fromtimestamp(c["ts"] / 1000, timezone.utc) if c else None + h = creator_history(conn, c.get("creator") if c else None, ct, g) + conn.execute("""INSERT INTO corpus_meta (mint, graduated_at, create_ts, creator, dev_sol, dev_tokens, dev_share, supply, mayhem, uri, name, symbol, ttg_min, rq0, pool_id, + prior_launches, prior_grads, prior_known, prior_rug_share, prior_moon_share) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint) DO UPDATE SET + graduated_at = COALESCE(corpus_meta.graduated_at, EXCLUDED.graduated_at), + create_ts = COALESCE(corpus_meta.create_ts, EXCLUDED.create_ts), creator = COALESCE(corpus_meta.creator, EXCLUDED.creator), + dev_sol = COALESCE(corpus_meta.dev_sol, EXCLUDED.dev_sol), dev_tokens = COALESCE(corpus_meta.dev_tokens, EXCLUDED.dev_tokens), + dev_share = COALESCE(corpus_meta.dev_share, EXCLUDED.dev_share), supply = COALESCE(corpus_meta.supply, EXCLUDED.supply), + mayhem = COALESCE(corpus_meta.mayhem, EXCLUDED.mayhem), uri = COALESCE(corpus_meta.uri, EXCLUDED.uri), + name = COALESCE(corpus_meta.name, EXCLUDED.name), symbol = COALESCE(corpus_meta.symbol, EXCLUDED.symbol), + ttg_min = COALESCE(corpus_meta.ttg_min, EXCLUDED.ttg_min), rq0 = COALESCE(corpus_meta.rq0, EXCLUDED.rq0), pool_id = COALESCE(corpus_meta.pool_id, EXCLUDED.pool_id), + prior_launches = COALESCE(corpus_meta.prior_launches, EXCLUDED.prior_launches), prior_grads = COALESCE(corpus_meta.prior_grads, EXCLUDED.prior_grads), + prior_known = COALESCE(corpus_meta.prior_known, EXCLUDED.prior_known), prior_rug_share = COALESCE(corpus_meta.prior_rug_share, EXCLUDED.prior_rug_share), + prior_moon_share = COALESCE(corpus_meta.prior_moon_share, EXCLUDED.prior_moon_share), updated_at = now()""", + (mint, g, ct, c.get("creator") if c else None, c.get("dev_sol") if c else None, c.get("dev_tokens") if c else None, + (float(c["dev_tokens"]) / float(c["supply"])) if c and c.get("dev_tokens") and c.get("supply") else None, + c.get("supply") if c else None, c.get("mayhem") if c else None, c.get("uri") if c else None, c.get("name") if c else None, c.get("symbol") if c else None, + ((e["timestamp"] - c["ts"]) / 60000.0) if c else None, e.get("quoteInPool"), e.get("poolId"), + h["prior_launches"], h["prior_grads"], h["prior_known"], h["prior_rug_share"], h["prior_moon_share"])) + conn.execute("INSERT INTO corpus_tokens (mint, graduated_at, grad_slot, source, status) VALUES (%s,%s,%s,'stream','pending') ON CONFLICT (mint) DO NOTHING", + (mint, g, int(e.get("block") or 0))) + st = self.curve.pop(mint, None) + if st is not None: # the token's whole curve life was seen: the archive's definition (curve_facts) + facts, insiders = curve_facts(st["slot"], st["creator"], st["supply"], st["initial_buy"], st["wallets"]) + conn.execute("UPDATE corpus_meta SET " + ", ".join(f"{c} = COALESCE({c}, %s)" for c in CURVE_COLS) + ", curve_known = COALESCE(curve_known, true) WHERE mint = %s", + (*[facts[c] for c in CURVE_COLS], mint)) + conn.cursor().executemany("INSERT INTO token_insiders (mint, wallet, kind) VALUES (%s,%s,%s) ON CONFLICT (mint, wallet) DO NOTHING", + [(mint, w, k) for w, k in insiders]) + self.insiders[mint] = frozenset(w for w, _ in insiders) + except Exception: + log.exception("graduation upsert failed for %s", mint) + + def flush(self, before_minute: int) -> int: + """Write every minute strictly older than ``before_minute`` (seconds). Rows leave memory only after the commit.""" + done = sorted(m for m in self.minutes if m < before_minute) + if not done: + return 0 + rows = [] + for m in done: + ts = datetime.fromtimestamp(m, timezone.utc) + for mint in self.minutes[m]: + r = self.row(m, mint) + if r is None or r.close is None: + continue + wc = minute_wallet_cols(r.wallets, self.insiders.get(mint, frozenset()), self.skill) + rows.append((mint, ts, r.pool_id, r.open, r.high, r.low, r.close, r.buy, r.sell, r.nb, r.ns, len(r.traders), r.resq, r.fee_rate, + wc["n_buyers"], wc["wash_sol"], wc["wash_buy_sol"], wc["top_sell_sol"], wc["insider_sell_sol"], wc["skill_buy"])) + if rows: + with transaction() as conn: + conn.cursor().executemany("INSERT INTO pump_minutes (mint, ts, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol, fee_rate, " + "n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint, ts) DO UPDATE SET close = EXCLUDED.close, high = greatest(pump_minutes.high, EXCLUDED.high), " + "low = least(pump_minutes.low, EXCLUDED.low), buy_sol = pump_minutes.buy_sol + EXCLUDED.buy_sol, sell_sol = pump_minutes.sell_sol + EXCLUDED.sell_sol, " + "n_buys = pump_minutes.n_buys + EXCLUDED.n_buys, n_sells = pump_minutes.n_sells + EXCLUDED.n_sells, n_traders = greatest(pump_minutes.n_traders, EXCLUDED.n_traders), " + "resq_sol = EXCLUDED.resq_sol, fee_rate = COALESCE(EXCLUDED.fee_rate, pump_minutes.fee_rate), n_buyers = EXCLUDED.n_buyers, wash_sol = EXCLUDED.wash_sol, " + "wash_buy_sol = EXCLUDED.wash_buy_sol, top_sell_sol = EXCLUDED.top_sell_sol, insider_sell_sol = EXCLUDED.insider_sell_sol, skill_buy = EXCLUDED.skill_buy", rows) + for m in done: + self.minutes.pop(m, None) + self.stats["flushed_minutes"] += len(done); self.stats["flushed_rows"] += len(rows) + newest = datetime.fromtimestamp(done[-1], timezone.utc) + if self.flushed_through is None or newest > self.flushed_through: # a late event for an old minute never rewinds it + self.flushed_through = newest + return len(rows) + + def flush_bound(self, now: float) -> int: + """Minutes strictly before this start are complete: event-time watermark, or the wall-clock fallback once no trade + has arrived for ``FLUSH_FALLBACK_S`` (a lagging stream still delivers the minute's events, so it waits for them).""" + cur = int(now // 60) * 60 + wm = int((self.max_event_s - FLUSH_GRACE_S) // 60) * 60 + if now - self.last_event_wall >= FLUSH_FALLBACK_S: + wm = max(wm, int((now - FLUSH_FALLBACK_S) // 60) * 60) + return min(cur, wm) + + +def fill_outcomes() -> int: + """Own 60-minute outcome for stream graduations 61+ minutes old, from the stream's minutes (candle files replace it later).""" + with transaction() as conn: + todo = conn.execute("""SELECT mint, graduated_at FROM corpus_meta WHERE own_dd60 IS NULL AND graduated_at IS NOT NULL + AND graduated_at BETWEEN now() - interval '6 hours' AND now() - interval '61 minutes'""").fetchall() + n = 0 + for t in todo: + rows = conn.execute("SELECT close FROM pump_minutes WHERE mint = %s AND ts >= %s AND ts < %s ORDER BY ts", + (t["mint"], t["graduated_at"], t["graduated_at"] + timedelta(minutes=60))).fetchall() + if not rows: + continue + closes = [float(r["close"]) for r in rows]; p0 = closes[0] + if p0 <= 0: + continue + conn.execute("UPDATE corpus_meta SET own_dd60 = %s, own_max60 = %s, updated_at = now() WHERE mint = %s AND own_dd60 IS NULL", + (min(closes) / p0 - 1.0, max(closes) / p0 - 1.0, t["mint"])) + n += 1 + return n + + +def archive_old(keep_days: int) -> int: + """Archive minute rows older than ``keep_days`` to Parquet, then remove them from the table.""" + cutoff = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=keep_days) + with transaction() as conn: + days = [r["d"] for r in conn.execute("SELECT DISTINCT date_trunc('day', ts) AS d FROM pump_minutes WHERE ts < %s", (cutoff,)).fetchall()] + n = 0 + for d in days: + with transaction() as conn: + rows = conn.execute("SELECT * FROM pump_minutes WHERE ts >= %s AND ts < %s ORDER BY mint, ts", (d, d + timedelta(days=1))).fetchall() + if rows: + out = config.CORPUS_DIR / "stream"; out.mkdir(parents=True, exist_ok=True) + path = out / f"{d:%Y-%m-%d}.parquet"; tmp = path.with_name(path.name + ".tmp") + pq.write_table(pa.Table.from_pylist([dict(r) for r in rows]), tmp, compression="zstd"); tmp.replace(path) + conn.execute("DELETE FROM pump_minutes WHERE ts >= %s AND ts < %s", (d, d + timedelta(days=1))) + n += len(rows) + return n + + +def _status(agg: Aggregator, extra: dict) -> None: + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('pumpstream_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps({**agg.stats, **extra, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str),)) + except Exception: + log.debug("pumpstream status write failed", exc_info=True) + + +def _logged(name: str, fn, *args): + try: + n = fn(*args) + if n: + log.info("%s: %d", name, n) + return n + except Exception: + log.exception("%s failed", name) + return 0 + + +def _load_insiders(agg: Aggregator) -> None: + """Insider wallets (creator, bundle, early buyers: token_insiders) of every mint traded today not loaded yet; a token's + set is fixed at its graduation, so loaded sets are kept.""" + todo = [m for m in agg.ref if m not in agg.insiders] + if not todo: + return + try: + with transaction() as conn: + rows = conn.execute("SELECT mint, wallet FROM token_insiders WHERE mint = ANY(%s)", (todo,)).fetchall() + except Exception: + log.exception("insider load failed; retried in 1 min") + return + got: dict[str, set] = {} + for r in rows: + got.setdefault(r["mint"], set()).add(r["wallet"]) + for m in todo: + agg.insiders[m] = frozenset(got.get(m, ())) + + +def load_skill(agg: Aggregator, now: float | None = None) -> None: + """The wallet-skill table of the current UTC day (the file training joins for that day), once per day.""" + from ..train.mature import SKILL_DIR + day = datetime.fromtimestamp(now or time.time(), timezone.utc).date().isoformat() + if agg.skill_day == day and agg.skill is not None: + return + f = SKILL_DIR / f"{day}.parquet" + if not f.exists(): + have = sorted(SKILL_DIR.glob("????-??-??.parquet")) if SKILL_DIR.exists() else [] + if not have: + agg.skill = None; agg.skill_day = day + return + f = have[-1] # a shipped table going stale beats none: without one the selector fails closed and the fly's skill inputs read as zero + log.warning("no wallet skill table for %s: using the newest on disk, %s (skill inputs %d days stale)", day, f.stem, + (datetime.fromisoformat(day) - datetime.fromisoformat(f.stem)).days) + t = pq.read_table(f, columns=["wallet", "bucket"]) + agg.skill = dict(zip(t["wallet"].to_pylist(), t["bucket"].to_pylist())); agg.skill_day = day + log.info("wallet skill table for %s loaded: %d wallets", day, len(agg.skill)) + + +def _load_blocked(agg: Aggregator) -> None: + """Union the custom pools in ``pump_pools`` (stream rows and the archive backfill) into the blocked set.""" + try: + with transaction() as conn: + agg.block(blocked_pool_ids(conn)) + except Exception: + log.exception("custom pool load failed; retried in 5 min") + + +async def _run(stop: threading.Event | None, agg: Aggregator) -> None: + ctx = ssl.create_default_context(cafile=certifi.where()); backoff = 1.0 + _load_blocked(agg); last_pools = time.time(); last_ins = 0.0 + try: + load_skill(agg) + except Exception: + log.exception("wallet skill load failed; retried in 5 min") + last_status = last_flush = last_outcomes = 0.0; last_archive = time.time() - 3000; last_msg = 0.0; rate_n = 0; rate_t = time.time() + bg: dict[str, asyncio.Task | None] = {"archive": None, "outcomes": None} + while not (stop is not None and stop.is_set()): + try: + async with websockets.connect(config.PUMPSTREAM_URL, ssl=ctx, max_size=8_000_000, ping_interval=20, ping_timeout=20) as ws: + log.info("pumpstream connected"); backoff = 1.0 + while not (stop is not None and stop.is_set()): + try: + msg = await asyncio.wait_for(ws.recv(), timeout=30) + except asyncio.TimeoutError: + log.warning("pumpstream: no events for 30 s; reconnecting"); break + last_msg = time.time(); rate_n += 1 + try: + e = orjson.loads(msg) + except orjson.JSONDecodeError: + continue + if isinstance(e, dict): + agg.ingest(e) + now = time.time() + if now - last_flush >= 1.0: + try: + agg.flush(agg.flush_bound(now)) + except Exception: + log.exception("minute flush failed; rows kept for the next attempt") + last_flush = now + if now - last_pools >= 300: + _load_blocked(agg); last_pools = now + try: + load_skill(agg, now) + except Exception: + log.exception("wallet skill load failed; retried in 5 min") + if now - last_ins >= 60: + _load_insiders(agg); last_ins = now + if now - last_status >= 5.0: + agg.write_creates(); agg.write_pools() + _status(agg, {"events_per_s": rate_n / max(now - rate_t, 1e-9), "pending_minutes": len(agg.minutes), "lag_s": now - last_msg, "connected": True, + "flushed_through": agg.flushed_through.isoformat() if agg.flushed_through else None, + "event_watermark": datetime.fromtimestamp(agg.max_event_s, timezone.utc).isoformat() if agg.max_event_s else None}) + rate_n = 0; rate_t = now; last_status = now + if now - last_outcomes >= 60 and (bg["outcomes"] is None or bg["outcomes"].done()): + bg["outcomes"] = asyncio.create_task(asyncio.to_thread(_logged, "stream outcomes filled", fill_outcomes)); last_outcomes = now + if now - last_archive >= 3600 and (bg["archive"] is None or bg["archive"].done()): + bg["archive"] = asyncio.create_task(asyncio.to_thread(_logged, "archived old minute rows", archive_old, config.PUMP_MINUTES_KEEP_DAYS)); last_archive = now + except Exception as e: + agg.stats["reconnects"] += 1; log.warning("pumpstream: %s; reconnecting in %.0fs", e, backoff) + _status(agg, {"connected": False, "last_error": str(e)[:200]}) + await asyncio.sleep(backoff); backoff = min(60.0, backoff * 2) + for t in bg.values(): + if t is not None: + try: + await t + except Exception: + pass + try: + agg.flush(int(time.time() // 60) * 60 + 60) + except Exception: + log.exception("final minute flush failed") + agg.write_creates(); agg.write_pools() + + +def main(stop_event: threading.Event | None = None) -> None: + setup("pumpstream"); agg = Aggregator() + record_event("info", "pumpstream", "pumpstream started", {"url": config.PUMPSTREAM_URL}) + try: + asyncio.run(_run(stop_event, agg)) + finally: + _status(agg, {"connected": False, "stage": "stopped"}) + record_event("info", "pumpstream", "pumpstream stopped", {k: agg.stats.get(k) for k in ("events", "trades", "dropped_band", "dropped_custom_pool", "flushed_rows", "creates", + "custom_pools", "migrates", "reconnects")}) diff --git a/fly_trader/ingest/replay_pull.py b/fly_trader/ingest/replay_pull.py new file mode 100644 index 0000000000000000000000000000000000000000..38587df34be86cd973ec0362ae488f55e6ec154c --- /dev/null +++ b/fly_trader/ingest/replay_pull.py @@ -0,0 +1,315 @@ +"""PumpAPI historical replay → hourly Parquet (Supervisor thread ``replay``). + +pumpapi.io publishes every event its decoded data stream emitted, hour by hour, since 2026-04-18: +``https://replay.pumpapi.io/YYYY/MM/DD/HH.jsonl.zst`` (~400 MB zstd, ~2 GB JSONL, ~1.7 M events; free, no key). +Each line is one decoded transaction: buys/sells with the trader, SOL and token amounts, price, and the pool's real +reserves; ``create`` with the creator wallet and metadata URI; ``migrate`` (graduation); ``createPool``. + +This worker downloads ``REPLAY_PARALLEL`` hours at a time (measured ~2–4 MB/s per stream), parses each in one +thread and writes, per hour, ``data/corpus/replay//_trades.parquet`` (pump.fun bonding-curve and +PumpSwap buys/sells, one row per trader leg; signatures and per-mint constants are left out to keep it ~70 MB/hour, the +lifecycle table carries creator, pool id, metadata URI and the reserves at migration) and ``_events.parquet`` (every non-trade lifecycle event across +all pools, small). Hours are processed newest first so the current regime lands first, and hours that close while the backfill runs +are fetched ahead of it, so the archive stays within ~2 hours of live; the registry is ``replay_hours``. The compressed download is a temporary file and is removed after a successful parse — the +public archive remains the source of truth. Downstream: ``train/replay_assemble.py`` turns days into per-token +candle/trade files for the feature builder. +""" +from __future__ import annotations + +import io +import json +import logging +import queue +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import httpx +import orjson +import pyarrow as pa +import pyarrow.parquet as pq +import zstandard + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup +from .corpus_pull import _db_retry, _sleep + +log = logging.getLogger(__name__) +HOUR_SQL = ("INSERT INTO replay_hours (hour, status, last_error) VALUES (%s, %s, %s) " + "ON CONFLICT (hour) DO UPDATE SET status = EXCLUDED.status, last_error = EXCLUDED.last_error, done_at = now()") +TRADE_POOLS = {"pump", "pump-amm"} +SKIP_ACTIONS = {"buy", "sell", "transfer", "add", "remove", "claimCreatorFees"} +TRADE_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot", pa.int64()), ("pool", pa.dictionary(pa.int8(), pa.string())), ("mint", pa.string()), + ("trader", pa.string()), ("side", pa.int8()), ("sol", pa.float64()), ("tokens", pa.float64()), ("price", pa.float64()), + ("quote_in_pool", pa.float64()), ("tokens_in_pool", pa.float64()), ("vquote", pa.float32()), ("n_legs", pa.int8()), + ("quote_mint", pa.dictionary(pa.int16(), pa.string())), ("pool_id", pa.dictionary(pa.int32(), pa.string()))]) +EVENT_SCHEMA = pa.schema([("ts", pa.timestamp("ms", tz="UTC")), ("slot", pa.int64()), ("sig", pa.string()), ("action", pa.string()), ("pool", pa.string()), + ("mint", pa.string()), ("pool_id", pa.string()), ("signer", pa.string()), ("creator_fee_addr", pa.string()), ("name", pa.string()), + ("symbol", pa.string()), ("uri", pa.string()), ("supply", pa.float64()), ("decimals", pa.int16()), ("token_program", pa.string()), + ("initial_buy", pa.float64()), ("quote_amount", pa.float64()), ("quote_in_pool", pa.float64()), ("tokens_in_pool", pa.float64()), + ("mayhem", pa.bool_()), ("pool_created_by", pa.string()), ("mint_authority", pa.string()), ("freeze_authority", pa.string())]) + + +def _f(x) -> float | None: + try: + return float(x) if x is not None else None + except (TypeError, ValueError): + return None + + +def _q(d: dict, new: str, old: str): + """A quote-side field under its current name, else under the archive's pre-2026-05-21 SOL-only name.""" + v = d.get(new) + return v if v is not None else d.get(old) + + +def parse_hour(path: Path) -> tuple[pa.Table, pa.Table, int]: + """Stream-decompress one hour file into (trades, events, n_events).""" + T = {name: [] for name in TRADE_SCHEMA.names}; E = {name: [] for name in EVENT_SCHEMA.names}; n = 0 + dctx = zstandard.ZstdDecompressor() + with open(path, "rb") as fh, dctx.stream_reader(fh) as reader: + for line in io.BufferedReader(reader, buffer_size=8 << 20): + if not line.strip(): + continue + try: + e = orjson.loads(line) + except orjson.JSONDecodeError: + continue + n += 1 + # the archive before 2026-05-21 names fields txType / solAmount / solInPool / vSolInBondingCurve; later: action / quote* + a = e.get("action") or e.get("txType"); pool = e.get("pool") + ts = e.get("timestamp") + if ts is None: + continue + tsdt = datetime.fromtimestamp(int(ts) / 1000, timezone.utc) + if a in ("buy", "sell"): + if pool not in TRADE_POOLS: + continue + legs = e.get("breakdown") or [{"action": a, "trader": e.get("txSigner"), "tokenAmount": e.get("tokenAmount"), "quoteAmount": _q(e, "quoteAmount", "solAmount")}] + vq = _q(e, "vQuoteInBondingCurve", "vSolInBondingCurve") if pool == "pump" else _q(e, "virtualQuoteInPool", "virtualSolInPool") + for b in legs: + T["ts"].append(tsdt); T["slot"].append(int(e.get("block") or 0)); T["pool"].append(pool); T["mint"].append(e.get("mint")) + T["trader"].append(b.get("trader") or e.get("txSigner")); T["side"].append(1 if (b.get("action") or b.get("txType")) == "buy" else -1) + T["sol"].append(_f(_q(b, "quoteAmount", "solAmount")) or 0.0); T["tokens"].append(_f(b.get("tokenAmount")) or 0.0); T["price"].append(_f(e.get("price"))) + T["quote_in_pool"].append(_f(_q(e, "quoteInPool", "solInPool"))); T["tokens_in_pool"].append(_f(e.get("tokensInPool"))); T["vquote"].append(_f(vq)) + T["n_legs"].append(len(legs)); T["quote_mint"].append(e.get("quoteMint")); T["pool_id"].append(e.get("poolId")) + elif a not in SKIP_ACTIONS: + E["ts"].append(tsdt); E["slot"].append(int(e.get("block") or 0)); E["sig"].append(e.get("signature")); E["action"].append(a); E["pool"].append(pool) + E["mint"].append(e.get("mint")); E["pool_id"].append(e.get("poolId")); E["signer"].append(e.get("txSigner")); E["creator_fee_addr"].append(e.get("creatorFeeAddress")) + E["name"].append(e.get("name")); E["symbol"].append(e.get("symbol")); E["uri"].append(e.get("uri")); E["supply"].append(_f(e.get("supply"))) + E["decimals"].append(int(e["decimals"]) if e.get("decimals") is not None else None); E["token_program"].append(e.get("tokenProgram")) + E["initial_buy"].append(_f(e.get("initialBuy"))); E["quote_amount"].append(_f(_q(e, "quoteAmount", "solAmount"))); E["quote_in_pool"].append(_f(_q(e, "quoteInPool", "solInPool"))) + E["tokens_in_pool"].append(_f(e.get("tokensInPool"))); E["mayhem"].append(bool(e.get("mayhemMode")) if e.get("mayhemMode") is not None else None) + E["pool_created_by"].append(e.get("poolCreatedBy")); E["mint_authority"].append(e.get("mintAuthority")); E["freeze_authority"].append(e.get("freezeAuthority")) + return pa.table(T, schema=TRADE_SCHEMA), pa.table(E, schema=EVENT_SCHEMA), n + + +def refetch_missing_pool_id() -> int: + """Hours whose trades file predates the ``pool_id`` column (written 2026-08-20..09-13 by an older parser) are marked + ``refetch``: ``plan_hours`` downloads them again and ``parse_hour`` writes the column, so the archive applies the live + stream's blocked-pool filter and dominant-pool choice (train/mature.py). Returns the number of hours marked.""" + hours = [] + for f in sorted(config.REPLAY_DIR.glob("*/*_trades.parquet")): + if "pool_id" not in pq.read_schema(f).names: + d = datetime.fromisoformat(f.parent.name) + hours.append(datetime(d.year, d.month, d.day, int(f.name[:2]), tzinfo=timezone.utc)) + if hours: + with transaction() as conn: + conn.cursor().executemany(HOUR_SQL, [(h, "refetch", "trades file lacks pool_id") for h in hours]) + return len(hours) + + +def plan_hours(now: datetime | None = None) -> list[datetime]: + now = now or datetime.now(timezone.utc) # callers pass the same ``now`` they seed the re-plan bound with + start = datetime.fromisoformat(config.REPLAY_START).replace(tzinfo=timezone.utc) + last = (now - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0) + with transaction() as conn: + done = {r["hour"] for r in conn.execute("SELECT hour FROM replay_hours WHERE status = 'done'").fetchall()} + hours = []; h = last + while h >= start: + if h not in done: + hours.append(h) + h -= timedelta(hours=1) + return hours # newest first + + +def _put_hour(sql: str, params: tuple) -> None: + with transaction() as conn: + conn.execute(sql, params) + + +def _kept_downloads(tmpdir: Path, planned: set[datetime]) -> list[tuple[datetime, Path, int, float]]: + """Complete hour files left by a stopped worker, for hours still planned: parsed instead of downloaded again. + Partial files are removed (the archive ignores Range requests, so a download cannot resume), as are complete + files for hours no longer planned.""" + out = [] + for f in sorted(tmpdir.glob("*")): + if f.suffix == ".zst": + try: + h = datetime.strptime(f.stem, "%Y%m%d%H").replace(tzinfo=timezone.utc) + except ValueError: + h = None + if h in planned: + out.append((h, f, f.stat().st_size, 0.0)); continue + f.unlink() + return out + + +def _download(client: httpx.Client, hour: datetime, tmp: Path, stop: threading.Event | None) -> int: + url = f"{config.REPLAY_URL}/{hour:%Y/%m/%d/%H}.jsonl.zst" + part = tmp.with_suffix(".part"); size = 0 + with client.stream("GET", url) as r: + if r.status_code == 404: + return -1 + r.raise_for_status() + with open(part, "wb") as f: + for chunk in r.iter_bytes(1 << 20): + if stop is not None and stop.is_set(): + raise InterruptedError + f.write(chunk); size += len(chunk) + part.rename(tmp) + return size + + +class _Status: + def __init__(self, total: int): + self.s = {"hours_total": total, "hours_done": 0, "trades": 0, "events": 0, "bytes": 0, "started_at": datetime.now(timezone.utc).isoformat(), "errors": 0} + self.lock = threading.Lock(); self.last = 0.0; self.t0 = time.time() + + def update(self, force=False, **kv): + with self.lock: + self.s.update(kv); el = time.time() - self.t0 + self.s["mb_s"] = self.s["bytes"] / 1e6 / el if el > 1 else 0.0 + self.s["hours_per_h"] = self.s["hours_done"] / el * 3600 if el > 1 else 0.0 + remaining = self.s["hours_total"] - self.s["hours_done"] + self.s["eta_h"] = remaining / self.s["hours_per_h"] if self.s["hours_per_h"] else None + self.s["updated_at"] = datetime.now(timezone.utc).isoformat() + if not force and time.time() - self.last < 3.0: + return + self.last = time.time(); payload = dict(self.s) + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('replay_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(payload, default=str),)) + except Exception: + log.debug("replay status write failed", exc_info=True) + + +def main(stop_event: threading.Event | None = None) -> None: + setup("replay"); stop = stop_event + tmpdir = config.REPLAY_DIR / "_tmp"; tmpdir.mkdir(parents=True, exist_ok=True) + now0 = datetime.now(timezone.utc); hours = plan_hours(now0); status = _Status(len(hours)) + preloaded = _kept_downloads(tmpdir, set(hours)) # finished before a restart: parse, don't download again + if preloaded: + kept = {p[0] for p in preloaded}; hours = [h for h in hours if h not in kept] + log.info("replay ingest: %d complete hour file(s) kept from before the restart", len(preloaded)) + record_event("info", "replay", "replay ingest started", {"hours_pending": len(hours), "parallel": config.REPLAY_PARALLEL}) + log.info("replay ingest: %d hours pending (newest first)", len(hours)) + todo: queue.Queue = queue.Queue(); ready: queue.Queue = queue.Queue(maxsize=config.REPLAY_PARALLEL + 2) + for h in hours: + todo.put(h) + client = httpx.Client(headers={"User-Agent": "fly-trader replay ingest"}, timeout=httpx.Timeout(60.0, read=600.0), follow_redirects=True) + + newest_planned = [(now0 - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0)] # the plan's upper bound (same clock) + fresh: queue.Queue = queue.Queue() # hours that closed after the plan was made; served before the backfill + retried: dict[datetime, float] = {} + + def replan(): + last = (datetime.now(timezone.utc) - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0) + h = newest_planned[0] + timedelta(hours=1) + while h <= last: + fresh.put(h); h += timedelta(hours=1) + if last > newest_planned[0]: + newest_planned[0] = last + # an hour fetched before the archive published it (404) or that failed is retried every 30 min for 12 h + try: + with transaction() as conn: + again = [r["hour"] for r in conn.execute("SELECT hour FROM replay_hours WHERE status IN ('missing','error') AND hour > now() - interval '12 hours'").fetchall()] + except Exception: + again = [] + for h in again: + if time.time() - retried.get(h, 0.0) >= 1800: + retried[h] = time.time(); fresh.put(h) + + def downloader(): + while not (stop is not None and stop.is_set()): + try: + h = fresh.get_nowait() + except queue.Empty: + try: + h = todo.get_nowait() + except queue.Empty: + return + tmp = tmpdir / f"{h:%Y%m%d%H}.zst" + for attempt in range(4): + try: + t0 = time.time(); size = _download(client, h, tmp, stop) + while not (stop is not None and stop.is_set()): + try: + ready.put((h, tmp, size, time.time() - t0), timeout=5.0); break + except queue.Full: + continue + break + except InterruptedError: + return + except Exception as e: + log.warning("download %s failed (%d): %s", h, attempt, e); time.sleep(10 * (attempt + 1)) + else: + while not (stop is not None and stop.is_set()): + try: + ready.put((h, None, 0, 0.0), timeout=5.0); break + except queue.Full: + continue + + if config.REPLAY_ASSEMBLE: # REPLAY_ASSEMBLE=0: a download-only run (e.g. refetch-pool-ids) leaves the builds alone + from ..train import replay_assemble + assembler = threading.Thread(target=replay_assemble.assemble_loop, args=(stop,), name="replay-assemble", daemon=True); assembler.start() + threads = [threading.Thread(target=downloader, name=f"replay-dl-{i}", daemon=True) for i in range(config.REPLAY_PARALLEL)] + for t in threads: + t.start() + try: + while not (stop is not None and stop.is_set()): + try: + h, tmp, size, dl_s = preloaded.pop(0) if preloaded else ready.get(timeout=5.0) + except queue.Empty: + replan() + if not any(t.is_alive() for t in threads) and ready.empty(): + if fresh.empty(): + _sleep(300, stop); replan() # caught up: poll for the next closed hour + if fresh.empty(): + continue + threads = [threading.Thread(target=downloader, name=f"replay-dl-{i}", daemon=True) for i in range(config.REPLAY_PARALLEL)] + for t in threads: + t.start() + continue + replan() + if tmp is None or size < 0: + _db_retry(stop, "replay_hours write", _put_hour, HOUR_SQL, (h, "missing" if size < 0 else "error", None if size < 0 else "download failed")) + status.update(errors=status.s["errors"] + (0 if size < 0 else 1)); continue + t0 = time.time() + try: + trades, events, n = parse_hour(tmp) + day = config.REPLAY_DIR / f"{h:%Y-%m-%d}"; day.mkdir(parents=True, exist_ok=True) + pq.write_table(trades, day / f"{h:%H}_trades.parquet", compression="zstd"); pq.write_table(events, day / f"{h:%H}_events.parquet", compression="zstd") + tmp.unlink() + _db_retry(stop, "replay_hours write", _put_hour, + "INSERT INTO replay_hours (hour, status, trades, events, bytes, took_s) VALUES (%s,'done',%s,%s,%s,%s) " + "ON CONFLICT (hour) DO UPDATE SET status = 'done', trades = EXCLUDED.trades, events = EXCLUDED.events, bytes = EXCLUDED.bytes, took_s = EXCLUDED.took_s, last_error = NULL, done_at = now()", + (h, trades.num_rows, events.num_rows, size, dl_s + time.time() - t0)) + status.update(hours_done=status.s["hours_done"] + 1, trades=status.s["trades"] + trades.num_rows, events=status.s["events"] + events.num_rows, + bytes=status.s["bytes"] + size, last_hour=h.isoformat(), last_parse_s=round(time.time() - t0, 1), last_dl_s=round(dl_s, 1)) + log.info("hour %s: %d events -> %d trade rows, %d lifecycle rows (%.0f MB, dl %.0fs, parse %.1fs)", h.isoformat(), n, trades.num_rows, events.num_rows, size / 1e6, dl_s, time.time() - t0) + except Exception as e: + log.exception("parse %s failed", h) + tmp.unlink(missing_ok=True) # the ~400 MB download; replan() refetches the hour + _db_retry(stop, "replay_hours write", _put_hour, HOUR_SQL, (h, "error", str(e)[:300])) + status.update(errors=status.s["errors"] + 1) + finally: + if stop is not None: + stop.set() + status.update(stage="stopped", force=True) + record_event("info", "replay", "replay ingest stopped", {k: status.s.get(k) for k in ("hours_done", "hours_total", "trades", "events", "bytes", "errors")}) + log.info("replay ingest stopped") diff --git a/fly_trader/logging_setup.py b/fly_trader/logging_setup.py new file mode 100644 index 0000000000000000000000000000000000000000..03d85296034230072cc2d69e67fb621d37d6d6b9 --- /dev/null +++ b/fly_trader/logging_setup.py @@ -0,0 +1,119 @@ +"""JSON-lines logging with a secret scrubber, safe for several workers in ONE process. + +Pattern from VOC dexlp/logging_setup.py. setup(name) is idempotent: the root logger gets one stderr +handler; each worker name gets a file handler (logs/.log) and an in-memory ring buffer (for +the console) that only accept records emitted from the thread named (or the main thread, for +the CLI). The scrubber removes API keys and any 87-88-char base58 string (a 64-byte secret key).""" +from __future__ import annotations + +import collections +import json +import logging +import re +import sys +import threading +from pathlib import Path + +from . import config + +_B58_SECRET = re.compile(r"\b[1-9A-HJ-NP-Za-km-z]{87,88}\b") +_API_KEY_QS = re.compile(r"(api[-_]?key=)[^&\s\"']+", re.IGNORECASE) +_API_KEY_HDR = re.compile(r"(['\"]?x-api-key['\"]?\s*[:=]\s*['\"]?)[^'\",\s}]+", re.IGNORECASE) +RING: dict[str, collections.deque] = {} +_configured: set[str] = set() +_lock = threading.Lock() + + +def scrub(text: str) -> str: + text = _B58_SECRET.sub("", text) + text = _API_KEY_QS.sub(r"\1", text) + text = _API_KEY_HDR.sub(r"\1", text) + for name in config.SECRET_ENV_NAMES: + val = config.env_str(name) + if val and len(val) >= 8 and val in text: + text = text.replace(val, "") + return text + + +class _ScrubFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + try: + msg = record.getMessage() + except Exception: + msg = str(record.msg) + record.msg = scrub(msg) + record.args = () + return True + + +class _ThreadFilter(logging.Filter): + """Keep records from the worker thread and its helper threads (``-*``); accept MainThread only when the worker itself + runs in the main thread (CLI use), never the console's Streamlit server thread.""" + + def __init__(self, name: str): + super().__init__() + self.name = name + import threading as _t + self.main_ok = _t.current_thread() is _t.main_thread() + + def filter(self, record: logging.LogRecord) -> bool: + tn = record.threadName or "" + return tn == self.name or tn.startswith(self.name + "-") or (self.main_ok and tn == "MainThread") + + +class _JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = {"ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S"), "level": record.levelname, + "logger": record.name, "thread": record.threadName, "msg": record.getMessage()} + if record.exc_info: + payload["exc"] = scrub(self.formatException(record.exc_info)) + return json.dumps(payload, ensure_ascii=False) + + +class _RingHandler(logging.Handler): + def __init__(self, name: str): + super().__init__() + RING.setdefault(name, collections.deque(maxlen=500)) + self.buf = RING[name] + + def emit(self, record: logging.LogRecord) -> None: + try: + self.buf.append(self.format(record)) + except Exception: + pass + + +def setup(name: str, level: int = logging.INFO, to_file: bool = True) -> logging.Logger: + with _lock: + root = logging.getLogger() + root.setLevel(level) + fmt = _JsonFormatter() + if "_root" not in _configured: + sh = logging.StreamHandler(sys.stderr) + sh.setFormatter(fmt) + sh.addFilter(_ScrubFilter()) + root.addHandler(sh) + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("websockets").setLevel(logging.WARNING) + logging.getLogger("watchdog").setLevel(logging.WARNING) + _configured.add("_root") + if name not in _configured: + tf = _ThreadFilter(name) + if to_file: + config.LOG_DIR.mkdir(parents=True, exist_ok=True) + fh = logging.FileHandler(Path(config.LOG_DIR) / f"{name}.log") + fh.setFormatter(fmt) + fh.addFilter(_ScrubFilter()) + fh.addFilter(tf) + root.addHandler(fh) + rh = _RingHandler(name) + rh.setFormatter(fmt) + rh.addFilter(_ScrubFilter()) + rh.addFilter(tf) + root.addHandler(rh) + _configured.add(name) + return logging.getLogger(name) + + +def tail(name: str, n: int = 60) -> str: + return "\n".join(list(RING.get(name, []))[-n:]) diff --git a/fly_trader/market/__init__.py b/fly_trader/market/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/market/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/market/exit_cost.py b/fly_trader/market/exit_cost.py new file mode 100644 index 0000000000000000000000000000000000000000..8188bc2c10fccb044a6e7818450b4f3810cbc25a --- /dev/null +++ b/fly_trader/market/exit_cost.py @@ -0,0 +1,87 @@ +"""Trading costs of a swap, as actually charged (measured 2026-09-15). + +- Pool fee. PumpSwap pools created by a pump.fun graduation charge a fee that steps down with the token's market cap + in SOL (``PUMP_FEE_TIERS``): measured from 157,059 archive trades (June–September 2026, ``poolFeeRate`` against + ``marketCapQuote``); the table reproduces the charged rate exactly on 99.7 % of them and is never more than one tier + off. Market cap = price × the token's supply (1 billion for pump.fun tokens, 2 billion in mayhem mode). The live + stream reports the charged rate itself, which is used when given (``pool_fee``). A token with unknown market cap is + charged the highest tier. +- Jupiter Ultra platform fee: 10 bps on every PumpSwap route (``feeBps`` of ``/order`` quotes with our key, from a + 58 SOL to a 169,000 SOL market cap; no age dependence). +- Network: the 5,000-lamport signature fee per transaction (Ultra quoted no priority or rent fees). +- Price impact. Constant-product pools (PumpSwap, Raydium v4/CPMM, Meteora DAMM): v / (v + R_q), v the traded value + and R_q the pool's quote reserve. Meteora DLMM: LAMBDA_IMPACT_DLMM · v / TVL, TVL ≈ 2·R_q. +""" +from __future__ import annotations + +import math + +import numpy as np + +from .. import config + +PUMP_SUPPLY = 1e9 +PUMP_FEE_TIERS = ((420, 0.0125), (1470, 0.012), (2461, 0.0115), (3440, 0.011), (4430, 0.0105), (9832, 0.01), (14744, 0.0095), (19649, 0.009), + (24519, 0.0085), (29616, 0.008), (34892, 0.0075), (39306, 0.007), (44800, 0.0065), (49054, 0.006), (54174, 0.0055), (58935, 0.0053), + (63864, 0.005), (69943, 0.0048), (73798, 0.0045), (78617, 0.0043), (83514, 0.004), (88407, 0.0038), (93311, 0.0035), (98178, 0.0033)) +PUMP_FEE_FLOOR = 0.003 # market cap above the last tier +JUPITER_FEE_BPS = 10 +TX_FEE_LAMPORTS = 5000 +_UPPER = np.array([u for u, _ in PUMP_FEE_TIERS], dtype=float) +_FEE = np.array([f for _, f in PUMP_FEE_TIERS] + [PUMP_FEE_FLOOR]) + + +def pool_fee_rate(mcap_sol): + """The pump.fun pool fee per swap for a token of this market cap (SOL); scalar or array.""" + if mcap_sol is None: + return float(_FEE[0]) + m = np.asarray(mcap_sol, dtype=float) + out = np.where(np.isfinite(m), _FEE[np.searchsorted(_UPPER, np.nan_to_num(m, nan=0.0), side="right")], _FEE[0]) + return float(out) if out.ndim == 0 else out + + +def impact_fraction(value_sol: float, res_quote_sol: float | None, program_label: str | None = None) -> float: + if value_sol <= 0: + return 0.0 + if res_quote_sol is None or res_quote_sol <= 0: + return 1.0 # unknown liquidity: assume the position cannot be exited + if program_label and any(lbl.lower() in program_label.lower() for lbl in ("dlmm",)): + return min(1.0, config.LAMBDA_IMPACT_DLMM * value_sol / (2.0 * res_quote_sol)) + return value_sol / (value_sol + res_quote_sol) + + +def fee_fraction(value_sol: float, mcap_sol: float | None, pool_fee: float | None = None) -> float: + """Fees of one swap of ``value_sol`` as a fraction of it: pool fee + Jupiter's platform fee + the network fee.""" + if value_sol <= 0: + return 0.0 + pf = pool_fee if pool_fee is not None and math.isfinite(pool_fee) and pool_fee > 0 else pool_fee_rate(mcap_sol) + return pf + JUPITER_FEE_BPS / 1e4 + TX_FEE_LAMPORTS / config.LAMPORTS_PER_SOL / value_sol + + +def exit_cost_fraction(value_sol: float, res_quote_sol: float | None, mcap_sol: float | None, program_label: str | None = None, + pool_fee: float | None = None) -> float: + """Everything a swap of ``value_sol`` loses, as a fraction of it: fees plus price impact.""" + if value_sol <= 0: + return 0.0 + return min(1.0, fee_fraction(value_sol, mcap_sol, pool_fee) + impact_fraction(value_sol, res_quote_sol, program_label)) + + +def cost_at_size(ec_0p1, res_quote_sol, size_sol: float | None = None): + """``exit_cost_0p1`` (priced for a fixed 0.1 SOL in market/features.py) repriced for the size the book actually + trades, as a fraction of one side. + + Positions are Kelly-sized up to ``MAX_POSITION_FRACTION`` of the bankroll and impact is ``value/(value+reserves)``, + so the stored feature understates a real exit. The fee part (pool, platform, network) is recovered from the stored + value and only impact recomputed, so a feature part never has to be rebuilt. ``size_sol`` defaults to the sizing + rule's own ceiling and is capped by ``MAX_POOL_SHARE`` of the pool, exactly as agent/sizing.py caps a live buy. + Unknown liquidity costs 1.0: a position that cannot be exited. Takes scalars or arrays.""" + ec = np.clip(np.asarray(ec_0p1, dtype=float), 0.0, 1.0) + r = np.asarray(res_quote_sol, dtype=float) + ok = np.isfinite(r) & (r > 0) + want = size_sol if size_sol is not None else (config.LABEL_SIZE_SOL or config.MAX_POSITION_FRACTION * config.CAPITAL_SOL) + sz = np.maximum(np.minimum(want, config.MAX_POOL_SHARE * np.where(ok, r, 0.0)), 1e-9) + tx = TX_FEE_LAMPORTS / config.LAMPORTS_PER_SOL + fees01 = np.clip(ec - np.where(ok, 0.1 / (0.1 + r), 1.0), 0.0, 1.0) # the stored value minus its own impact term + fees = np.clip(fees01 - tx / 0.1 + tx / sz, 0.0, 1.0) # the network fee is per trade, not per SOL + out = np.where(ok, np.clip(fees + np.where(ok, sz / (sz + r), 1.0), 0.0, 1.0), 1.0) + return float(out) if np.ndim(ec_0p1) == 0 else out diff --git a/fly_trader/market/features.py b/fly_trader/market/features.py new file mode 100644 index 0000000000000000000000000000000000000000..0411b87da6cd4aa47b3700c1d0c75d0e5ea88300 --- /dev/null +++ b/fly_trader/market/features.py @@ -0,0 +1,272 @@ +"""Per-token rolling state and the feature vector, one engine for training (``train/mature.py``) and trading. + +Feature definitions follow VOC dexlp/env/obs_spec.py (multi-timeframe momentum / realized vol / +imbalance / toxicity, Hawkes burstiness, drawdown, overextension, exit cost) restricted to the +token-intrinsic subset; LP-position slots are dropped. Every feature is standardized downstream +(robust scaling, ``train/scaling.py``), so scales only need to be consistent. + +Data structures: append-only Python lists with prefix sums, so each window statistic is O(log n) +via bisect; window highs use per-block maxima; unique-signer counts use sliding counters with eviction +pointers. Lists are compacted when they hold more than twice the 3 h window. +""" +from __future__ import annotations + +import math +from bisect import bisect_left +from collections import Counter +from dataclasses import dataclass, field + +from .. import config +from .exit_cost import PUMP_SUPPLY, exit_cost_fraction + +WINDOWS = {"1m": 60.0, "5m": 300.0, "15m": 900.0, "1h": 3600.0, "3h": 10800.0} +HAWKES_BETA = 1.0 / 60.0 # per second; intensity decays with a 1-minute time constant +EWMA_1H_TAU_S = 3600.0 # 1-hour time constant of the price EWMA (continuous time, seconds) +EXIT_COST_SIZE_SOL = 0.1 # exit_cost_0p1 is priced for a fixed 0.1 SOL position (not the configured size) +HIGH_BLOCK = 256 # block size of the running window-high index +FEATURE_VERSION = 3 # bump on any change to feature values; stored feature parts from another version are rebuilt + # 3: exit_cost_0p1 at the real fees (pool fee tier by market cap, Jupiter 10 bps, network fee) + +FEATURES: list[str] = [ + "ret_1m", "ret_5m", "ret_15m", "ret_1h", "ret_3h", + "rvol_5m", "rvol_15m", "rvol_1h", "rvol_3h", + "imb_1m", "imb_5m", "imb_15m", "imb_1h", + "logvol_1m", "logvol_5m", "logvol_15m", "logvol_1h", + "logn_1m", "logn_5m", "logn_15m", "logn_1h", + "logsigners_15m", "logsigners_1h", + "hawkes", "log_since_last", "log_liquidity_sol", "log_age_h", + "dd_1h", "dd_3h", "overext_1h", "vpin_15m", "mom_decel", "vol_divergence", "exit_cost_0p1", + "organic_score", "log_holders", "log_liq_usd", "top_holders_pct", "dev_balance_pct", + "is_sus", "is_verified", "net_buyers_1h", "holder_change_1h", "price_change_24h", "token2022", +] +D = len(FEATURES) +FIDX = {n: i for i, n in enumerate(FEATURES)} +STATS_FEATURES = FEATURES[FIDX["organic_score"]:] +STATS_BIT = 1 << 62 # beyond any feature index; feature_mask is a bigint + + +@dataclass +class TokenMeta: + mint: str + pool: str | None = None + program_label: str | None = None + graduated_at: float | None = None # epoch seconds + token2022: bool = False + stats: dict | None = None # latest token_stats row (dict) or None + supply: float = PUMP_SUPPLY # token supply: market cap = price × supply sets the pool fee tier + + +@dataclass +class TokenState: + mint: str + ts: list[float] = field(default_factory=list) + logp: list[float] = field(default_factory=list) + cum_vol: list[float] = field(default_factory=lambda: [0.0]) + cum_buy: list[float] = field(default_factory=lambda: [0.0]) + cum_sq: list[float] = field(default_factory=lambda: [0.0]) + signers: list[int] = field(default_factory=list) + hawkes: float = 0.0 + last_ts: float | None = None + last_price: float | None = None + last_res_quote_sol: float | None = None + ewma_1h: float | None = None + blk_max: list[float] = field(default_factory=list) # max log price of each HIGH_BLOCK-entry block of logp (window highs) + sig_ptr: dict[str, int] = field(default_factory=lambda: {"15m": 0, "1h": 0}) + sig_cnt: dict[str, Counter] = field(default_factory=lambda: {"15m": Counter(), "1h": Counter()}) + n_since_compact: int = 0 + + # -- ingestion -- + def append(self, ts: float, price: float, sol_vol: float, is_buy: bool, signer: str | None, + res_quote_sol: float | None) -> None: + if price is None or price <= 0 or not math.isfinite(price): + return + lp = math.log(price) + if self.last_ts is not None: + dt = max(0.0, ts - self.last_ts) + self.hawkes = self.hawkes * math.exp(-HAWKES_BETA * dt) + 1.0 + dlp = lp - self.logp[-1] + self.cum_sq.append(self.cum_sq[-1] + dlp * dlp) + w = 1.0 - math.exp(-dt / EWMA_1H_TAU_S) + self.ewma_1h = (1 - w) * self.ewma_1h + w * price if self.ewma_1h is not None else price + else: + self.hawkes = 1.0 + self.cum_sq.append(0.0) + self.ewma_1h = price + self.ts.append(ts) + self.logp.append(lp) + if (len(self.logp) - 1) % HIGH_BLOCK == 0: + self.blk_max.append(lp) + elif lp > self.blk_max[-1]: + self.blk_max[-1] = lp + self.cum_vol.append(self.cum_vol[-1] + sol_vol) + self.cum_buy.append(self.cum_buy[-1] + (sol_vol if is_buy else 0.0)) + h = hash(signer) if signer else 0 + self.signers.append(h) + for k in self.sig_cnt: + self.sig_cnt[k][h] += 1 + self.last_ts = ts + self.last_price = price + if res_quote_sol is not None and res_quote_sol > 0: + self.last_res_quote_sol = res_quote_sol + self.n_since_compact += 1 + if self.n_since_compact > 20000 and len(self.ts) > 2 * self._count_since(ts - WINDOWS["3h"]): + self._compact(ts - WINDOWS["3h"]) + + def _count_since(self, t0: float) -> int: + return len(self.ts) - bisect_left(self.ts, t0) + + def _compact(self, t0: float) -> None: + i = max(0, bisect_left(self.ts, t0) - 1) # keep the last trade before the window: it is the window's start price + if i <= 0: + self.n_since_compact = 0 + return + base_vol, base_buy, base_sq = self.cum_vol[i], self.cum_buy[i], self.cum_sq[i] + for k in self.sig_ptr: # entries dropped before the window pointer reached them still count: retire them + cnt = self.sig_cnt[k] + for j in range(self.sig_ptr[k], i): + h = self.signers[j]; cnt[h] -= 1 + if cnt[h] <= 0: + del cnt[h] + self.ts = self.ts[i:] + self.logp = self.logp[i:] + self.signers = self.signers[i:] + self.cum_vol = [x - base_vol for x in self.cum_vol[i:]] + self.cum_buy = [x - base_buy for x in self.cum_buy[i:]] + self.cum_sq = [x - base_sq for x in self.cum_sq[i:]] + self.blk_max = [max(self.logp[j:j + HIGH_BLOCK]) for j in range(0, len(self.logp), HIGH_BLOCK)] + for k in self.sig_ptr: + self.sig_ptr[k] = max(0, self.sig_ptr[k] - i) + self.n_since_compact = 0 + + # -- queries -- + def _advance_signers(self, key: str, now: float) -> None: + t0 = now - WINDOWS[key] + ptr = self.sig_ptr[key] + cnt = self.sig_cnt[key] + while ptr < len(self.ts) and self.ts[ptr] < t0: + h = self.signers[ptr] + cnt[h] -= 1 + if cnt[h] <= 0: + del cnt[h] + ptr += 1 + self.sig_ptr[key] = ptr + + def high_from(self, i: int) -> float: + """Max log price of entries i..n-1 (i < n): the head of i's block, then whole-block maxima.""" + b = i // HIGH_BLOCK + 1 + hi = max(self.logp[i:b * HIGH_BLOCK]) + return max(hi, max(self.blk_max[b:])) if b < len(self.blk_max) else hi + + def volume_since(self, t0: float) -> float: + i = bisect_left(self.ts, t0) + return self.cum_vol[-1] - self.cum_vol[i] + + def rvol(self, now: float, window_s: float = 900.0) -> float: + """Realized volatility (root of summed squared log price changes) over the window.""" + i = bisect_left(self.ts, now - window_s) + return math.sqrt(max(0.0, self.cum_sq[-1] - self.cum_sq[i])) if self.ts else 0.0 + + def price_at(self, t: float) -> float | None: + """Last price at or before t (None if no swap that early).""" + i = bisect_left(self.ts, t + 1e-9) + if i == 0: + return None + return math.exp(self.logp[i - 1]) + + def features(self, now: float, meta: TokenMeta) -> tuple[list[float], int]: + f = [0.0] * D + mask = 0 + n = len(self.ts) + if n == 0 or self.last_price is None: + return f, mask + lp_now = self.logp[-1] + p_now = self.last_price + idx = {k: bisect_left(self.ts, now - w) for k, w in WINDOWS.items()} + avail = {k: idx[k] < n for k in WINDOWS} + # returns: price at window start vs now (needs a price before the window start) + for k in ("1m", "5m", "15m", "1h", "3h"): + i = idx[k] + if i > 0: + f[FIDX[f"ret_{k}"]] = lp_now - self.logp[i - 1] + mask |= 1 << FIDX[f"ret_{k}"] + elif n > 1: + f[FIDX[f"ret_{k}"]] = lp_now - self.logp[0] # partial history; still informative + mask |= 1 << FIDX[f"ret_{k}"] + for k in ("5m", "15m", "1h", "3h"): + i = idx[k] + if avail[k]: + f[FIDX[f"rvol_{k}"]] = math.sqrt(max(0.0, self.cum_sq[-1] - self.cum_sq[i])) + mask |= 1 << FIDX[f"rvol_{k}"] + for k in ("1m", "5m", "15m", "1h"): + i = idx[k] + vol = self.cum_vol[-1] - self.cum_vol[i] + buy = self.cum_buy[-1] - self.cum_buy[i] + cnt = n - i + f[FIDX[f"imb_{k}"]] = (2 * buy - vol) / (vol + 1e-9) if vol > 0 else 0.0 + f[FIDX[f"logvol_{k}"]] = math.log1p(vol) + f[FIDX[f"logn_{k}"]] = math.log1p(cnt) + mask |= (1 << FIDX[f"imb_{k}"]) | (1 << FIDX[f"logvol_{k}"]) | (1 << FIDX[f"logn_{k}"]) + for k in ("15m", "1h"): + self._advance_signers(k, now) + f[FIDX[f"logsigners_{k}"]] = math.log1p(len(self.sig_cnt[k])) + mask |= 1 << FIDX[f"logsigners_{k}"] + dt = max(0.0, now - self.last_ts) + f[FIDX["hawkes"]] = self.hawkes * math.exp(-HAWKES_BETA * dt) + f[FIDX["log_since_last"]] = math.log1p(dt) + mask |= (1 << FIDX["hawkes"]) | (1 << FIDX["log_since_last"]) + liq = self.last_res_quote_sol + if liq is not None: + f[FIDX["log_liquidity_sol"]] = math.log1p(2.0 * liq) + mask |= 1 << FIDX["log_liquidity_sol"] + age_h = None + if meta.graduated_at is not None: + age_h = max(0.0, (now - meta.graduated_at) / 3600.0) + f[FIDX["log_age_h"]] = math.log1p(age_h) + mask |= 1 << FIDX["log_age_h"] + # drawdown from window highs; overextension vs EWMA + for k in ("1h", "3h"): + i = idx[k] + if avail[k]: + f[FIDX[f"dd_{k}"]] = lp_now - self.high_from(i) + mask |= 1 << FIDX[f"dd_{k}"] + if self.ewma_1h: + f[FIDX["overext_1h"]] = p_now / self.ewma_1h - 1.0 + mask |= 1 << FIDX["overext_1h"] + # VPIN-like toxicity: mean |imbalance| over five 3-minute buckets in the last 15 min + i15 = idx["15m"] + if avail["15m"]: + tox, nb = 0.0, 0 + for b in range(5): + t_a, t_b = now - 900 + 180 * b, now - 900 + 180 * (b + 1) + ia, ib = bisect_left(self.ts, t_a), bisect_left(self.ts, t_b) + v = self.cum_vol[ib] - self.cum_vol[ia] + if v > 0: + bv = self.cum_buy[ib] - self.cum_buy[ia] + tox += abs(2 * bv - v) / v + nb += 1 + f[FIDX["vpin_15m"]] = tox / nb if nb else 0.0 + mask |= 1 << FIDX["vpin_15m"] + f[FIDX["mom_decel"]] = f[FIDX["ret_1m"]] - f[FIDX["ret_5m"]] / 5.0 + f[FIDX["vol_divergence"]] = math.log1p((self.cum_vol[-1] - self.cum_vol[idx["5m"]]) / 5.0) - \ + math.log1p((self.cum_vol[-1] - self.cum_vol[idx["1h"]]) / 60.0) + mask |= (1 << FIDX["mom_decel"]) | (1 << FIDX["vol_divergence"]) + f[FIDX["exit_cost_0p1"]] = exit_cost_fraction(EXIT_COST_SIZE_SOL, liq, p_now * meta.supply, meta.program_label) # real fees at this market cap + mask |= 1 << FIDX["exit_cost_0p1"] + # Jupiter token stats + st = meta.stats + if st: + f[FIDX["organic_score"]] = (st.get("organic_score") or 0.0) / 100.0 + f[FIDX["log_holders"]] = math.log1p(st.get("holder_count") or 0) + f[FIDX["log_liq_usd"]] = math.log1p(st.get("liquidity_usd") or 0.0) + f[FIDX["top_holders_pct"]] = float(st.get("top_holders_pct") or 0.0) + f[FIDX["dev_balance_pct"]] = float(st.get("dev_balance_pct") or 0.0) + f[FIDX["is_sus"]] = 1.0 if st.get("is_sus") else 0.0 + f[FIDX["is_verified"]] = 1.0 if st.get("is_verified") else 0.0 + s1 = (st.get("stats") or {}).get("stats1h") or {} + s24 = (st.get("stats") or {}).get("stats24h") or {} + f[FIDX["net_buyers_1h"]] = float(s1.get("numNetBuyers") or 0.0) + f[FIDX["holder_change_1h"]] = float(s1.get("holderChange") or 0.0) + f[FIDX["price_change_24h"]] = float(s24.get("priceChange") or 0.0) + mask |= STATS_BIT + f[FIDX["token2022"]] = 1.0 if meta.token2022 else 0.0 + return f, mask diff --git a/fly_trader/ops/__init__.py b/fly_trader/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/ops/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/ops/procs.py b/fly_trader/ops/procs.py new file mode 100644 index 0000000000000000000000000000000000000000..c71c66e1019de891b65aca99382e06520e84ca50 --- /dev/null +++ b/fly_trader/ops/procs.py @@ -0,0 +1,141 @@ +"""Worker processes: started detached (their own session) so Streamlit reruns/restarts never kill them. +PIDs and commands are recorded in `processes`; liveness is checked with psutil, not trusted from the row.""" +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import psutil + +from .. import config +from ..db.apilog import record_event +from ..db.connection import connect, transaction + +WORKERS: dict[str, list[str]] = { + "discover": ["discover"], + "runner": ["run"], + "train": ["train-selector"], + "replay": ["replay-pull"], + "pumpstream": ["pumpstream"], +} +PYTHON = str(config.REPO_ROOT / ".venv" / "bin" / "python") + + +def _cmd(name: str, extra: list[str] | None = None) -> list[str]: + return [PYTHON, "-m", "fly_trader", *WORKERS[name], *(extra or [])] + + +def _is_ours(pid: int, name: str) -> bool: + try: + p = psutil.Process(pid) + cl = " ".join(p.cmdline()) + return "fly_trader" in cl and WORKERS[name][0] in cl and p.status() != psutil.STATUS_ZOMBIE + except (psutil.NoSuchProcess, psutil.AccessDenied): + return False + + +def reconcile(conn) -> None: + """Mark dead EXTERNAL worker processes stopped. Thread rows (cmd[0] == 'thread') belong to the console's + Supervisor and are left alone.""" + rows = conn.execute("SELECT id, name, pid, cmd FROM processes WHERE stopped_at IS NULL").fetchall() + for r in rows: + if r["cmd"] and r["cmd"][0] == "thread": + continue + if not _is_ours(r["pid"], r["name"]): + conn.execute("UPDATE processes SET stopped_at = now(), exit_code = COALESCE(exit_code, -1) WHERE id = %s", (r["id"],)) + + +def alive(name: str) -> dict | None: + with transaction() as conn: + reconcile(conn) + r = conn.execute("SELECT * FROM processes WHERE name = %s AND stopped_at IS NULL AND cmd[1] <> 'thread' " + "ORDER BY started_at DESC LIMIT 1", (name,)).fetchone() + return dict(r) if r else None + + +def start(name: str, extra: list[str] | None = None, started_by: str = "console") -> int: + if name not in WORKERS: + raise ValueError(f"unknown worker {name}") + if (a := alive(name)) is not None: + return int(a["pid"]) + with transaction() as conn: # a console thread already running this worker: never launch a second copy + t = conn.execute("SELECT pid FROM processes WHERE name = %s AND stopped_at IS NULL AND cmd[1] = 'thread'", (name,)).fetchone() + if t: + raise RuntimeError(f"{name} is already running as a console thread (pid {t['pid']}); stop it there first") + config.LOG_DIR.mkdir(parents=True, exist_ok=True) + log_path = config.LOG_DIR / f"{name}.out.log" + env = dict(os.environ, PYTHONUNBUFFERED="1") + cmd = _cmd(name, extra) + with open(log_path, "ab") as out: + proc = subprocess.Popen(cmd, cwd=str(config.REPO_ROOT), stdout=out, stderr=subprocess.STDOUT, + start_new_session=True, env=env) + try: + with transaction() as conn: + conn.execute("INSERT INTO processes (name, pid, cmd, log_path, started_by) VALUES (%s,%s,%s,%s,%s)", + (name, proc.pid, cmd, str(log_path), started_by)) + except Exception as e: # e.g. processes_live_uniq raced by another starter: do not leave an unregistered worker running + proc.terminate() + raise RuntimeError(f"cannot register {name}: {type(e).__name__}: {e}") from e + record_event("info", "procs", f"started {name}", {"pid": proc.pid, "cmd": cmd}) + return proc.pid + + +def stop(name: str, grace_s: float = 30.0) -> bool: + a = alive(name) + if a is None: + return False + pid = int(a["pid"]) + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + t0 = time.monotonic() + while time.monotonic() - t0 < grace_s and _is_ours(pid, name): + time.sleep(0.5) + killed = False + if _is_ours(pid, name): + os.kill(pid, signal.SIGKILL) + killed = True + with transaction() as conn: + conn.execute("UPDATE processes SET stopped_at = now(), exit_code = %s WHERE id = %s", (-9 if killed else 0, a["id"])) + record_event("info", "procs", f"stopped {name}" + (" (SIGKILL)" if killed else ""), {"pid": pid}) + return True + + +def tail_log(name: str, lines: int = 60) -> str: + p = config.LOG_DIR / f"{name}.out.log" + if not p.exists(): + return "" + with open(p, "rb") as f: + f.seek(0, 2) + size = f.tell() + f.seek(max(0, size - 64000)) + data = f.read().decode("utf-8", "replace") + return "\n".join(data.splitlines()[-lines:]) + + +def run_streamlit(port: int = 8501) -> None: + app = config.REPO_ROOT / "fly_trader" / "ui" / "app.py" + cmd = [PYTHON, "-m", "streamlit", "run", str(app), "--server.port", str(port), "--server.headless", "true", + "--browser.gatherUsageStats", "false"] + os.execv(PYTHON, cmd) + + +def worker_cli(action: str, name: str | None) -> None: + if action == "status": + for n in WORKERS: + a = alive(n) + print(f"{n:10s} {'alive pid=' + str(a['pid']) if a else 'stopped'}") + return + if not name: + raise SystemExit("worker name required") + if action == "start": + print(f"{name} pid={start(name, started_by='cli')}") + elif action == "stop": + print(f"{name} stopped={stop(name)}") diff --git a/fly_trader/ops/reset.py b/fly_trader/ops/reset.py new file mode 100644 index 0000000000000000000000000000000000000000..b1540cb685e01e4934365be78fe59c7c3175d01d --- /dev/null +++ b/fly_trader/ops/reset.py @@ -0,0 +1,141 @@ +"""Reset the training state so a fresh runner never carries over a bad run — without touching the race or the fly. + +``reset_training_state`` (every runner start, ``RESET_ON_START``) wipes, after archiving them as CSV under +data/pg_archive/reset_/, what belongs to no book that must persist: positions/orders/fills and wealth marks of any +book but the live book and the two race books (``PRESERVED_BOOKS``), decisions and beats of runs that are not the race +sessions' (``RACE_RUN_KINDS``), those runs, and brain snapshots that are not models (``MODEL_KINDS``). NEVER touched: +the race books (paper_selector, paper_fly) and the live book with their runs, beats and decisions; the plastic fly's +tables (fly_*) and state file; the circuit (kill switch, trip, failure count, peak wealth — only rails.reset_circuit +re-arms it); wallet events, tokens, pools, market data, API logs, events, other ui_settings. + +``reset_fly`` is the explicit console action: the plastic fly starts again from its bootstrap (its book, its learned +KC→MBON changes, scored minutes, calibrations, rollbacks and snapshots are archived and cleared; the bootstrap is kept). + +``reset_training_stats`` is the training-side counterpart, run at the start of every training run: the previous runs' +walk-forward / iteration events and the console's training status are archived and cleared (models are kept). +""" +from __future__ import annotations + +import logging +import shutil +from datetime import datetime, timezone +from pathlib import Path + +from .. import config +from ..db.apilog import record_event +from ..db.connection import transaction + +log = logging.getLogger(__name__) + +PRESERVED_BOOKS = ("live", "paper_selector", "paper_fly") +RACE_RUN_KINDS = ("selector", "fly") +MODEL_KINDS = ("selector", "fly_selector", "fly_plastic") +FLY_BOOK, FLY_RUN_KIND = "paper_fly", "fly" +FLY_TABLES = ["fly_scored", "fly_calibrations", "fly_updates", "fly_rollbacks"] +BOOK_TABLES = ["positions", "orders", "fills"] +TRAINING_EVENTS = {"selector": "(source = 'selector' AND (message LIKE 'walk-forward%' OR message LIKE 'selector saved%'))", # per regimen: a fly + "fly_selector": "source = 'fly_selector'"} # run keeps the selector's + + +def _sql_list(xs) -> str: + return "(" + ",".join(f"'{x}'" for x in xs) + ")" + + +_BOOKS, _KINDS, _RACE = _sql_list(PRESERVED_BOOKS), _sql_list(MODEL_KINDS), _sql_list(RACE_RUN_KINDS) +_RACE_RUNS = f"SELECT run_id FROM runs WHERE kind IN {_RACE}" +_KEPT_DECISIONS = (f"SELECT decision_id FROM orders WHERE book IN {_BOOKS} AND decision_id IS NOT NULL " + f"UNION SELECT entry_decision_id FROM positions WHERE book IN {_BOOKS} AND entry_decision_id IS NOT NULL " + f"UNION SELECT exit_decision_id FROM positions WHERE book IN {_BOOKS} AND exit_decision_id IS NOT NULL " + f"UNION SELECT id FROM decisions WHERE run_id IN ({_RACE_RUNS})") +_LIVE_DECISIONS = ("SELECT decision_id FROM orders WHERE book = 'live' AND decision_id IS NOT NULL " + "UNION SELECT entry_decision_id FROM positions WHERE book = 'live' AND entry_decision_id IS NOT NULL " + "UNION SELECT exit_decision_id FROM positions WHERE book = 'live' AND exit_decision_id IS NOT NULL") + + +def _archive(conn, table: str, where: str, out_dir: Path) -> int: + n = conn.execute(f"SELECT count(*) AS n FROM {table} {where}").fetchone()["n"] + if n: + with open(out_dir / f"{table}.csv", "w") as f: + with conn.cursor() as cur: + with cur.copy(f"COPY (SELECT * FROM {table} {where}) TO STDOUT WITH CSV HEADER") as copy: + for chunk in copy: + f.write(bytes(chunk).decode("utf-8")) + return int(n) + + +def _wipe(conn, plan: list[tuple[str, str]], out_dir: Path | None) -> dict[str, int]: + """Archive (when ``out_dir``) then delete each (table, WHERE clause); in order, so later clauses may read earlier tables.""" + counts: dict[str, int] = {} + if out_dir is not None: + out_dir.mkdir(parents=True, exist_ok=True) + for t, where in plan: + counts[t] = counts.get(t, 0) + _archive(conn, t, where, out_dir) + for t, where in plan: + conn.execute(f"DELETE FROM {t} {where}") + return counts + + +def reset_training_state(reason: str = "runner start", archive: bool = True) -> dict: + out_dir = config.PG_ARCHIVE_DIR / f"reset_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" + plan = [("decisions", f"WHERE id NOT IN ({_KEPT_DECISIONS})"), + ("beats", f"WHERE run_id IS NULL OR run_id NOT IN ({_RACE_RUNS})"), + ("wealth_marks", f"WHERE book NOT IN {_BOOKS}"), + ("brain_snapshots", f"WHERE kind NOT IN {_KINDS}"), + *[(t, f"WHERE book NOT IN {_BOOKS}") for t in BOOK_TABLES], + ("runs", f"WHERE kind NOT IN ('connectome_build','calibration') AND kind NOT IN {_RACE}")] + with transaction() as conn: + counts = _wipe(conn, plan, out_dir if archive else None) + conn.execute(f"UPDATE brain_state SET live_snapshot_id = CASE WHEN (SELECT kind FROM brain_snapshots WHERE id = live_snapshot_id) IN {_KINDS} THEN live_snapshot_id END, " + f"pending_snapshot_id = CASE WHEN (SELECT kind FROM brain_snapshots WHERE id = pending_snapshot_id) IN {_KINDS} THEN pending_snapshot_id END, updated_at = now() WHERE singleton") + conn.execute("DELETE FROM ui_settings WHERE key = 'selector_status'") + conn.execute("INSERT INTO circuit_events (kind, detail) VALUES ('training_reset', %s)", (f'{{"reason": "{reason}"}}',)) + record_event("info", "reset", f"training state reset ({reason})", {"archived_to": str(out_dir) if archive else None, "rows": counts}) + log.info("training state reset (%s): %s", reason, counts) + return {"archived_to": str(out_dir) if archive else None, "rows": counts} + + +def fly_state_dir() -> Path: + return config.BRAIN_DIR / "plastic" + + +def reset_fly(reason: str = "operator", archive: bool = True) -> dict: + """The plastic fly starts again from its bootstrap: its paper book, sessions, learned state and statistics are + archived and cleared. The live book, the selector's book and the bootstrap snapshots are kept.""" + out_dir = config.PG_ARCHIVE_DIR / f"reset_fly_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" + fly_runs = f"SELECT run_id FROM runs WHERE kind = '{FLY_RUN_KIND}'" + plan = [("decisions", f"WHERE run_id IN ({fly_runs}) AND id NOT IN ({_LIVE_DECISIONS})"), + ("beats", f"WHERE run_id IN ({fly_runs})"), + ("wealth_marks", f"WHERE book = '{FLY_BOOK}'"), + *[(t, f"WHERE book = '{FLY_BOOK}'") for t in BOOK_TABLES], + ("brain_snapshots", "WHERE kind = 'fly_plastic'"), + ("runs", f"WHERE kind = '{FLY_RUN_KIND}'"), + *[(t, "") for t in FLY_TABLES]] + with transaction() as conn: + counts = _wipe(conn, plan, out_dir if archive else None) + conn.execute("DELETE FROM book_state WHERE book = %s", (FLY_BOOK,)) + conn.execute("DELETE FROM ui_settings WHERE key = 'fly_status'") + d = fly_state_dir() + if d.exists(): + if archive: + out_dir.mkdir(parents=True, exist_ok=True); shutil.move(str(d), str(out_dir / "plastic")) + else: + shutil.rmtree(d) + record_event("warning", "reset", f"plastic fly reset to its bootstrap ({reason})", {"archived_to": str(out_dir) if archive else None, "rows": counts}) + log.info("plastic fly reset (%s): %s", reason, counts) + return {"archived_to": str(out_dir) if archive else None, "rows": counts} + + +def reset_training_stats(regimen: str, reason: str = "training start", archive: bool = True) -> dict: + """Clear the previous runs' statistics of this training ``regimen`` (selector walk-forward, fly comparisons; the console's training status) so a new run's numbers are never shown next to an old run's. + Other regimens' results and all trained models (brain_snapshots) are kept.""" + where = TRAINING_EVENTS[regimen] + out_dir = config.PG_ARCHIVE_DIR / f"training_{regimen}_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" + with transaction() as conn: + n = 0 + if archive: + out_dir.mkdir(parents=True, exist_ok=True) + n = _archive(conn, "events", "WHERE " + where, out_dir) + conn.execute("DELETE FROM events WHERE " + where) + conn.execute("DELETE FROM ui_settings WHERE key = 'training_status'") + log.info("training stats reset (%s): %d events", reason, n) + return {"archived_to": str(out_dir) if archive else None, "events": n} diff --git a/fly_trader/ops/supervisor.py b/fly_trader/ops/supervisor.py new file mode 100644 index 0000000000000000000000000000000000000000..610c02d16ed3cc5a692b75fcde5ecc563241beea --- /dev/null +++ b/fly_trader/ops/supervisor.py @@ -0,0 +1,223 @@ +"""In-process supervisor: every worker is a thread inside the console process. Nothing is spawned. + +The Supervisor is a process-wide singleton (get_supervisor()) shared by `fly-trader ui` (which +starts autostart workers before handing the main thread to Streamlit) and by the Streamlit script +runs (which import the same module). Each worker runs its ordinary entry point with a stop event; +stopping sets the event and joins. Start/stop are recorded in `processes` (pid = this process, +cmd = ['thread', name]) and `events`, so the audit trail is unchanged. +""" +from __future__ import annotations + +import logging +import os +import threading +import time +import traceback +from datetime import datetime, timezone + +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import scrub, setup, tail + +log = logging.getLogger(__name__) + + +def _entry(name: str): + if name == "discover": + from ..ingest import discovery + return discovery.run_forever + if name == "runner": + from ..agent import runner + return runner.main + if name == "train": # the training pipeline: selector, then the fly imitating it, every 7 days (train/pipeline.py) + from ..train import pipeline + return pipeline.main + if name == "replay": + from ..ingest import replay_pull + return replay_pull.main + if name == "pumpstream": + from ..ingest import pumpstream + return pumpstream.main + raise KeyError(name) + + +WORKERS = ("discover", "runner", "train", "replay", "pumpstream") + + +class Supervisor: + def __init__(self): + self.lock = threading.Lock() + self.threads: dict[str, threading.Thread] = {} + self.stops: dict[str, threading.Event] = {} + self.started_at: dict[str, datetime] = {} + self.stopped_at: dict[str, datetime] = {} + self.errors: dict[str, str] = {} + self.row_ids: dict[str, int] = {} + self.autostarted = False + self.pid = os.getpid() + setup("console") + with transaction() as conn: # thread rows left by a previous console process are stale + live_pids = {p.pid for p in __import__("psutil").process_iter()} + rows = conn.execute("SELECT id, pid FROM processes WHERE stopped_at IS NULL AND cmd[1] = 'thread'").fetchall() + for r in rows: + # A row with our own pid is a previous console's too: this one has registered nothing yet. In a container + # the console is always pid 1, so without this every restart refused to register its workers. + if int(r["pid"]) == self.pid or int(r["pid"]) not in live_pids: + conn.execute("UPDATE processes SET stopped_at = now(), exit_code = -1 WHERE id = %s", (r["id"],)) + + # ---- queries ---- + def alive(self, name: str) -> bool: + t = self.threads.get(name) + return bool(t and t.is_alive()) + + def status(self) -> dict[str, dict]: + out = {} + for n in WORKERS: + out[n] = {"alive": self.alive(n), "started_at": self.started_at.get(n), "stopped_at": self.stopped_at.get(n), + "error": self.errors.get(n), "stopping": bool(self.stops.get(n) and self.stops[n].is_set() and self.alive(n))} + return out + + def external(self) -> dict[str, dict]: + """Worker processes running OUTSIDE this app (from the CLI); the console refuses to double-run them.""" + from . import procs + out = {} + for n in WORKERS: + a = procs.alive(n) + if a and int(a["pid"]) != self.pid: + out[n] = a + return out + + # ---- control ---- + def start(self, name: str, started_by: str = "console") -> bool: + if name not in WORKERS: + raise KeyError(name) + with self.lock: + if self.alive(name): + return False + if name in self.external(): + raise RuntimeError(f"{name} is already running as an external process; stop it first") + try: + fn = _entry(name) # imports first: a broken import must not leave a registered row + except Exception as e: + raise RuntimeError(f"cannot load {name}: {type(e).__name__}: {scrub(str(e))[:200]}") from e + try: + with transaction() as conn: + row = conn.execute("INSERT INTO processes (name, pid, cmd, log_path, started_by) VALUES (%s,%s,%s,%s,%s) RETURNING id", + (name, self.pid, ["thread", name], f"logs/{name}.log", started_by)).fetchone() + except Exception as e: # e.g. processes_live_uniq: another console (or a stale row) owns this worker + raise RuntimeError(f"cannot register {name}: {type(e).__name__}: {scrub(str(e))[:200]}") from e + ev = threading.Event() + t = threading.Thread(target=self._run, args=(name, fn, ev), name=name, daemon=True) + self.row_ids[name] = int(row["id"]); self.stops[name] = ev; self.threads[name] = t + try: + t.start() + except Exception as e: + with transaction() as conn: + conn.execute("UPDATE processes SET stopped_at = now(), exit_code = -1 WHERE id = %s", (int(row["id"]),)) + raise RuntimeError(f"cannot start {name}: {type(e).__name__}: {e}") from e + self.errors.pop(name, None) + self.started_at[name] = datetime.now(timezone.utc) + self.stopped_at.pop(name, None) + record_event("info", "console", f"started {name} thread", {"pid": self.pid, "by": started_by}) + return True + + def _run(self, name: str, fn, ev: threading.Event) -> None: + code = 0 + try: + fn(stop_event=ev) + except Exception as e: + code = 1 + self.errors[name] = scrub(f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}") + log.exception("%s thread crashed", name) + record_event("error", "console", f"{name} thread crashed: {type(e).__name__}: {e}") + finally: + self.stopped_at[name] = datetime.now(timezone.utc) + try: + with transaction() as conn: + conn.execute("UPDATE processes SET stopped_at = now(), exit_code = %s WHERE id = %s", (code, self.row_ids.get(name))) + except Exception: + pass + + def request_stop(self, name: str, by: str = "console") -> bool: + """Ask a worker to stop without waiting (the console's buttons); the audit row is written here.""" + ev = self.stops.get(name) + if ev is None or not self.alive(name): + return False + ev.set() + record_event("info", "console", f"stop requested for {name} thread", {"pid": self.pid, "by": by}) + return True + + def stop(self, name: str, grace_s: float = 90.0) -> bool: + t = self.threads.get(name) + ev = self.stops.get(name) + if not t or not t.is_alive() or ev is None: + return False + ev.set() + t.join(grace_s) + still = t.is_alive() + record_event("info", "console", f"stop requested for {name}" + (" (still finishing)" if still else " (stopped)")) + return not still + + def stop_all(self) -> None: + for n in WORKERS: + if self.alive(n): + self.stops[n].set() + for n in WORKERS: + t = self.threads.get(n) + if t and t.is_alive(): + t.join(120) + + def autostart(self) -> list[str]: + self.autostarted = True # process-level: the console runs autostart once, not once per browser tab + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'autostart'").fetchone() + if not r: + return [] + import json + val = r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}") + started = [] + for n in WORKERS: + if val.get(n) and not self.alive(n): + try: + if self.start(n, started_by="autostart"): + started.append(n) + except Exception as e: + log.warning("autostart %s: %s", n, scrub(str(e))) + return started + + def log_tail(self, name: str, n: int = 60) -> str: + return tail(name, n) + + +_SUP: Supervisor | None = None +_SUP_LOCK = threading.Lock() + + +def get_supervisor() -> Supervisor: + global _SUP + with _SUP_LOCK: + if _SUP is None: + _SUP = Supervisor() + return _SUP + + +def run_console(port: int = 8501) -> None: + """`fly-trader ui`: start autostart workers in THIS process, then run Streamlit in the main thread.""" + from .. import config + from ..db import schema + schema.apply_schema() # idempotent: tables added to the schema exist before any worker uses them + sup = get_supervisor() + try: + started = sup.autostart() + except Exception as e: + started = []; log.exception("autostart failed: %s", scrub(str(e))) + log.info("console pid=%d autostarted=%s", os.getpid(), started) + app = str(config.REPO_ROOT / "fly_trader" / "ui" / "app.py") + from streamlit.web import bootstrap + flag_options = {"server.port": port, "server.headless": True, "browser.gatherUsageStats": False, + "server.fileWatcherType": "none", "logger.level": "warning"} + bootstrap.load_config_options(flag_options) # the CLI does this before run(); without it port/headless/watcher flags are ignored + try: + bootstrap.run(app, False, [], flag_options) + finally: + sup.stop_all() diff --git a/fly_trader/train/__init__.py b/fly_trader/train/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/train/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/train/corpus_backtest.py b/fly_trader/train/corpus_backtest.py new file mode 100644 index 0000000000000000000000000000000000000000..629db563b714037ba7f0e21e3bc7d1a1a64ebd3d --- /dev/null +++ b/fly_trader/train/corpus_backtest.py @@ -0,0 +1,244 @@ +"""Walk-forward event backtest of rule strategies on the corpus feature rows (``train/corpus_features.py``). + +Per token, rows are the minutes it traded after graduation. A candidate entry at row i fills against the pool at +that minute's close, ``close·(1+impact)/(1−fee)`` (an AMM always fills at its current price); exits are evaluated +on every traded minute's close (trailing stop from the running peak, hard stop from the cost basis, time stop) +and fill at that close ``·(1−impact)·(1−fee)``. Busy minutes can move within the seconds it takes to land, so this +is mildly optimistic there; sparse minutes are exact. Impact is constant-product +against the reserve approximation carried in the rows. One position per token, cooldown after an exit. + +Folds are graduation days: the first ``select_frac`` of days choose the configuration (highest mean net return +with at least ``min_trades`` trades), the remaining days score it; a random-entry baseline with the same entry +rate and exits is reported for the test days. Strategies live in ``STRATEGIES``; each is a list of named +candidate masks over the raw feature columns, crossed with the exit grid. +""" +from __future__ import annotations + +import argparse +import glob +import itertools +import math +import sys +import time +from datetime import timedelta + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq + +from .. import config +from ..market.features import FEATURE_VERSION +from .corpus_features import _epoch_s +from .corpus_meta import load_features + +SIZE = config.MAX_POSITION_SOL +MIN_RESQ_SOL = 20.0 # entries only into pools with at least this much SOL (the live executability gate) +EXITS = list(itertools.product((0.05, 0.10, 0.20), (0.03, 0.08), (30, 120))) # trail, hard, max hold (min) + + +def load(feature_dir=None, max_tokens: int | None = None, days: int | None = None, universe: str = "graduation") -> pd.DataFrame: + root = feature_dir or (config.CORPUS_DIR / "features_mature" if universe == "mature" else config.CORPUS_FEATURES_DIR) + files = sorted(glob.glob(str(root / "*" / "part*.parquet"))) + if days: + cutoff = (pd.Timestamp.utcnow() - timedelta(days=days)).date().isoformat() + files = [f for f in files if f.split("/")[-2] >= cutoff] + if not files: + raise SystemExit("no feature parts found") + df = pd.concat([pq.read_table(f).to_pandas() for f in files], ignore_index=True) + df["day"] = df["ts"].dt.floor("D") + # folds: graduation day for the graduation universe, calendar day for the mature universe + df["grad_day"] = df["day"].dt.date.astype(str) if universe == "mature" else df.groupby("mint")["ts"].transform("min").dt.date.astype(str) + if max_tokens: + keep = df["mint"].drop_duplicates().iloc[:max_tokens] + df = df[df["mint"].isin(keep)] + # drop tokens whose series mixes price scales (secondary pools in other quotes) or carries impossible reserves + df = df.sort_values(["mint", "ts"]) + jump = (df["close"] / df.groupby("mint")["close"].shift(1)).fillna(1.0) + off = ((jump > 50) | (jump < 1 / 50) | (df["resq"] > 100000)).astype(np.int8) + df["broken"] = off.groupby(df["mint"]).cummax().astype(bool) # entries blocked from a token's first scale break on; exits still see the prices + try: + stale = sum(1 for f in files if (pq.read_schema(f).metadata or {}).get(b"fly_version", b"0") != str(FEATURE_VERSION).encode()) + if stale: + print(f"(warning: {stale} of {len(files)} feature parts come from another feature version; rebuild for exact numbers)") + except Exception as e: + print(f"(warning: feature versions not checked: {e})") + try: + meta = load_features() + if len(meta): + df = df.merge(meta, on="mint", how="left") + except Exception as e: + print(f"(corpus_meta not merged: {e})") + return df.sort_values(["mint", "ts"]).reset_index(drop=True) + + +# ---------------------------------------------------------------- candidate rules (raw feature units) +def _s1(d: pd.DataFrame, r: float, k: float, a: float) -> np.ndarray: + v15, v1h = np.expm1(d["logvol_15m"].to_numpy()), np.expm1(d["logvol_1h"].to_numpy()) + return ((d["dd_1h"].to_numpy() >= -0.002) & (d["ret_15m"].to_numpy() >= r) & (v15 >= k * v1h / 4.0) & (d["vol_divergence"].to_numpy() > 0) + & (d["age_h"].to_numpy() >= a) & ~d["has_trades"].to_numpy()) + + +def _s2(d: pd.DataFrame, g: float, i: float, a: float) -> np.ndarray: + s15, s1h = np.expm1(d["logsigners_15m"].to_numpy()), np.expm1(d["logsigners_1h"].to_numpy()) + return ((s15 >= g * s1h / 4.0) & (s15 >= 10) & (d["imb_15m"].to_numpy() >= i) & (d["net_buyers_1h"].to_numpy() > 0) & (d["ret_5m"].to_numpy() > 0) + & (d["age_h"].to_numpy() >= a) & d["has_trades"].to_numpy()) + + +def _s3(d: pd.DataFrame, top: float, pre_top: float, hc: float) -> np.ndarray: + pre = d["pre_top10_pct"].to_numpy() + return ((d["age_h"].to_numpy() >= 1.0) & (d["top_holders_pct"].to_numpy() <= top) & (np.isnan(pre) | (pre <= pre_top)) + & (d["holder_change_1h"].to_numpy() >= hc) & (d["net_buyers_1h"].to_numpy() > 0) & (d["ret_5m"].to_numpy() > 0) + & (d["imb_5m"].to_numpy() > 0) & (d["dd_1h"].to_numpy() >= -0.05) & d["has_trades"].to_numpy()) + + +def _grad_price(d: pd.DataFrame) -> np.ndarray: + """First post-graduation close per mint, from candle rows only (sampled tokens also carry trade-path rows).""" + c = d["close"].where(~d["has_trades"]) + return c.groupby(d["mint"]).transform("first").to_numpy() + + +def _s3b(d: pd.DataFrame, r: float, a_lo: float) -> np.ndarray: + p0 = _grad_price(d) + age = d["age_h"].to_numpy() + return ((age >= a_lo) & (age <= 1.0) & (d["close"].to_numpy() >= p0) & (d["ret_15m"].to_numpy() >= r) & (d["logvol_15m"].to_numpy() >= math.log1p(5.0)) + & ~d["has_trades"].to_numpy()) + + +def _s4(d: pd.DataFrame, age_min: float, rel_min: float, ret15_min: float) -> np.ndarray: + p0 = _grad_price(d) + return ((d["age_h"].to_numpy() * 60 >= age_min) & (d["close"].to_numpy() >= rel_min * p0) & (d["ret_15m"].to_numpy() >= ret15_min) & ~d["has_trades"].to_numpy()) + + +EXITS_S4 = [(1.0, 1.0, 30), (1.0, 1.0, 60), (1.0, 1.0, 120), (0.20, 0.15, 60), (0.20, 0.15, 120), (0.10, 0.08, 60), (0.30, 0.25, 120)] # (trail, hard, hold); 1.0 = never +def _s5(d: pd.DataFrame, ttg: float, dev_max: float, need_grad: bool, rug_max: float) -> np.ndarray: + """Survivor entry restricted to organic launches and creators with a usable record (point-in-time columns).""" + base = _s4(d, 60, 1.0, -1.0) + m = base & (d["ttg_min"].to_numpy() >= ttg) & (d["dev_sol"].fillna(0).to_numpy() <= dev_max) + if need_grad: + m &= d["prior_grads"].fillna(0).to_numpy() >= 1 + if rug_max < 1.0: + m &= ~(d["prior_rug_share"].fillna(0).to_numpy() > rug_max) + return m + + +def _s6(d: pd.DataFrame, age_min_h: float, r1h: float, imb: float, dd: float) -> np.ndarray: + """Mature momentum: token at least age_min_h old (or older than the archive), 1 h return and 15 m buy imbalance positive, near the 1 h high.""" + age = d["age_h"].to_numpy(); old = np.isnan(age) | (age >= age_min_h) + return (old & (d["ret_1h"].to_numpy() >= r1h) & (d["ret_15m"].to_numpy() > 0) & (d["imb_15m"].to_numpy() >= imb) & (d["dd_1h"].to_numpy() >= dd) + & (d["logvol_15m"].to_numpy() >= math.log1p(5.0))) + + +MATURE_STRATEGIES = { + "S6 mature momentum": lambda d: [(f"age>={a}h ret1h>={r} imb15>={i} dd1h>={dd}", _s6(d, a, r, i, dd)) for a in (12, 48) for r in (0.05, 0.15) for i in (0.0, 0.3) for dd in (-0.05, -0.02)], + "S6 control: mature, any minute with volume": lambda d: [(f"age>={a}h", (np.isnan(d["age_h"].to_numpy()) | (d["age_h"].to_numpy() >= a)) & (d["logvol_15m"].to_numpy() >= math.log1p(5.0))) for a in (12, 48)], +} +STRATEGIES = { + "S5 organic survivor (creation-time + creator screens)": lambda d: [(f"ttg>={t}m dev<={dv} prior_grad={pg} rug<={rg}", _s5(d, t, dv, pg, rg)) for t in (10, 30, 60) for dv in (2.0, 1e9) for pg in (False, True) for rg in (0.5, 1.0)], + "S4 survivor (age since graduation, price vs graduation price)": lambda d: [(f"age>={a}m rel>={r} ret15>={q}", _s4(d, a, r, q)) for a in (60, 120, 180) for r in (1.0, 1.5) for q in (-1.0, 0.0)], + "S4 control: age only (no price filter)": lambda d: [(f"age>={a}m", _s4(d, a, 0.0, -1.0)) for a in (60, 180)], + "S1 breakout+volume (candles, all tokens)": lambda d: [(f"r={r} k={k} age>={a}", _s1(d, r, k, a)) for r in (0.03, 0.06, 0.10) for k in (1.5, 3.0) for a in (0.5, 6.0)], + "S2 participation flow (trade rows)": lambda d: [(f"g={g} i={i} age>={a}", _s2(d, g, i, a)) for g in (1.5, 3.0) for i in (0.2, 0.5) for a in (0.5, 6.0)], + "S3 quality survivor (trade rows)": lambda d: [(f"top10<={t} pre_top10<={pt} holders_chg>={hc}", _s3(d, t, pt, hc)) for t in (60.0, 80.0) for pt in (90.0, 100.0) for hc in (0.0, 0.2)], + "S3b graduation survivor (candles)": lambda d: [(f"ret15>={r} age>={a}", _s3b(d, r, a)) for r in (0.0, 0.05) for a in (0.25, 0.5)], +} + + +# ---------------------------------------------------------------- simulation +def simulate(d: pd.DataFrame, cand: np.ndarray, fee_side: float, trail: float, hard: float, max_hold_min: int, cooldown_min: int = 5) -> np.ndarray: + """Returns array of (entry fold-day ordinal, net_return) per closed trade. Fills happen against the pool at the signal + minute's close (an AMM always fills; the close is the pool's current price), entry ``close·(1+impact)/(1−fee)``, + exit ``close·(1−impact)·(1−fee)``. Trailing and hard stops are checked on every traded minute's close; the time stop + exits at the last close at or before ``entry + max_hold`` when the next traded minute is past it.""" + mints = d["mint"].to_numpy(); ts = _epoch_s(d["ts"]) + cl, resq = d["close"].to_numpy(), d["resq"].to_numpy() + gday = pd.to_datetime(d["grad_day"]).map(pd.Timestamp.toordinal).to_numpy() + imp = SIZE / (SIZE + np.where(np.isfinite(resq) & (resq > 0), resq, 5.0)) + hold_s = max_hold_min * 60; out = [] + starts = np.r_[0, np.flatnonzero(mints[1:] != mints[:-1]) + 1, len(mints)] + for s, e in zip(starts[:-1], starts[1:]): + first = np.flatnonzero(cand[s:e]) + if len(first) == 0: + continue + pos = False; basis = peak = 0.0; t_in = 0.0; d_in = 0; cd_until = -1.0 + for i in range(s + int(first[0]), e): + t = ts[i] + if pos and t - t_in > hold_s: # time stop fell between traded minutes: the previous row is the last close before it + j = i - 1 + out.append((d_in, cl[j] * (1 - imp[j]) * (1 - fee_side) / basis - 1)); pos = False; cd_until = t_in + hold_s + cooldown_min * 60 + if pos: + peak = max(peak, cl[i]) + if (cl[i] / peak - 1 <= -trail) or (cl[i] / basis - 1 <= -hard) or (t - t_in >= hold_s): + out.append((d_in, cl[i] * (1 - imp[i]) * (1 - fee_side) / basis - 1)); pos = False; cd_until = t + cooldown_min * 60 + elif cand[i] and t >= cd_until: + basis = cl[i] * (1 + imp[i]) / (1 - fee_side); peak = cl[i]; t_in = t; d_in = gday[i]; pos = True + if pos: # still open at the end of the rows: mark at the last close + out.append((d_in, cl[e - 1] * (1 - imp[e - 1]) * (1 - fee_side) / basis - 1)) + return np.array(out, dtype=float).reshape(-1, 2) + + +def stats(r: np.ndarray) -> dict: + if len(r) == 0: + return dict(n=0, mean=np.nan, med=np.nan, win=np.nan, pf=np.nan, pnl=0.0, lo=np.nan, hi=np.nan, top3=np.nan) + g = r[r > 0].sum(); l = -r[r <= 0].sum(); rng = np.random.default_rng(0) + bs = np.array([rng.choice(r, len(r)).mean() for _ in range(1000)]) + return dict(n=len(r), mean=r.mean(), med=np.median(r), win=(r > 0).mean(), pf=g / l if l > 0 else np.inf, pnl=SIZE * r.sum(), + lo=np.percentile(bs, 2.5), hi=np.percentile(bs, 97.5), top3=SIZE * (r.sum() - np.sort(r)[-3:].sum()) if len(r) >= 3 else np.nan) + + +def fmt(s: dict) -> str: + return (f"n={s['n']:5d} mean {s['mean']*100:+6.2f}% [{s['lo']*100:+.2f}..{s['hi']*100:+.2f}] median {s['med']*100:+6.2f}% win {s['win']*100:3.0f}% " + f"PF {s['pf']:.2f} P&L {s['pnl']:+.3f} SOL (minus top-3 {s['top3']:+.3f})") + + +def run(df: pd.DataFrame, fee_side: float, select_frac: float = 0.6, min_trades: int = 30) -> None: + days = sorted(df["grad_day"].unique()); n_sel = max(1, int(len(days) * select_frac)) + sel_days = set(days[:n_sel]); split_ord = pd.Timestamp(days[n_sel]).toordinal() if n_sel < len(days) else 10**9 + n_tok = df["mint"].nunique(); n_tr = df.loc[df["has_trades"], "mint"].nunique() + print(f"\nfee/side {fee_side:.4f} | {n_tok:,} tokens ({n_tr:,} with trade rows) | {len(df):,} rows | days {days[0]}..{days[-1]} | selection {len(sel_days)} days, test {len(days)-n_sel} days", flush=True) + gate = np.isfinite(df["resq"].to_numpy()) & (df["resq"].to_numpy() >= MIN_RESQ_SOL) & ~df["broken"].to_numpy() # executability gate, as live + for sname, make in STRATEGIES.items(): + rows = []; t0 = time.time() + for vname, cand in make(df): + cand = cand & gate + if not cand.any(): + continue + for trail, hard, mh in (EXITS_S4 if sname[:2] in ("S4", "S5", "S6") else EXITS): + tr = simulate(df, cand, fee_side, trail, hard, mh) + sel, tst = tr[tr[:, 0] < split_ord, 1], tr[tr[:, 0] >= split_ord, 1] + rows.append((vname, trail, hard, mh, stats(sel), stats(tst), cand)) + if not rows: + print(f"\n=== {sname}: no candidates"); continue + ok = [r for r in rows if r[4]["n"] >= min_trades] or rows + best = max(ok, key=lambda r: r[4]["mean"] if np.isfinite(r[4]["mean"]) else -9) + vname, trail, hard, mh, ss, st, cand = best + tested = [r for r in rows if r[5]["n"] >= 10] + pos_frac = np.mean([r[5]["mean"] > 0 for r in tested]) if tested else np.nan + p_sig = cand.sum() / max(1, gate.sum()); rb = [] # entry rate among the rows the gate lets through + for seed in range(5): + rc = (np.random.default_rng(seed).random(len(df)) < p_sig) & gate + trr = simulate(df, rc, fee_side, trail, hard, mh); rb.append(stats(trr[trr[:, 0] >= split_ord, 1])["mean"]) + print(f"\n=== {sname} === {len(rows)} configs in {time.time()-t0:.0f}s; share positive on TEST (n>=10): {pos_frac*100:.0f}%") + print(f" chosen on selection: {vname} | trail {trail*100:.0f}% hard {hard*100:.0f}% max hold {mh} min") + print(f" selection: {fmt(ss)}\n TEST : {fmt(st)}") + print(f" random entries, same rate & exits, TEST mean: {np.nanmean(rb)*100:+.2f}%") + for r in sorted(tested, key=lambda r: -r[5]["mean"])[:3]: + print(f" best-on-test (reference): {r[0]} trail {r[1]*100:.0f}% hard {r[2]*100:.0f}% hold {r[3]}m -> test n={r[5]['n']} mean {r[5]['mean']*100:+.2f}% PF {r[5]['pf']:.2f}; selection {r[4]['mean']*100:+.2f}%") + sys.stdout.flush() + + +def main(argv=None) -> None: + ap = argparse.ArgumentParser(); ap.add_argument("--max-tokens", type=int, default=None); ap.add_argument("--days", type=int, default=None) + ap.add_argument("--fees", default="0.003,0.0055"); ap.add_argument("--only", default=None, help="substring filter on strategy names") + ap.add_argument("--universe", default="graduation", choices=["graduation", "mature"]); a = ap.parse_args(argv) + df = load(max_tokens=a.max_tokens, days=a.days, universe=a.universe) + global STRATEGIES + if a.universe == "mature": + STRATEGIES = dict(MATURE_STRATEGIES) + if a.only: + STRATEGIES = {k: v for k, v in STRATEGIES.items() if a.only.lower() in k.lower()} + for fee in (float(x) for x in a.fees.split(",")): + run(df, fee) + + +if __name__ == "__main__": + main() diff --git a/fly_trader/train/corpus_features.py b/fly_trader/train/corpus_features.py new file mode 100644 index 0000000000000000000000000000000000000000..dc477fd039a66af0f2712b608ccfdb8730d2d92b --- /dev/null +++ b/fly_trader/train/corpus_features.py @@ -0,0 +1,269 @@ +"""Corpus → observation rows: the same 45 features the runner computes (``market/features.py``), one row per +minute in which the token traded, for every pulled token. + +* candle path (every token): each 1-minute candle is fed to ``TokenState`` as one synthetic trade at the close + with the candle's SOL volume; features that need per-trade detail (imbalance, counts, signers, Hawkes, + VPIN, since-last) are zeroed and unmasked. Realised vol is candle-based here and trade-based live. +* trade path (the sampled tokens with per-trade rows): the real trades replay through ``TokenState`` exactly + as the live tape does; holders, top-10 concentration, net buyers and holder change are reconstructed from + wallet flows and written into the Jupiter-stat slots (proxies: traders only, not all token accounts). +* liquidity: PumpSwap quote reserve ≈ ``CORPUS_RQ0_SOL·sqrt(p/p_grad)`` (tape check 2026-09-13: per-token + correlation of log reserve with ½·log price = 0.997). Only post-graduation minutes become rows; the bonding-curve phase is summarised in pre_* columns. + +Output: ``data/corpus/features//part-.parquet`` (columns: mint, ts, phase, t_rel_min, +open/high/low/close, volume_sol, resq, age_h, has_trades, mask, and one column per feature name); registry +``corpus_features``. Runs as a thread inside the ``corpus`` worker and as ``fly-trader build-corpus-features``. +""" +from __future__ import annotations + +import json +import logging +import math +import threading +import time +from collections import Counter, defaultdict, deque +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config, logging_setup +from ..db.connection import transaction +from ..market.features import FEATURE_VERSION, FEATURES, FIDX, TokenMeta, TokenState + +log = logging.getLogger(__name__) +_failed: set[str] = set() # tokens whose build raised in this process: skipped until restart, never registered +CANDLE_ONLY_MASKED = ["imb_1m", "imb_5m", "imb_15m", "imb_1h", "logn_1m", "logn_5m", "logn_15m", "logn_1h", + "logsigners_15m", "logsigners_1h", "hawkes", "log_since_last", "vpin_15m"] +PROXY_STATS = ["log_holders", "top_holders_pct", "net_buyers_1h", "holder_change_1h"] +PRE_COLS = ["pre_minutes", "pre_trades", "pre_buyers", "pre_vol_sol", "pre_top10_pct"] # bonding-curve summary at graduation (MELT-style) +BASE_COLS = ["mint", "ts", "phase", "t_rel_min", "open", "high", "low", "close", "volume_sol", "resq", "age_h", "has_trades", "mask"] + PRE_COLS +SCHEMA = pa.schema([("mint", pa.string()), ("ts", pa.timestamp("ms", tz="UTC")), ("phase", pa.string()), ("t_rel_min", pa.int32()), + ("open", pa.float64()), ("high", pa.float64()), ("low", pa.float64()), ("close", pa.float64()), ("volume_sol", pa.float64()), + ("resq", pa.float64()), ("age_h", pa.float32()), ("has_trades", pa.bool_()), ("mask", pa.int64())] + + [(name, pa.float32()) for name in PRE_COLS] + [(name, pa.float32()) for name in FEATURES]) + + +def _epoch_s(col: pd.Series): + """tz-aware datetime column → float seconds since the epoch, whatever the stored unit (ms/ns).""" + return ((col - pd.Timestamp(0, tz="UTC")) / pd.Timedelta(seconds=1)).to_numpy(dtype="float64") + + +def _rq(price: float, p_grad: float) -> float: + return config.CORPUS_RQ0_SOL * math.sqrt(max(price, 1e-30) / p_grad) + + +def _row(mint: str, ts_s: float, g: float, o: float, h: float, l: float, c: float, vol: float, resq: float | None, + has_trades: bool, f: list[float], mask: int, pre: dict) -> dict: + r = {"mint": mint, **{k: float(pre.get(k, float("nan"))) for k in PRE_COLS}, "ts": datetime.fromtimestamp(ts_s, timezone.utc), + "phase": "amm" if ts_s + 60.0 > g else "curve", # rows start at the first candle that opens after graduation + "t_rel_min": max(0, int(math.floor((ts_s - g) / 60.0))), "open": o, "high": h, "low": l, "close": c, "volume_sol": vol, + "resq": resq if resq is not None else float("nan"), "age_h": max(0.0, (ts_s - g) / 3600.0), "has_trades": has_trades, "mask": mask} + for i, name in enumerate(FEATURES): + r[name] = float(f[i]) + return r + + +def candle_rows(mint: str, g: float, candle_path: str) -> list[dict]: + """Rows for every post-graduation 1-minute candle; the bonding-curve candles only feed the pre-migration summary.""" + cd = pq.read_table(candle_path).to_pandas() + cd = cd[cd["interval"] == "1m"].sort_values("ts") + if cd.empty: + return [] + ts_s = _epoch_s(cd["ts"]); amm = ts_s >= g + if not amm.any(): + return [] + pre_vol = float(cd["volume_sol"].to_numpy()[~amm].sum()); pre_min = float((g - ts_s[~amm].min()) / 60.0) if (~amm).any() else 0.0 + pre = {"pre_minutes": pre_min, "pre_trades": float("nan"), "pre_buyers": float("nan"), "pre_vol_sol": pre_vol, "pre_top10_pct": float("nan")} + cd = cd[amm]; ts_s = ts_s[amm] + p_grad = float(cd["close"].iloc[0]) + real = cd["resq_sol"].to_numpy(dtype="float64") if "resq_sol" in cd.columns else None + st = TokenState(mint); meta = TokenMeta(mint=mint, program_label="Pump.fun Amm", graduated_at=g) + out = [] + for i, (t0, o, h, l, c, v) in enumerate(zip(ts_s, cd["open"], cd["high"], cd["low"], cd["close"], cd["volume_sol"])): + t_end = float(t0) + 60.0; price = float(c) + resq = float(real[i]) if real is not None and math.isfinite(real[i]) and real[i] > 0 else _rq(price, p_grad) + st.append(t_end - 1e-3, price, float(v), True, None, resq) + f, mask = st.features(t_end, meta) + for name in CANDLE_ONLY_MASKED: + f[FIDX[name]] = 0.0; mask &= ~(1 << FIDX[name]) + out.append(_row(mint, float(t0), g, float(o), float(h), float(l), price, float(v), resq, False, f, mask, pre)) + return out + + +def trade_rows(mint: str, g: float, trade_path: str) -> list[dict]: + """One row per post-graduation minute with trades. Curve-phase trades only build wallet balances and the trailing-hour + buyer/seller sets (their prices are on the curve's own scale); the 85 SOL migration transfer is dropped.""" + td = pq.read_table(trade_path).to_pandas().sort_values("ts") + if td.empty: + return [] + ts_s = _epoch_s(td["ts"]) + keep = ~((td["program"].to_numpy() == "pump") & (ts_s >= g - 5.0) & (td["sol"].to_numpy() >= 50.0)) + td = td[keep]; ts_s = ts_s[keep] + prices = td["price_sol"].to_numpy(); sols = td["sol"].to_numpy(); sides = td["side"].to_numpy(); toks = td["tokens"].to_numpy(); wallets = td["wallet"].to_numpy() + amm = (ts_s >= g) & (prices > 0) + if not amm.any(): + return [] + p_grad = float(prices[amm][0]) + st = TokenState(mint); meta = TokenMeta(mint=mint, program_label="Pump.fun Amm", graduated_at=g) + bal: dict = defaultdict(float) + recent: deque = deque() # (ts, wallet, side) for the trailing hour + buyers_1h: Counter = Counter(); sellers_1h: Counter = Counter() + holders_hist: deque = deque() # (ts, holders) + pre_buyers: set = set(); pre_trades = 0; pre_vol = 0.0; pre_min = float((g - ts_s.min()) / 60.0) if ts_s.min() < g else 0.0 + + def book(t: float, w, side: int, tok: float) -> None: + if side == 1: + bal[w] += tok; buyers_1h[w] += 1 + else: + bal[w] = max(0.0, bal[w] - tok); sellers_1h[w] += 1 + recent.append((t, w, side)) + + def expire(t_end: float) -> None: + while recent and recent[0][0] < t_end - 3600.0: + _, w, side = recent.popleft(); cnt = buyers_1h if side == 1 else sellers_1h + cnt[w] -= 1 + if cnt[w] <= 0: + del cnt[w] + + out = []; n = len(ts_s); i = 0 + while i < n and ts_s[i] < g: # curve phase + t, side, tok, w, s_ = float(ts_s[i]), int(sides[i]), float(toks[i]), wallets[i], float(sols[i]); i += 1 + book(t, w, side, tok); pre_trades += 1; pre_vol += s_ + if side == 1: + pre_buyers.add(w) + pos0 = [v for v in bal.values() if v > 0] + pre = {"pre_minutes": pre_min, "pre_trades": float(pre_trades), "pre_buyers": float(len(pre_buyers)), "pre_vol_sol": pre_vol, + "pre_top10_pct": (100.0 * sum(sorted(pos0)[-10:]) / sum(pos0)) if pos0 else float("nan")} + while i < n: # AMM phase, minute by minute + minute = math.floor(ts_s[i] / 60.0) * 60.0 + o = h = l = c = None; vol = 0.0 + while i < n and ts_s[i] < minute + 60.0: + t, p, s_, side, tok, w = float(ts_s[i]), float(prices[i]), float(sols[i]), int(sides[i]), float(toks[i]), wallets[i] + i += 1 + if not (p > 0 and math.isfinite(p)): + continue + st.append(t, p, s_, side == 1, w, _rq(p, p_grad)) + o = p if o is None else o; h = p if h is None else max(h, p); l = p if l is None else min(l, p); c = p; vol += s_ + book(t, w, side, tok) + if c is None: + continue + t_end = minute + 60.0; expire(t_end) + f, mask = st.features(t_end, meta) + pos = [v for v in bal.values() if v > 0] + holders = len(pos); total = sum(pos) + top10 = sum(sorted(pos)[-10:]) / total if total > 0 else 0.0 + while holders_hist and holders_hist[0][0] < t_end - 3660.0: + holders_hist.popleft() + h_ago = holders_hist[0][1] if holders_hist else None + holders_hist.append((t_end, holders)) + f[FIDX["log_holders"]] = math.log1p(holders); f[FIDX["top_holders_pct"]] = 100.0 * top10 + f[FIDX["net_buyers_1h"]] = float(len(buyers_1h) - len(sellers_1h)) + f[FIDX["holder_change_1h"]] = (holders / h_ago - 1.0) if h_ago else 0.0 + for name in PROXY_STATS: + mask |= 1 << FIDX[name] + out.append(_row(mint, minute, g, o, h, l, c, vol, _rq(c, p_grad), True, f, mask, pre)) + return out + + +def build_token(mint: str, graduated_at: datetime, candle_path: str | None, trade_path: str | None) -> tuple[list[dict], int]: + g = graduated_at.timestamp(); rows: list[dict] = []; n_trade = 0 + if candle_path and Path(candle_path).exists(): + rows += candle_rows(mint, g, candle_path) + if trade_path and Path(trade_path).exists(): + tr = trade_rows(mint, g, trade_path); n_trade = len(tr); rows += tr + return rows, n_trade + + +def _pending(limit: int) -> list[dict]: + with transaction() as conn: + rows = conn.execute("SELECT t.mint, t.graduated_at, t.candle_path, t.trade_path FROM corpus_tokens t LEFT JOIN corpus_features f USING (mint) " + "WHERE t.status = 'done' AND f.mint IS NULL ORDER BY t.graduated_at DESC LIMIT %s", (limit + len(_failed),)).fetchall() + return [r for r in rows if r["mint"] not in _failed][:limit] + + +def _commit_parts(rows_by_day: dict[str, list[dict]], reg: list[tuple]) -> None: + """Parts are written to temp files and renamed into place inside the registry transaction: a failed insert leaves + neither a part nor a registration, so the tokens are rebuilt once, never duplicated.""" + ms = int(time.time() * 1000); staged = [] + try: + for day, rows in rows_by_day.items(): + if not rows: + continue + path = config.CORPUS_FEATURES_DIR / day / f"part-{ms}.parquet"; path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.tmp") + pq.write_table(pa.Table.from_pylist(rows, schema=SCHEMA).replace_schema_metadata({b"fly_version": str(FEATURE_VERSION).encode()}), tmp, compression="zstd") + staged.append((day, tmp, path)) + paths = {day: str(path) for day, _, path in staged} + with transaction() as conn: + conn.cursor().executemany("INSERT INTO corpus_features (mint, rows, trade_rows, has_trades, path) VALUES (%s,%s,%s,%s,%s) ON CONFLICT (mint) DO NOTHING", + [(m, r, tr, ht, paths.get(day)) for m, r, tr, ht, day in reg]) + for _, tmp, path in staged: + tmp.replace(path) + finally: + for _, tmp, _ in staged: + tmp.unlink(missing_ok=True) + + +def _status(**kv) -> None: + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('corpus_features_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps({**kv, "updated_at": datetime.now(timezone.utc).isoformat()}, default=str),)) + except Exception: + log.debug("features status write failed", exc_info=True) + + +def build_batch(limit: int = 200) -> int: + """Build features for up to ``limit`` pulled tokens without rows yet. Returns tokens attempted (a token whose build + raises is not registered, so a later process retries it; this one skips it).""" + todo = _pending(limit) + if not todo: + return 0 + rows_by_day: dict[str, list[dict]] = defaultdict(list); reg: list[tuple] = []; t0 = time.time() + for t in todo: + try: + rows, n_trade = build_token(t["mint"], t["graduated_at"], t["candle_path"], t["trade_path"]) + except Exception as e: + log.warning("features %s failed: %s", t["mint"], e); _failed.add(t["mint"]); continue + day = t["graduated_at"].astimezone(timezone.utc).date().isoformat() + rows_by_day[day].extend(rows) + reg.append((t["mint"], len(rows), n_trade, n_trade > 0, day)) + if reg: + _commit_parts(rows_by_day, reg) + n_rows = sum(len(v) for v in rows_by_day.values()) + log.info("features: %d tokens (%d failed), %d rows in %.1fs", len(todo), len(todo) - len(reg), n_rows, time.time() - t0) + return len(todo) + + +def build_loop(stop_event: threading.Event | None = None, idle_s: float = 20.0) -> None: + built = 0; rows_total = 0 + while not (stop_event is not None and stop_event.is_set()): + try: + n = build_batch() + except Exception as e: + log.exception("feature build failed"); _status(stage="error", last_error=str(e)[:200]); n = 0 + if n: + built += n + with transaction() as conn: + c = conn.execute("SELECT count(*) AS n, coalesce(sum(rows),0) AS rows, count(*) FILTER (WHERE has_trades) AS with_trades FROM corpus_features").fetchone() + _status(stage="building", built_this_run=built, tokens=int(c["n"]), rows=int(c["rows"]), with_trades=int(c["with_trades"])) + continue + _status(stage="idle", built_this_run=built) + end = time.time() + idle_s + while time.time() < end and not (stop_event is not None and stop_event.is_set()): + time.sleep(1.0) + + +def main() -> None: + """CLI: build everything pending, then exit.""" + logging_setup.setup("corpus") + total = 0 + while True: + n = build_batch(500) + if not n: + break + total += n + print(f"built features for {total} tokens") diff --git a/fly_trader/train/corpus_meta.py b/fly_trader/train/corpus_meta.py new file mode 100644 index 0000000000000000000000000000000000000000..c8ff780c4d2d1106c46f90925c14b9717e425962 --- /dev/null +++ b/fly_trader/train/corpus_meta.py @@ -0,0 +1,351 @@ +"""Creation-time facts and point-in-time creator history for every graduated token (table ``corpus_meta``). + +From the replay lifecycle rows: the ``create`` event (creator wallet, the creator's own first buy in SOL and tokens, +supply, mayhem flag, metadata URI) and the ``migrate`` event (graduation time, pool id, real quote reserve at +migration). Derived: ``ttg_min`` (minutes from creation to graduation; instant graduations are bundled launches), +``dev_share`` (creator's first buy as a share of supply). + +Creator history is strictly point-in-time as of the token's own graduation: ``prior_launches`` (the creator's +earlier creates), ``prior_grads`` (the creator's earlier graduations), and among those with a known outcome, +``prior_rug_share`` (fell 90 % within 60 min of graduation) and ``prior_moon_share`` (doubled within 60 min). +The token's own outcome columns (``own_*``) exist only to feed other tokens' history; never use them as features. + +``creator_history()`` is the same definition for one token, used live by ``ingest/pumpstream.py`` at graduation; it +reads the creator's creates from ``pump_events`` (the archive's creates are backfilled there by ``rebuild``, the stream +adds new ones) and earlier graduations from this table (``graduated_at``, outcomes known 60 min after graduation — +the stream fills ``own_dd60``/``own_max60`` from its own minutes at that time, the archive rebuild later replaces them). + +``rebuild()`` recomputes the whole table from the archive (events are small) and fills outcomes incrementally from +the assembled candle files. Runs inside the ``replay`` worker after each assembly round and as +``fly-trader build-corpus-meta``. It first copies the archive's directly created PumpSwap pools into ``pump_pools`` +(``backfill_pump_pools``), which the stream and ``train/mature.py`` exclude from the universe (``blocked_pool_ids``). +""" +from __future__ import annotations + +import logging +import time +from pathlib import Path + +import duckdb +import numpy as np +import pandas as pd +import pyarrow.parquet as pq + +from .. import config +from ..db.connection import transaction + +log = logging.getLogger(__name__) +CURVE_COLS = ["bundle_share", "dev_hold_share", "dev_sold_frac", "grad_hhi"] +FEATURE_COLS = ["ttg_min", "dev_sol", "dev_share", "mayhem", "rq0", "prior_launches", "prior_grads", "prior_known", "prior_rug_share", "prior_moon_share", + *CURVE_COLS, "curve_known"] + + +def curve_facts(create_slot: int | None, creator: str | None, supply: float | None, initial_buy: float | None, wallets: dict) -> tuple[dict, list[tuple[str, str]]]: + """Graduation-time rug facts from a token's bonding-curve trading, one definition for the archive (``update_curve_facts``) + and the live stream (``ingest/pumpstream.py``). ``wallets``: wallet → [first buy slot or None, tokens bought, tokens sold] + over the curve legs up to the migration. The creator's first buy rides on the create event (``initial_buy``, not a + trade leg — measured 2026-09-15: a creator leg in the create slot occurs only when the create carried no initial buy). + - bundle_share: tokens held at migration by non-creator wallets whose first buy is in the create slot (same block as + the creation: bought together with it) / supply; + - dev_hold_share: the creator's held tokens (initial buy + curve buys − sells) / supply; + - dev_sold_frac: the creator's sold tokens / its bought tokens (initial buy included); + - grad_hhi: Herfindahl concentration of holdings at migration (Σ of squared shares of all positive holdings). + Returns (facts, insider wallets [(wallet, kind)]: the creator and bundle wallets).""" + sup = float(supply) if supply else 0.0; ib = float(initial_buy or 0.0) + held, bundle, insiders = [], 0.0, [] + dev_b, dev_s = ib, 0.0 + for w, (first, bought, sold) in wallets.items(): + net = float(bought) - float(sold) + if w == creator: + dev_b += float(bought); dev_s += float(sold) + continue + if net > 0: + held.append(net) + if first is not None and create_slot is not None and int(first) == int(create_slot): + bundle += max(net, 0.0); insiders.append((w, "bundle")) + dev_net = dev_b - dev_s + if creator: + insiders.insert(0, (creator, "creator")) + if dev_net > 0: + held.append(dev_net) + tot = sum(held) + facts = {"bundle_share": bundle / sup if sup > 0 else None, "dev_hold_share": max(dev_net, 0.0) / sup if sup > 0 else None, + "dev_sold_frac": dev_s / dev_b if dev_b > 0 else 0.0, "grad_hhi": sum((h / tot) ** 2 for h in held) if tot > 0 else None} + return facts, insiders + + +def _outcomes(rows: list[dict]) -> list[tuple]: + """(mint, dd60, max60, alive6h) from a token's candle file (post-graduation 1m candles).""" + out = [] + for r in rows: + try: + t = pq.read_table(r["candle_path"], columns=["ts", "interval", "close"]).to_pandas() + except Exception: + continue + g = r["graduated_at"]; t = t[t["ts"] >= g].sort_values("ts") + if t.empty: + out.append((r["mint"], None, None, False)); continue + p0 = float(t["close"].iloc[0]); w = t[t["ts"] < g + pd.Timedelta(minutes=60)] + dd = float(w["close"].min() / p0 - 1) if len(w) else None; mx = float(w["close"].max() / p0 - 1) if len(w) else None + alive = bool((t["ts"].max() - g) >= pd.Timedelta(hours=6)) + out.append((r["mint"], dd, mx, alive)) + return out + + +def rebuild(outcome_batch: int = 20000) -> int: + t0 = time.time(); con = duckdb.connect(); ev = str(config.REPLAY_DIR / "*" / "*_events.parquet") + if not list(config.REPLAY_DIR.glob("*/*_events.parquet")): + return 0 + n_pp = backfill_pump_pools() # before mature.loop_once aggregates the new days + creates =con.execute(f"""SELECT mint, min(ts) AS create_ts, arg_min(signer, ts) AS creator, arg_min(quote_amount, ts) AS dev_sol, + arg_min(initial_buy, ts) AS dev_tokens, arg_min(supply, ts) AS supply, arg_min(mayhem, ts) AS mayhem, arg_min(uri, ts) AS uri, + arg_min(name, ts) AS name, arg_min(symbol, ts) AS symbol, arg_min(sig, ts) AS sig FROM read_parquet('{ev}') WHERE action = 'create' AND pool = 'pump' GROUP BY mint""").df() + migr = con.execute(f"""SELECT mint, min(ts) AS g, arg_min(quote_in_pool, ts) AS rq0, arg_min(pool_id, ts) AS pool_id + FROM read_parquet('{ev}') WHERE action = 'migrate' AND pool = 'pump-amm' GROUP BY mint""").df() + con.close() + meta = migr.merge(creates, on="mint", how="left") + meta["ttg_min"] = (meta["g"] - meta["create_ts"]).dt.total_seconds() / 60.0 + meta["dev_share"] = meta["dev_tokens"] / meta["supply"] + # outcomes: existing ones from the table, new ones from candle files + with transaction() as conn: + known = pd.DataFrame(conn.execute("SELECT mint, own_dd60, own_max60, own_alive6h FROM corpus_meta WHERE own_alive6h IS NOT NULL OR own_dd60 IS NOT NULL").fetchall()) + todo = conn.execute("SELECT t.mint, t.graduated_at, t.candle_path FROM corpus_tokens t LEFT JOIN corpus_meta m USING (mint) " + "WHERE t.candle_path IS NOT NULL AND t.status = 'done' AND (m.mint IS NULL OR m.own_alive6h IS NULL) ORDER BY t.graduated_at DESC LIMIT %s", (outcome_batch,)).fetchall() + new = pd.DataFrame(_outcomes(todo), columns=["mint", "own_dd60", "own_max60", "own_alive6h"]) if todo else pd.DataFrame(columns=["mint", "own_dd60", "own_max60", "own_alive6h"]) + # candle-file outcomes win over stream-computed ones; stream values survive until the day is assembled + outcomes = pd.concat([new, known], ignore_index=True).drop_duplicates("mint") if len(known) or len(new) else new + meta = meta.merge(outcomes, on="mint", how="left") + # point-in-time creator history (as of this token's graduation) + cr = creates.sort_values("create_ts").copy(); cr["prior_launches"] = cr.groupby("creator").cumcount() + meta = meta.merge(cr[["mint", "prior_launches"]], on="mint", how="left") + hist = meta.dropna(subset=["creator"]).sort_values("g").copy() + hist["rug"] = (hist["own_dd60"] <= -0.9).astype(float).where(hist["own_dd60"].notna()) + hist["moon"] = (hist["own_max60"] >= 1.0).astype(float).where(hist["own_max60"].notna()) + hist["prior_grads"] = hist.groupby("creator").cumcount() + # outcomes of an earlier graduation A are known 60 min after g_A; B may only count A when g_A + 60 min <= g_B (no look-ahead) + known = np.zeros(len(hist), int); rugs = np.zeros(len(hist)); moons = np.zeros(len(hist)) + g_s = (hist["g"] - pd.Timestamp(0, tz="UTC")).dt.total_seconds().to_numpy(); rug_v = hist["rug"].to_numpy(); moon_v = hist["moon"].to_numpy() + for _, idx in hist.groupby("creator").indices.items(): + idx = np.asarray(idx); gs = g_s[idx]; order = np.argsort(gs, kind="stable"); idx = idx[order]; gs = gs[order] + j = 0; k_cnt = 0; r_sum = 0.0; m_sum = 0.0 + for b in range(len(idx)): + while j < b and gs[j] + 3600.0 <= gs[b]: + if not np.isnan(rug_v[idx[j]]): + k_cnt += 1; r_sum += rug_v[idx[j]]; m_sum += moon_v[idx[j]] + j += 1 + known[idx[b]] = k_cnt; rugs[idx[b]] = r_sum; moons[idx[b]] = m_sum + hist["prior_known"] = known + hist["prior_rug_share"] = np.where(known > 0, rugs / np.maximum(known, 1), np.nan) + hist["prior_moon_share"] = np.where(known > 0, moons / np.maximum(known, 1), np.nan) + meta = meta.merge(hist[["mint", "prior_grads", "prior_known", "prior_rug_share", "prior_moon_share"]], on="mint", how="left") + meta["graduated_at"] = meta["g"] + cols = ["mint", "graduated_at", "create_ts", "creator", "dev_sol", "dev_tokens", "dev_share", "supply", "mayhem", "uri", "name", "symbol", "ttg_min", "rq0", "pool_id", + "prior_launches", "prior_grads", "prior_known", "prior_rug_share", "prior_moon_share", "own_dd60", "own_max60", "own_alive6h"] + rows = [tuple(_pg(v) for v in rec) for rec in meta[cols].itertuples(index=False, name=None)] + with transaction() as conn: + conn.cursor().executemany( + "INSERT INTO corpus_meta (" + ",".join(cols) + ") VALUES (" + ",".join(["%s"] * len(cols)) + ") ON CONFLICT (mint) DO UPDATE SET " + + ", ".join(f"{c} = EXCLUDED.{c}" for c in cols if c != "mint") + ", updated_at = now()", rows) + n_cr = _backfill_creates(creates) + n_curve = update_curve_facts() + log.info("corpus_meta rebuilt: %d graduated tokens (%d with creator, %d with outcomes), %d archive creates added to pump_events, %d custom pools to pump_pools, " + "curve facts for %d, in %.0fs", len(meta), int(meta["creator"].notna().sum()), int(meta["own_dd60"].notna().sum()), n_cr, n_pp, n_curve, time.time() - t0) + return len(meta) + + +def update_curve_facts() -> int: + """Curve facts and insiders (``curve_facts``) for graduated tokens whose whole curve life (create hour .. graduation + hour) is in the archive and that have none yet; the archive's values replace the stream's (COALESCEd at graduation).""" + from datetime import timedelta + with transaction() as conn: + todo = conn.execute("SELECT mint, create_ts, graduated_at, creator, supply, dev_tokens FROM corpus_meta " + "WHERE curve_known IS NOT TRUE AND create_ts IS NOT NULL AND graduated_at IS NOT NULL").fetchall() + done = {r["hour"].astimezone(__import__("datetime").timezone.utc) for r in conn.execute("SELECT hour FROM replay_hours WHERE status IN ('done','missing')").fetchall()} + ready = [] + for r in todo: + h = r["create_ts"].replace(minute=0, second=0, microsecond=0); g = r["graduated_at"] + ok = True + while h <= g: + if h not in done: + ok = False; break + h += timedelta(hours=1) + if ok: + ready.append(r) + if not ready: + return 0 + days = sorted({(r["create_ts"] + timedelta(hours=k)).date() for r in ready for k in range(0, int((r["graduated_at"] - r["create_ts"]).total_seconds() // 3600) + 2)}) + trade_files = [str(f) for d in days for f in sorted((config.REPLAY_DIR / d.isoformat()).glob("*_trades.parquet"))] + event_files = [str(f) for d in days for f in sorted((config.REPLAY_DIR / d.isoformat()).glob("*_events.parquet"))] + if not trade_files: + return 0 + import pyarrow as pa + con = duckdb.connect() + con.register("todo", pa.table({"mint": pa.array([r["mint"] for r in ready], pa.string()), + "g": pa.array([r["graduated_at"] for r in ready], pa.timestamp("us", tz="UTC"))})) + slots = dict(con.execute("SELECT e.mint, arg_min(e.slot, e.ts) FROM read_parquet(?, union_by_name = true) e JOIN todo USING (mint) " + "WHERE e.action = 'create' AND e.pool = 'pump' GROUP BY e.mint", [event_files]).fetchall()) if event_files else {} + legs = con.execute("""SELECT t.mint, t.trader, min(t.slot) FILTER (WHERE t.side = 1) AS first_buy, + coalesce(sum(t.tokens) FILTER (WHERE t.side = 1), 0) AS bought, coalesce(sum(t.tokens) FILTER (WHERE t.side = -1), 0) AS sold + FROM read_parquet(?, union_by_name = true) t JOIN todo d ON d.mint = t.mint + WHERE t.pool = 'pump' AND t.ts <= d.g AND t.trader IS NOT NULL GROUP BY t.mint, t.trader""", [trade_files]).df() + con.close() + by_mint = {m: g for m, g in legs.groupby("mint")} if len(legs) else {} + upd, ins = [], [] + for r in ready: + g = by_mint.get(r["mint"]) + wallets = {} if g is None else {w: [None if pd.isna(f) else int(f), float(b), float(s_)] for w, f, b, s_ in zip(g["trader"], g["first_buy"], g["bought"], g["sold"])} + facts, insiders = curve_facts(slots.get(r["mint"]), r["creator"], r["supply"], r["dev_tokens"], wallets) + upd.append((*[_pg(facts[c]) for c in CURVE_COLS], r["mint"])); ins.extend((r["mint"], w, k) for w, k in insiders) + with transaction() as conn: + with conn.cursor() as cur: + cur.executemany("UPDATE corpus_meta SET " + ", ".join(f"{c} = %s" for c in CURVE_COLS) + ", curve_known = true, updated_at = now() WHERE mint = %s", upd) + cur.execute("CREATE TEMP TABLE _ti (mint text, wallet text, kind text) ON COMMIT DROP") + with cur.copy("COPY _ti FROM STDIN") as cp: + for rec in ins: + cp.write_row(rec) + cur.execute("INSERT INTO token_insiders (mint, wallet, kind) SELECT DISTINCT ON (mint, wallet) mint, wallet, kind FROM _ti ON CONFLICT (mint, wallet) DO NOTHING") + log.info("curve facts: %d tokens, %d insider wallets", len(upd), len(ins)) + return len(upd) + + +def _pg(x): + """A pandas/numpy value as a Postgres parameter (text cannot hold NUL bytes; token names sometimes do).""" + if x is None or (isinstance(x, float) and np.isnan(x)) or x is pd.NaT: + return None + if isinstance(x, str): + return x.replace("\x00", "") + if isinstance(x, (np.floating,)): + return float(x) + if isinstance(x, (np.integer,)): + return int(x) + if isinstance(x, (np.bool_,)): + return bool(x) + if isinstance(x, pd.Timestamp): + return x.to_pydatetime() + return x + + +def _complete_days(conn) -> set[str]: + """UTC days whose 24 replay hours are all ingested (their creates can no longer change).""" + rows = conn.execute("SELECT (hour AT TIME ZONE 'UTC')::date AS d, count(*) AS n FROM replay_hours WHERE status IN ('done','missing') GROUP BY 1").fetchall() + return {r["d"].isoformat() for r in rows if r["n"] >= 24} + + +def _backfill_creates(creates: pd.DataFrame) -> int: + """Copy the archive's creates into ``pump_events``, so live creator lookups count the same launches training counts. + Replay ingests hours newest first, so a time watermark would skip older hours that arrive later; instead every + create day not yet recorded as complete is sent (duplicates are no-ops) and complete days are recorded.""" + import json + from datetime import datetime + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'pump_events_backfill'").fetchone() + complete = _complete_days(conn) + done = set(((r or {}).get("value") or {}).get("days") or []) + day = pd.to_datetime(creates["create_ts"], utc=True).dt.strftime("%Y-%m-%d") + new = creates[~day.isin(done) & creates["sig"].notna()] + n = 0 + if not new.empty: + cols = ["sig", "create_ts", "mint", "creator", "dev_sol", "dev_tokens", "supply", "mayhem", "name", "symbol", "uri"] + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("CREATE TEMP TABLE _cr (sig text, ts timestamptz, mint text, signer text, dev_sol float8, dev_tokens float8, supply float8, " + "mayhem boolean, name text, symbol text, uri text) ON COMMIT DROP") + with cur.copy("COPY _cr FROM STDIN") as cp: + for rec in new[cols].itertuples(index=False, name=None): + cp.write_row(tuple(_pg(x) for x in rec)) + cur.execute("INSERT INTO pump_events (sig, ts, action, pool, mint, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri) " + "SELECT sig, ts, 'create', 'pump', mint, signer, dev_sol, dev_tokens, supply, mayhem, name, symbol, uri FROM _cr ON CONFLICT (sig) DO NOTHING") + n = cur.rowcount + done |= set(day.unique()) & complete + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('pump_events_backfill', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps({"days": sorted(done), "at": datetime.now().isoformat()}),)) + return int(n) + + +def backfill_pump_pools() -> int: + """Copy the archive's directly created PumpSwap pools (``createPool`` on ``pump-amm`` not by a pump.fun migration; + missing ``pool_created_by`` counts as custom) into ``pump_pools``. Only day directories not yet recorded as complete + are read (duplicates are no-ops), so a re-run costs the newest day's files.""" + import json + from datetime import datetime + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = 'pump_pools_backfill'").fetchone() + complete = _complete_days(conn) + done = set(((r or {}).get("value") or {}).get("days") or []) + days = {p.name: sorted(str(f) for f in p.glob("*_events.parquet")) for p in config.REPLAY_DIR.glob("*") if p.is_dir() and p.name not in done} + days = {d: fs for d, fs in days.items() if fs} + if not days: + return 0 + files = [f for fs in days.values() for f in fs]; con = duckdb.connect() + cols = [c[0] for c in con.execute("SELECT * FROM read_parquet(?, union_by_name = true) LIMIT 0", [files]).description] + by = "pool_created_by" if "pool_created_by" in cols else "NULL::VARCHAR" + pools = con.execute(f"""SELECT pool_id, arg_min(mint, ts) AS mint, arg_min({by}, ts) AS created_by, min(ts) AS ts FROM read_parquet(?, union_by_name = true) + WHERE action = 'createPool' AND pool = 'pump-amm' AND coalesce({by}, 'custom') <> 'pump' AND pool_id IS NOT NULL GROUP BY pool_id""", [files]).df() + con.close(); n = 0 + if not pools.empty: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("CREATE TEMP TABLE _pp (pool_id text, mint text, created_by text, ts timestamptz) ON COMMIT DROP") + with cur.copy("COPY _pp FROM STDIN") as cp: + for rec in pools[["pool_id", "mint", "created_by", "ts"]].itertuples(index=False, name=None): + cp.write_row(tuple(_pg(x) for x in rec)) + cur.execute("INSERT INTO pump_pools (pool_id, mint, created_by, ts, source) SELECT pool_id, mint, created_by, ts, 'archive' FROM _pp ON CONFLICT (pool_id) DO NOTHING") + n = cur.rowcount + done |= set(days) & complete + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('pump_pools_backfill', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps({"days": sorted(done), "at": datetime.now().isoformat()}),)) + return int(n) + + +def blocked_pool_ids(conn) -> list[str]: + """Blocked pools that can carry universe legs. The stream and ``train/mature.py`` both keep only pump.fun-origin mints + (suffix ``pump``) before this check, so pools of other mints never match either way.""" + return [r["pool_id"] for r in conn.execute("SELECT pool_id FROM pump_pools WHERE mint IS NULL OR right(mint, 4) = 'pump'").fetchall()] + + +def creator_history(conn, creator: str | None, create_ts, graduated_at) -> dict: + """Point-in-time creator history for one token, the same definitions ``rebuild`` computes in bulk: + prior_launches = the creator's creates before this token's creation; prior_grads = the creator's graduations before + this graduation; prior_known / rug / moon share = among those, the ones whose 60-minute outcome was known by now.""" + out = {"prior_launches": None, "prior_grads": None, "prior_known": None, "prior_rug_share": None, "prior_moon_share": None} + if not creator: + return out + if create_ts is not None: + out["prior_launches"] = int(conn.execute("SELECT count(*) AS n FROM pump_events WHERE signer = %s AND action = 'create' AND ts < %s", + (creator, create_ts)).fetchone()["n"]) + r = conn.execute("""SELECT count(*) AS grads, + count(*) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS known, + avg(CASE WHEN own_dd60 <= -0.9 THEN 1.0 ELSE 0.0 END) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS rug, + avg(CASE WHEN own_max60 >= 1.0 THEN 1.0 ELSE 0.0 END) FILTER (WHERE own_dd60 IS NOT NULL AND graduated_at + interval '60 minutes' <= %(g)s) AS moon + FROM corpus_meta WHERE creator = %(c)s AND graduated_at < %(g)s""", {"c": creator, "g": graduated_at}).fetchone() + out["prior_grads"] = int(r["grads"]); out["prior_known"] = int(r["known"]) + out["prior_rug_share"] = float(r["rug"]) if r["rug"] is not None else None + out["prior_moon_share"] = float(r["moon"]) if r["moon"] is not None else None + return out + + +def load_features() -> pd.DataFrame: + with transaction() as conn: + rows = conn.execute("SELECT mint, " + ", ".join(FEATURE_COLS) + " FROM corpus_meta").fetchall() + df = pd.DataFrame(rows) + if df.empty: + return pd.DataFrame(columns=["mint"] + FEATURE_COLS) + df["mayhem"] = df["mayhem"].map({True: 1.0, False: 0.0}) + df["curve_known"] = df["curve_known"].map({True: 1.0, False: 0.0}) + return df + + +def main() -> None: + from ..logging_setup import setup + setup("replay") + prev = None + while True: + n = rebuild() + with transaction() as conn: + left = conn.execute("SELECT count(*) AS n FROM corpus_tokens t LEFT JOIN corpus_meta m USING (mint) WHERE t.candle_path IS NOT NULL AND t.status = 'done' AND (m.mint IS NULL OR m.own_alive6h IS NULL)").fetchone()["n"] + print(f"corpus_meta: {n} tokens; outcomes still missing for {left}") + if not left or left == prev: + break + prev = left diff --git a/fly_trader/train/decisions.py b/fly_trader/train/decisions.py new file mode 100644 index 0000000000000000000000000000000000000000..59450588bf4e537bbbb79fa635901f6bc1146dc2 --- /dev/null +++ b/fly_trader/train/decisions.py @@ -0,0 +1,253 @@ +"""Decision points shared by the selector models: one row per eligible minute of the mature universe. + +Rows come from ``train/mature.py`` feature parts (pump.fun-origin PumpSwap tokens of any age, 1-minute candles +fed through the live feature engine) joined with ``corpus_meta`` creation-time and creator columns. Eligible = +pool ≥ ``MIN_RESQ_SOL``, 15-minute volume ≥ ``MIN_VOL_15M_SOL``, contamination-free series, and a full label horizon ahead. +Label: net forward return over ``horizon_min`` (fill at the signal minute's close; ``fwd_pess`` fills at the +next traded minute's open) after trading costs; ``y`` = return above ``label_thr``. Costs default to the paper +broker's model on both sides (``exit_cost_0p1``: the real fees — pool fee tier by market cap, Jupiter 10 bps, network +fee — + constant-product impact of a 0.1 SOL position; entry at the signal minute, exit at the exit minute) — the same costs the paper book pays. A flat +``fee`` round trip can be given instead (tests, sensitivity runs). ``random_trades`` is the no-skill baseline. +""" +from __future__ import annotations + +import glob +import math +from dataclasses import dataclass, field +from datetime import date + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq + +from .. import config +from ..market.features import FEATURE_VERSION, FEATURES +from .corpus_features import _epoch_s +from .corpus_meta import CURVE_COLS, FEATURE_COLS as META_COLS, load_features +from .flow import FLOW_COLS, MARKET_COLS, SKILL_COLS +from .mature import part_current + +EXTRA_COLS = ["traders_15m", "traders_1h", "n_trades_1m", "hod_s", "hod_c", "age_known", "meta_known"] +# Inputs that carry no information (feature ablation 2026-09-15: the 120-min walk-forward over 149 days at real costs, +# repeated with each group removed; removing these lowered neither the rank correlation with net returns nor the +# out-of-sample profit): five stream features the archive cannot reproduce and the eleven Jupiter stats (always 0 — no +# history before 2026-09-12), realized volatility (IC 0.1221 → 0.1240, +0.29 → +0.38 SOL/day without it) and the +# creator's launch history (0.1221 → 0.1218, +0.29 → +0.29). All removed together (37 of 62 inputs): IC 0.1191, +# +0.50 SOL/day, +3.0 % per trade (all inputs: +1.5 %). +DROPPED_COLS = frozenset(["logsigners_15m", "logsigners_1h", "hawkes", "log_since_last", "vpin_15m", + "organic_score", "log_holders", "log_liq_usd", "top_holders_pct", "dev_balance_pct", "is_sus", "is_verified", + "net_buyers_1h", "holder_change_1h", "price_change_24h", "token2022", + "rvol_5m", "rvol_15m", "rvol_1h", "rvol_3h", + "prior_launches", "prior_grads", "prior_moon_share"]) +# Input groups of the strategy stack (2026-09-15), each kept only if the walk-forward objective says so (train/selector.py): +# rug — graduation-time curve facts, the creator's rug history and insider selling; flow — wallet flow and market volume; +# skill — buying by skilled wallets. LEGACY_COLS are the 37 inputs of the single-strategy selector. +GROUPS = {"rug": [*CURVE_COLS, "curve_known", "prior_rug_share", "prior_known", "insider_sell_share_1m", "log_insider_sell_15m"], + "flow": ["top_sell_share_1m", "buyers_ret", "buyers_slope", "org_imb_5m", "org_imb_15m", "wash_share_15m", "log_org_vol_15m", *MARKET_COLS], + "skill": list(SKILL_COLS)} +_GROUPED = {c for g in GROUPS.values() for c in g} +LEGACY_COLS = [c for c in FEATURES + EXTRA_COLS + META_COLS if c not in DROPPED_COLS and c not in _GROUPED] +X_COLS = LEGACY_COLS + [c for g in GROUPS.values() for c in g] +# Hold and gates fitted to the market at real costs (2026-09-15 sweep over 149 days: holds 10–240 min × age × pool × +# 15-min volume × buy line, walk-forward): a 120-minute hold with pools ≥ 10 SOL and ≥ 5 SOL traded in 15 minutes (and +# tokens ≥ 6 h past graduation, train/selector.MIN_AGE_H) made money in every month and on 46–49 of 64 days in each +# half; the old 30 min / 20 SOL / 24 h lost money May–July. +HOLD_MIN = 120 +MIN_RESQ_SOL, MIN_VOL_15M_SOL = 10.0, 5.0 + + +@dataclass +class DecisionSet: + X: np.ndarray # [N, F] float32, raw (not standardized) + y: np.ndarray # [N] int8 + fwd: np.ndarray # [N] float32 net forward return, close fill + fwd_pess: np.ndarray # [N] float32 net forward return, next-open fill + day: np.ndarray # [N] datetime.date + ts: np.ndarray # [N] float64 epoch seconds + mint: np.ndarray # [N] str + cols: list[str] + horizon_s: float + fwd_h: dict = field(default_factory=dict) # hold (min) → [N] float32 net return (next-open fill) over that hold; NaN past the data + + @property + def days(self) -> list[date]: + return sorted(set(self.day.tolist())) + + def subset(self, mask: np.ndarray) -> "DecisionSet": + """The rows of ``mask`` (e.g. the days before a bootstrap), labels for every hold included.""" + m = np.asarray(mask, dtype=bool) + return DecisionSet(X=self.X[m], y=self.y[m], fwd=self.fwd[m], fwd_pess=self.fwd_pess[m], day=self.day[m], ts=self.ts[m], mint=self.mint[m], + cols=list(self.cols), horizon_s=self.horizon_s, fwd_h={h: v[m] for h, v in self.fwd_h.items()}) + + def mask_days(self, lo: date | None = None, hi: date | None = None) -> np.ndarray: + m = np.ones(len(self.y), bool) + if lo is not None: + m &= self.day >= lo + if hi is not None: + m &= self.day <= hi + return m + + +def _label_exit_cost(df) -> np.ndarray: + """One side's real cost per row, at the size the book trades (market/exit_cost.cost_at_size). The live engine puts + the same value in every bar it hands the fly (agent/minute_engine.py), so what the fly learns from and what it was + trained on are one cost model.""" + from ..market.exit_cost import cost_at_size + return cost_at_size(df["exit_cost_0p1"].to_numpy(dtype=float), df["resq"].to_numpy(dtype=float)) + + +def build(days: int | None = 45, horizon_min: int = HOLD_MIN, fee: float | None = None, label_thr: float = 0.03, feature_dir=None, holds=None) -> DecisionSet: + """``holds``: extra holds (minutes) whose labels are computed in the same pass (``DecisionSet.fwd_h``).""" + root = feature_dir or (config.CORPUS_DIR / "features_mature") + files = sorted(glob.glob(str(root / "*" / "part.parquet"))) + if days: + files = files[-days - 1:] + if not files: + raise RuntimeError("no mature feature parts; run build-mature") + stale = [f for f in files if not part_current(f)] + if stale: + raise RuntimeError(f"{len(stale)} feature part(s) were built with another feature or aggregation version (e.g. {stale[0]}); run build-mature") + need = ["mint", "ts", "open", "close", "resq", "age_h", "traders_15m", "traders_1h", "n_trades_1m"] + FEATURES + FLOW_COLS + SKILL_COLS + MARKET_COLS + optional = set(FLOW_COLS + SKILL_COLS + MARKET_COLS) # parts that predate the wallet-flow columns (tests' synthetic parts) read as 0 + df = pd.concat([pq.read_table(f, columns=[c for c in need if c not in optional or c in pq.read_schema(f).names]).to_pandas() for f in files], + ignore_index=True).sort_values(["mint", "ts"]).reset_index(drop=True) + for c in optional: + if c not in df: + df[c] = 0.0 + # a mint whose series breaks scale (secondary pool in another quote, impossible reserve) is ineligible FROM THAT MINUTE ON — + # never before it, so no decision row is judged with the token's future (rows before a dump keep their labels) + jump = (df["close"] / df.groupby("mint")["close"].shift(1)).fillna(1.0) + df["_off"] = ((jump > 50) | (jump < 1 / 50) | (df["resq"] > 1e5)).astype(np.int8) + broken = df.groupby("mint")["_off"].cummax().to_numpy().astype(bool) + df = df.merge(load_features(), on="mint", how="left") + df["meta_known"] = df["ttg_min"].notna().astype(np.float32) if "ttg_min" in df else np.float32(0.0) + H = horizon_min * 60.0 + ts = _epoch_s(df["ts"]); cl = df["close"].to_numpy(); op = df["open"].to_numpy() + mints = df["mint"].to_numpy(); starts = np.r_[0, np.flatnonzero(mints[1:] != mints[:-1]) + 1, len(mints)] + fwd = np.full(len(df), np.nan); fwdp = np.full(len(df), np.nan) + hold_list = sorted({int(h) for h in (holds or [])}); fh = {h: np.full(len(df), np.nan) for h in hold_list} + ec = _label_exit_cost(df) if fee is None else None # one-side cost fraction per minute, at the size the book trades + for s, e in zip(starts[:-1], starts[1:]): + t = ts[s:e]; j = np.searchsorted(t, t + H, side="right") - 1 + s + exit_px = cl[j] + keep = (1 - ec[s:e]) * (1 - ec[j]) if fee is None else 1 - fee + fwd[s:e] = exit_px / cl[s:e] * keep - 1 + c = cl[s:e] # an exit on a one-minute print 50x off that reverts the next minute is a data artifact + if e - s > 2: + r_prev = np.r_[1.0, c[1:] / c[:-1]]; r_next = np.r_[c[1:] / c[:-1], 1.0] + spike = ((r_prev > 50) & (r_next < 1 / 50)) | ((r_prev < 1 / 50) & (r_next > 50)) + fwd[s:e][spike[j - s]] = np.nan + entry = cl[s:e].copy() # pessimistic entry: the next traded minute's open when it is within 2 minutes + if e - s > 1: + nxt_ok = np.r_[np.diff(t) <= 120.0, False] + idx = np.flatnonzero(nxt_ok); entry[idx] = op[s + idx + 1] + fwdp[s:e] = exit_px / entry * keep - 1 + for hm in hold_list: # the same label for another hold: exit at the last close at or before t + hold + jh = np.searchsorted(t, t + hm * 60.0, side="right") - 1 + s + kh = (1 - ec[s:e]) * (1 - ec[jh]) if fee is None else 1 - fee + v = cl[jh] / entry * kh - 1 + if e - s > 2: + v[spike[jh - s]] = np.nan + fh[hm][s:e] = v + hod = df["ts"].dt.hour + df["ts"].dt.minute / 60.0 + df["hod_s"], df["hod_c"] = np.sin(2 * np.pi * hod / 24), np.cos(2 * np.pi * hod / 24) + df["age_known"] = df["age_h"].notna().astype(np.float32) + df["mayhem"] = df["mayhem"].astype(float) if "mayhem" in df else 0.0 + last_ts = df["ts"].max() + elig = (np.isfinite(df["resq"]) & (df["resq"] >= MIN_RESQ_SOL) & (df["logvol_15m"] >= math.log1p(MIN_VOL_15M_SOL)) + & np.isfinite(fwd) & (df["ts"] < last_ts - pd.Timedelta(minutes=horizon_min + 5))).to_numpy() & ~broken + for hm in hold_list: # no label where that hold runs past the data + fh[hm][(df["ts"] >= last_ts - pd.Timedelta(minutes=hm + 5)).to_numpy()] = np.nan + df = df[elig].reset_index(drop=True); fwd = fwd[elig]; fwdp = fwdp[elig]; ts = ts[elig] + X = df[X_COLS].astype(np.float32).replace([np.inf, -np.inf], np.nan).fillna(0.0).to_numpy() + return DecisionSet(X=X, y=(fwd > label_thr).astype(np.int8), fwd=fwd.astype(np.float32), fwd_pess=fwdp.astype(np.float32), + day=df["ts"].dt.date.to_numpy(), ts=ts, mint=df["mint"].to_numpy(), cols=list(X_COLS), horizon_s=H, + fwd_h={hm: fh[hm][elig].astype(np.float32) for hm in hold_list}) + + +def taken_idx(ts: np.ndarray, mint: np.ndarray, horizon_s: float, idx: np.ndarray) -> np.ndarray: + """Of the candidate row indices ``idx``, the ones a picker takes: one position per token, re-entry only once the hold + has passed (the rule of ``trades_from_picks``), in (mint, ts) order. Vectorised: each taken pick jumps to the token's + first candidate at least ``horizon_s`` later, so the loop runs once per trade of the busiest token, not once per row.""" + idx = np.asarray(idx, dtype=int) + if len(idx) == 0: + return idx + idx = idx[np.lexsort((ts[idx], mint[idx]))] + t = np.asarray(ts[idx], dtype=np.float64); m = mint[idx] + hv = np.asarray(horizon_s, dtype=np.float64) # a scalar, or one hold per row of ``ts`` (each position exits after its own) + hs = np.full(len(idx), float(hv)) if hv.ndim == 0 else hv[idx] + if not (hs > 0).all(): # a zero hold lets a token re-enter at the same minute: the frontier below would never advance + raise ValueError(f"{int((hs <= 0).sum())} of {len(hs)} picked rows have a hold of zero or less; a position must be held for a positive time") + first = np.r_[True, m[1:] != m[:-1]]; gid = np.cumsum(first) - 1; starts = np.flatnonzero(first) + ends = np.r_[starts[1:], len(idx)] + span = float(t.max() - t.min()) + float(hs.max()) + 1.0 + key = gid * span + (t - t.min()) # sorted: (token, time) in one exact float64 key + nxt = np.searchsorted(key, key + hs, side="left") + has_next = nxt < ends[gid] + out = []; frontier = starts + while len(frontier): + out.append(frontier) + frontier = nxt[frontier[has_next[frontier]]] + return idx[np.sort(np.concatenate(out))] + + +def taken_rows(ds: DecisionSet, pick: np.ndarray, hold_s=None) -> np.ndarray: + """Row indices of the trades a picker takes (the same rule as ``trades_from_picks``); ``hold_s``: per-row holds.""" + return taken_idx(ds.ts, ds.mint, ds.horizon_s if hold_s is None else hold_s, np.flatnonzero(pick)) + + +def trades_from_picks(ds: DecisionSet, pick: np.ndarray, returns: np.ndarray | None = None, with_days: bool = False, hold_s=None): + """Net returns of the trades a picker would take: one position per token, re-entry only after the hold (``hold_s``: + per-row holds, each position exits after its own). With ``with_days`` also returns the entry day of each trade (so + per-day tables respect holds across midnight).""" + r_all = ds.fwd_pess if returns is None else returns + tr = taken_idx(ds.ts, ds.mint, ds.horizon_s if hold_s is None else hold_s, np.flatnonzero(pick)) + r = np.asarray(r_all[tr], dtype=np.float32) + return (r, np.asarray(ds.day[tr], dtype=object)) if with_days else r + + +def random_trades(ds: DecisionSet, rows: np.ndarray, n_picks: int, seeds: int = 20, hold_s=None, returns: np.ndarray | None = None) -> np.ndarray: + """The no-skill baseline: ``n_picks`` random minutes among ``rows`` (the model's pick count on the same eligible + minutes), traded with the same hold rule, fills and costs; ``seeds`` draws pooled.""" + idx = np.flatnonzero(rows) + if n_picks <= 0 or len(idx) == 0: + return np.array([], dtype=np.float32) + out = [] + for s in range(seeds): + pick = np.zeros(len(ds.y), bool); pick[np.random.default_rng(s).choice(idx, min(n_picks, len(idx)), replace=False)] = True + out.append(trades_from_picks(ds, pick, returns=returns, hold_s=hold_s)) + return np.concatenate(out) + + +def _ranks(a: np.ndarray) -> np.ndarray: + """1-based ranks with ties sharing their average rank (what Spearman needs).""" + _, inv, cnt = np.unique(np.asarray(a), return_inverse=True, return_counts=True) + avg = np.cumsum(cnt) - (cnt - 1) / 2.0 + return avg[inv.ravel()] + + +def rank_corr(a: np.ndarray, b: np.ndarray) -> float | None: + """Spearman correlation (Pearson on average ranks); None when either side is constant or empty.""" + a = np.asarray(a, dtype=np.float64); b = np.asarray(b, dtype=np.float64) + if len(a) < 2 or len(a) != len(b): + return None + ra = _ranks(a); rb = _ranks(b); ra -= ra.mean(); rb -= rb.mean() + den = float(np.sqrt((ra * ra).sum() * (rb * rb).sum())) + return float((ra * rb).sum() / den) if den > 0 else None + + +def summarize(r: np.ndarray) -> dict: + if len(r) == 0: + return {"n": 0, "mean": None, "median": None, "win": None, "pf": None} + g = float(r[r > 0].sum()); l = float(-r[r <= 0].sum()) + return {"n": int(len(r)), "mean": float(r.mean()), "median": float(np.median(r)), "win": float((r > 0).mean()), "pf": (g / l) if l > 0 else None} # None: no losing trade (JSON has no inf) + + +def evaluate(ds: DecisionSet, scores: np.ndarray, test: np.ndarray, threshold, label: str = "", hold_s=None, returns: np.ndarray | None = None) -> dict: + """Trades on rows of ``test`` whose score ≥ threshold (a scalar or per-row lines); metrics per day and pooled.""" + pick = test & (scores >= threshold); r, rdays = trades_from_picks(ds, pick, returns=returns, with_days=True, hold_s=hold_s) + per_day = {} + for d in sorted(set(ds.day[test].tolist())): + per_day[str(d)] = summarize(r[rdays == d]) + pooled = summarize(r); pooled["days_positive"] = sum(1 for v in per_day.values() if v["mean"] is not None and v["mean"] > 0); pooled["days"] = len(per_day) + return {"label": label, "pooled": pooled, "per_day": per_day} diff --git a/fly_trader/train/flow.py b/fly_trader/train/flow.py new file mode 100644 index 0000000000000000000000000000000000000000..b063353ee23f5cf02168e172827316401bcfd58d --- /dev/null +++ b/fly_trader/train/flow.py @@ -0,0 +1,143 @@ +"""Per-minute wallet flow: one definition for the archive (``train/mature.py``) and the live stream +(``ingest/pumpstream.py``), so the selector's new inputs are computed identically in training and live. + +Per (mint, minute), over the minute's dominant-pool legs with a known trader (both paths drop the others): each +trader's bought and sold SOL → ``n_buyers`` (traders who bought), ``wash_sol`` (bought + sold SOL of traders who did +both in the minute: round trips, the wash-trading signature — Szwajcok et al. 2026), ``wash_buy_sol`` (their buy side), +``top_sell_sol`` (the largest single trader's selling), ``insider_sell_sol`` (selling by the token's creator, bundle +and early wallets: ``token_insiders``), ``skill_buy`` (bought SOL by wallet-skill decile 0–9 of the day's table, +index 10 = wallet without a score; None when the day has no table). The archive computes them in SQL +(``mature.aggregate_day``), the stream with ``minute_wallet_cols``; ``tests/test_flow_parity.py`` holds them equal. + +``FlowWindow`` turns a mint's minutes into the derived model inputs (``FLOW_COLS`` + ``SKILL_COLS``), fed the same way +by ``mature.build_day`` and the live engine (``agent/minute_engine.py``); ``mkt_vol_1h`` is the market's total SOL volume +over the last hour (all mints), computed from the same minute rows in both paths. +""" +from __future__ import annotations + +import math +from collections import deque + +import numpy as np + +N_SKILL = 11 # deciles 0..9 (9 = most skilled) + 10 = no score +UNKNOWN = 10 +WINDOW_S = 3600.0 +WINDOW_15M_S = 900 # the skill inputs' window (FlowWindow's r15) +FLOW_COLS = ["top_sell_share_1m", "insider_sell_share_1m", "log_insider_sell_15m", "buyers_ret", "buyers_slope", + "org_imb_5m", "org_imb_15m", "wash_share_15m", "log_org_vol_15m"] +SKILL_COLS = [f"skill_top{q}_15m" for q in range(1, 10)] + ["skill_known_15m"] +MARKET_COLS = ["mkt_vol_1h"] +RAW_COLS = ["n_buyers", "wash_sol", "wash_buy_sol", "top_sell_sol", "insider_sell_sol", "skill_buy"] + + +def minute_wallet_cols(traders: dict, insiders=frozenset(), skill: dict | None = None) -> dict: + """The raw wallet columns of one minute from ``traders``: wallet → [bought SOL, sold SOL].""" + n_buyers = 0; wash = wash_b = top_sell = ins_sell = 0.0 + sk = None if skill is None else [0.0] * N_SKILL + for w, (b, s) in traders.items(): + if b > 0: + n_buyers += 1 + if b > 0 and s > 0: + wash += b + s; wash_b += b + top_sell = max(top_sell, s) + if w in insiders: + ins_sell += s + if sk is not None: + sk[skill.get(w, UNKNOWN)] += b + return {"n_buyers": n_buyers, "wash_sol": wash, "wash_buy_sol": wash_b, "top_sell_sol": top_sell, "insider_sell_sol": ins_sell, "skill_buy": sk} + + +def _z(x) -> float: + try: + v = float(x) + except (TypeError, ValueError): + return 0.0 + return v if math.isfinite(v) else 0.0 + + +class FlowWindow: + """A mint's last hour of minutes (only minutes it traded) → the derived flow inputs at a minute's end.""" + __slots__ = ("q",) + + def __init__(self): + self.q: deque = deque() + + def append(self, t_end: float, buy: float, sell: float, n_buyers, wash_sol, wash_buy_sol, top_sell_sol, insider_sell_sol, skill_buy) -> None: + sk = None if skill_buy is None else [_z(v) for v in skill_buy] + self.q.append((t_end, _z(buy), _z(sell), _z(n_buyers), _z(wash_sol), _z(wash_buy_sol), _z(top_sell_sol), _z(insider_sell_sol), sk)) + while self.q and self.q[0][0] <= t_end - WINDOW_S: + self.q.popleft() + + def features(self, t_end: float) -> dict: + rows = [r for r in self.q if r[0] <= t_end] + cur = rows[-1] if rows and rows[-1][0] == t_end else None + + def win(lo, hi=t_end): + return [r for r in rows if lo < r[0] <= hi] + + def s(rs, k): + return sum(r[k] for r in rs) + + r15, r5 = win(t_end - 900), win(t_end - 300) + buy15, sell15, wash15 = s(r15, 1), s(r15, 2), s(r15, 4) + out = {"top_sell_share_1m": cur[6] / cur[2] if cur and cur[2] > 0 else 0.0, + "insider_sell_share_1m": cur[7] / cur[2] if cur and cur[2] > 0 else 0.0, + "log_insider_sell_15m": math.log1p(s(r15, 7)), + "buyers_ret": math.log1p(cur[3] if cur else 0.0) - math.log1p(s(win(t_end - 360, t_end - 60), 3) / 5.0), + "buyers_slope": math.log1p(s(r5, 3)) - math.log1p(s(win(t_end - 600, t_end - 300), 3)), + "wash_share_15m": wash15 / (buy15 + sell15) if buy15 + sell15 > 0 else 0.0, + "log_org_vol_15m": math.log1p(max(buy15 + sell15 - wash15, 0.0))} + for name, rs in (("org_imb_5m", r5), ("org_imb_15m", r15)): + ob = s(rs, 1) - s(rs, 5); os_ = s(rs, 2) - (s(rs, 4) - s(rs, 5)) + out[name] = (ob - os_) / (ob + os_) if ob + os_ > 0 else 0.0 + sk = [0.0] * N_SKILL + for r in r15: + if r[8] is not None: + for k in range(N_SKILL): + sk[k] += r[8][k] + total = sum(sk); known = total - sk[UNKNOWN] + for q in range(1, 10): + out[f"skill_top{q}_15m"] = sum(sk[q:UNKNOWN]) / total if total > 0 else 0.0 + out["skill_known_15m"] = known / total if total > 0 else 0.0 + return out + + +def skill_features(mint: np.ndarray, ts_s: np.ndarray, skill_buy: np.ndarray) -> np.ndarray: + """``FlowWindow``'s skill inputs (``SKILL_COLS``, in order) for many minute rows at once: row i's window is its mint's + rows starting in (t_i − 900 s, t_i]; ``skill_buy`` [n, N_SKILL] with zeros where the minute had no table (FlowWindow + skips those minutes).""" + n = len(ts_s); out = np.zeros((n, len(SKILL_COLS)), np.float32) + if not n: + return out + _, code = np.unique(np.asarray(mint).astype(str), return_inverse=True) + key = code.astype(np.int64) * 10**10 + np.asarray(ts_s, dtype=np.int64) + order = np.argsort(key, kind="stable"); ks = key[order]; sb = np.asarray(skill_buy, dtype=np.float64)[order] + cs = np.vstack([np.zeros((1, N_SKILL)), np.cumsum(sb, axis=0)]); cn = np.r_[0, np.cumsum(sb.sum(1) > 0)] + lo = np.searchsorted(ks, ks - WINDOW_15M_S, side="right"); hi = np.arange(1, n + 1) + w = cs[hi] - cs[lo]; has = (cn[hi] - cn[lo]) > 0 # a window without a table minute stays exactly 0 + total = w.sum(1); ok = has & (total > 0); res = np.zeros((n, len(SKILL_COLS))) + for j, q in enumerate(range(1, 10)): + res[ok, j] = w[ok, q:UNKNOWN].sum(1) / total[ok] + res[ok, 9] = (total[ok] - w[ok, UNKNOWN]) / total[ok] + out[order] = np.clip(res, 0.0, 1.0) + return out + + +class MarketWindow: + """The market's total SOL volume per minute (every mint's buy + sell) over the last hour → ``mkt_vol_1h``.""" + __slots__ = ("q",) + + def __init__(self): + self.q: deque = deque() + + def add(self, t_end: float, vol: float) -> None: + if self.q and self.q[-1][0] == t_end: + self.q[-1] = (t_end, self.q[-1][1] + vol) + else: + self.q.append((t_end, vol)) + while self.q and self.q[0][0] <= t_end - WINDOW_S: + self.q.popleft() + + def value(self, t_end: float) -> float: + return math.log1p(sum(v for t, v in self.q if t_end - WINDOW_S < t <= t_end)) diff --git a/fly_trader/train/fly_calibrate.py b/fly_trader/train/fly_calibrate.py new file mode 100644 index 0000000000000000000000000000000000000000..b2111d3144a56a6cc67a7e47fbc80a957a41a93a --- /dev/null +++ b/fly_trader/train/fly_calibrate.py @@ -0,0 +1,55 @@ +"""The fly's buy line and sizing bands, recalibrated from its own scores and the realized outcomes — no training. + +Every day (live: the first minute after 00:00 UTC; replay: every day boundary) on the scored minutes whose labels +resolved in the last ``WINDOW_DAYS`` days. Candidate lines: the selector's ``LINE_CANDIDATES`` plus the window's score +quantiles (a compressed score scale still gets a usable line); the line with the most total net profit (fixed size, one +position per token, the hold) over at least ``MIN_LINE_TRADES`` trades wins — the selector's rule — otherwise the previous line +stays. Sizing: ``agent/sizing.build_table`` on the window's trades at that line (the previous table if too few). +The fly never trades the selector's line: its scores live on their own scale. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np + +from ..agent import sizing +from .decisions import taken_idx +from .selector import LINE_CANDIDATES, MIN_EV, MIN_LINE_TRADES + +WINDOW_DAYS = 7 +QUANTILES = (0.95, 0.98, 0.99, 0.995, 0.998, 0.999) + + +@dataclass +class Calibration: + line: float + sizing: list + trades: int # trades at the chosen line in the window + mean: float | None # their mean net return + total: float # their summed net return (fixed size) + changed: bool + lines: list = field(default_factory=list) + + +def candidates(scores: np.ndarray) -> list[float]: + s = np.asarray(scores, dtype=np.float64); s = s[np.isfinite(s)] + q = np.quantile(s, QUANTILES).tolist() if len(s) else [] + return sorted({float(c) for c in (*LINE_CANDIDATES, *q)}) + + +def calibrate(ts: np.ndarray, mint: np.ndarray, horizon_s: float, scores: np.ndarray, rets: np.ndarray, prev_line: float | None = None, + prev_sizing: list | None = None, min_trades: int = MIN_LINE_TRADES) -> Calibration: + """Line and sizing from one window of (ts, mint, score, realized net return) rows.""" + scores = np.asarray(scores, dtype=np.float64); rets = np.asarray(rets, dtype=np.float64) + rows = np.flatnonzero(np.isfinite(scores) & np.isfinite(rets)) + table = [] + for c in candidates(scores[rows]): + tr = taken_idx(ts, mint, horizon_s, rows[scores[rows] >= c]); r = rets[tr] + table.append({"line": c, "trades": int(len(r)), "mean": float(r.mean()) if len(r) else None, "total": float(r.sum())}) + ok = [t for t in table if t["trades"] >= min_trades] + line = max(ok, key=lambda t: t["total"])["line"] if ok else (prev_line if prev_line is not None else MIN_EV) + tr = taken_idx(ts, mint, horizon_s, rows[scores[rows] >= line]); r = rets[tr] + table_s = sizing.build_table(scores[tr] - line, r) or list(prev_sizing or []) + return Calibration(line=float(line), sizing=table_s, trades=int(len(r)), mean=float(r.mean()) if len(r) else None, total=float(r.sum()), + changed=prev_line is None or float(line) != float(prev_line), lines=table) diff --git a/fly_trader/train/fly_governance.py b/fly_trader/train/fly_governance.py new file mode 100644 index 0000000000000000000000000000000000000000..65950df9a0bd91f9d80b840b6adc73b1628b2fd9 --- /dev/null +++ b/fly_trader/train/fly_governance.py @@ -0,0 +1,66 @@ +"""The checks that roll the fly's plasticity back (live, hourly) — and are counted in the replay to measure false alarms. + +Any one trips a rollback: +- shadow: over the last ``SHADOW_DAYS`` days of resolved picks (each arm at its own line, one position per token), the + plastic fly's mean net return per trade is more than one standard error below its frozen shadow's (the bootstrap fly, + D = 0, scoring the same minutes), with at least ``SHADOW_MIN_TRADES`` trades on each side; +- ic: the rank correlation of its scores with the realized returns over the last ``IC_HOURS`` hours is below 0 + (at least ``IC_MIN_ROWS`` resolved minutes); +- drift: ‖D‖_F / ‖w0‖_F above ``DRIFT_MAX``. +""" +from __future__ import annotations + +import numpy as np + +from .decisions import rank_corr, taken_idx + +SHADOW_DAYS, SHADOW_MIN_TRADES = 3, 30 +IC_HOURS, IC_MIN_ROWS, IC_SAMPLE = 24, 5000, 50_000 +DRIFT_MAX = 0.5 + + +def shadow_check(r_plastic: np.ndarray, r_frozen: np.ndarray, min_trades: int = SHADOW_MIN_TRADES) -> tuple[bool, dict]: + rp, rf = np.asarray(r_plastic, dtype=np.float64), np.asarray(r_frozen, dtype=np.float64) + d = {"n_plastic": int(len(rp)), "n_frozen": int(len(rf)), "mean_plastic": float(rp.mean()) if len(rp) else None, "mean_frozen": float(rf.mean()) if len(rf) else None} + if len(rp) < min_trades or len(rf) < min_trades: + return False, d + se = float(np.sqrt(rp.var(ddof=1) / len(rp) + rf.var(ddof=1) / len(rf))) + d["se"] = se + return bool(rp.mean() - rf.mean() < -se), d + + +def ic_check(scores: np.ndarray, labels: np.ndarray, min_rows: int = IC_MIN_ROWS, sample: int = IC_SAMPLE, seed: int = 0) -> tuple[bool, float | None]: + s, y = np.asarray(scores, dtype=np.float64), np.asarray(labels, dtype=np.float64) + ok = np.isfinite(s) & np.isfinite(y); s, y = s[ok], y[ok] + if len(s) < min_rows: + return False, None + if len(s) > sample: + i = np.random.default_rng(seed).choice(len(s), sample, replace=False); s, y = s[i], y[i] + ic = rank_corr(s, y) + return bool(ic is not None and ic < 0), ic + + +def drift_check(drift: float, cap: float = DRIFT_MAX) -> bool: + return bool(drift > cap) + + +def picks_returns(ts: np.ndarray, mint: np.ndarray, horizon_s: float, scores: np.ndarray, lines: np.ndarray, rets: np.ndarray) -> np.ndarray: + """Net returns of the trades an arm took: rows scored at/above the line in force, one position per token.""" + idx = np.flatnonzero(np.asarray(scores) >= np.asarray(lines)) + return np.asarray(rets)[taken_idx(ts, mint, horizon_s, idx)] + + +def check(shadow: tuple[np.ndarray, np.ndarray] | None, ic_rows: tuple[np.ndarray, np.ndarray] | None, drift: float) -> dict: + """All three checks; ``triggers`` lists the ones that tripped.""" + out = {"triggers": [], "drift": float(drift)} + if shadow is not None: + t, d = shadow_check(*shadow); out["shadow"] = d + if t: + out["triggers"].append("shadow") + if ic_rows is not None: + t, ic = ic_check(*ic_rows); out["ic"] = ic + if t: + out["triggers"].append("ic") + if drift_check(drift): + out["triggers"].append("drift") + return out diff --git a/fly_trader/train/fly_replay.py b/fly_trader/train/fly_replay.py new file mode 100644 index 0000000000000000000000000000000000000000..f910b8de6f48e42446987667c976b493ca4ed346 --- /dev/null +++ b/fly_trader/train/fly_replay.py @@ -0,0 +1,344 @@ +"""The proof before the fly trades: bootstrapped once, then left to learn from the market for weeks, on the corpus. + +1. Decision points for the whole corpus with every strategy hold's labels (``train/decisions.build``); ``S`` = the + ``start_day``-th day. The fly is bootstrapped for ``S`` (``fly_selector.bootstrap``: the selector's strategy stack, + fitted on the days before ``S − 8`` only, teaches it once, one head per strategy on its own dopamine channel; each + strategy calibrates its own line on the week before ``S``). The stack's walk-forward needs enough days in each of its + two halves, so ``S`` is at least ``DEFAULT_START`` days in. Nothing is retrained after that. +2. Every tradable minute from ``S`` on, in time order, under every configuration of ``CONFIGS`` side by side (α, τ½; + α = 0 is the frozen fly) — exactly what the live fly does each minute: + a. tags whose labels are known (scored at ``t`` with ``t + the strategy's hold + 60 s`` ≤ now) are captured: the + realized net return over that strategy's hold teaches only that strategy's channel (``brain/plastic.py``); + b. at each UTC day boundary each strategy's line and sizing are recalibrated (``train/fly_calibrate.py``) on the + frozen fly's scores of the minutes resolved in the last seven days and shared by every configuration, so the + arms differ only in their weights (per-configuration lines once traded different slices: ``_recalibrate``); + c. each strategy's candidate rows (the selector's trigger) are scored; only the rows a configuration would have + traded (at or above its own line) teach it -- the candidate set is overwhelmingly losers and learning from it + dragged every prediction toward that mean (brain/plastic.row_weights); + d. every hour the rollback checks (``train/fly_governance.py``) are evaluated per strategy and counted, not acted on. +3. Each strategy's configuration is the one with the most total net profit on its own trades of the first half of the + replay days (≥ 100 trades; plastic configurations only). Channels are disjoint, so the combination is exact. The + combined book (per row the strategy the fly's own decision picks, each position on its own hold) is judged on the + second half against the frozen fly and random picks with the selector's deploy rule. The verdict is stored in + ``ui_settings['fly_replay']`` (with ``FLY_VERSION``); the live fly needs a passed verdict. +""" +from __future__ import annotations + +import json +import logging +import math +import threading +import time +from datetime import date, datetime, timezone + +import numpy as np +import torch + +from ..brain import plastic +from ..db.apilog import record_event +from ..db.connection import transaction +from . import fly_calibrate, fly_governance as gov, fly_selector, selector +from . import progress as prog +from .decisions import DecisionSet, build, evaluate, random_trades, summarize, taken_idx + +log = logging.getLogger(__name__) +KEY = "fly_replay" +DEFAULT_START = selector.WARMUP_DAYS + 2 * 35 + fly_selector.CALIB_DAYS + fly_selector.PURGE_DAYS # 99: 5 weeks in each of the stack's walk-forward halves +START_DAY = DEFAULT_START +CHUNK = 2048 +CONFIGS = [(0.0, math.inf)] + [(a, h) for a in (1e-4, 3e-4, 1e-3, 3e-3) for h in (1.0, 3.0, 7.0, 30.0)] +LABEL_LAG_S = 60.0 # a label is known once the exit minute has closed +HOUR_S, DAY_S = 3600.0, 86400.0 + + +def _day_start(t: float) -> float: + return math.floor(t / DAY_S) * DAY_S + + +def _labels(ds: DecisionSet, hold_min: int) -> np.ndarray: + return ds.fwd_h[hold_min] if hold_min in ds.fwd_h else ds.fwd_pess + + +def run(days: int | None = None, start_day: int = START_DAY, configs: list | None = None, stop: threading.Event | None = None, device=None, + ds: DecisionSet | None = None, fly=None, boot: dict | None = None, graph=None, save_verdict: bool = True) -> dict: + """The replay; ``ds``/``fly``/``boot`` may be given (tests, or a bootstrap already run).""" + from .strategies import HOLDS_MIN + configs = [(float(a), float(h)) for a, h in (configs or CONFIGS)] + if configs[0][0] != 0.0: + configs = [(0.0, math.inf)] + configs # configuration 0 is always the frozen fly + t0 = time.time() + if ds is None: + prog.update("fly replay: building decision points", 0, 1, force=True) + ds = build(days=days, holds=HOLDS_MIN) + dl = ds.days + if len(dl) <= start_day + 2: + raise RuntimeError(f"the corpus has {len(dl)} days; the replay needs more than {start_day + 2}") + S = dl[start_day] + if fly is None: + fly, boot = fly_selector.bootstrap(ds, S, stop=stop, graph=graph, device=device) + if fly is None: + return {"stopped": True, **(boot or {})} + boot = boot or {} + names = fly.strategies; NS = len(names); holds_min = [int(fly.rules[k]["hold_min"]) for k in names] + lags = [h * 60.0 + LABEL_LAG_S for h in holds_min] + S_epoch = datetime(S.year, S.month, S.day, tzinfo=timezone.utc).timestamp() + uni = selector.in_universe(ds.X, ds.cols) + hist_from = S_epoch - fly_calibrate.WINDOW_DAYS * DAY_S + rows = np.flatnonzero(uni & (ds.ts >= hist_from)) + order = rows[np.argsort(ds.ts[rows], kind="stable")] + ts_o = ds.ts[order]; n = len(order) + live_from = int(np.searchsorted(ts_o, S_epoch, side="left")) # rows before it: history only (calibration seed) + bank = plastic.PlasticBank(fly.net, configs, fly_selector.SCALE, learn=fly.net.learn.cpu().numpy(), read=fly.net.read.cpu().numpy()) + C = bank.C + scores = np.full((C, NS, n), np.nan, np.float32) + lines = np.array([[fly.lines[k] for k in names]] * C, dtype=np.float64); sizings = [[list(fly.sizings[k]) for k in names] for _ in range(C)] + line_log: dict = {} + pending = plastic.PendingTags(fly.net.n_kc, fly.net.k_active) + labels = [torch.tensor(_labels(ds, h), dtype=torch.float32) for h in holds_min] + starts = np.flatnonzero(np.r_[True, ts_o[1:] != ts_o[:-1]]); ends = np.r_[starts[1:], n] + day_done = _day_start(ts_o[live_from]) if live_from < n else None + next_gov = (ts_o[live_from] + HOUR_S) if live_from < n else None + gov_counts = {(c, s): {"shadow": 0, "ic": 0, "drift": 0, "any": 0, "checks": 0} for c in range(C) for s in range(NS)} + learn: dict = {} # per UTC day, per configuration: what the mushroom body actually did + nu_set = False; g = 0; n_updates = 0 + fly.net.eval() + while g < len(starts): + if stop is not None and stop.is_set(): + return {"stopped": True} + g1 = g; rows_in = 0 + while g1 < len(starts) and rows_in < CHUNK: + rows_in += ends[g1] - starts[g1]; g1 += 1 + a, b = starts[g], ends[g1 - 1] + Y, u0, k = fly.parts_all(ds.X[order[a:b]]); trig = fly.triggers(ds.X[order[a:b]], ds.cols) + if not nu_set: + bank.estimate_nu(Y[:, 0], u0, k); nu_set = True + for gi in range(g, g1): + s0, s1 = starts[gi], ends[gi]; now = float(ts_o[s0]); lo, hi = s0 - a, s1 - a + if s0 >= live_from: + tags = pending.pop_due(now, device=bank.dev) + if tags is not None: + r = torch.tensor([float(labels[s_][row]) for row, s_ in tags.keys], dtype=torch.float32) + _learn_add(learn, now, bank.update(tags, r, now)); n_updates += len(tags) + if day_done is not None and _day_start(now) > day_done: + day_done = _day_start(now) + _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log) + if next_gov is not None and now >= next_gov: + next_gov = now + HOUR_S + _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, gov_counts) + for s_ in range(NS): + loc = np.flatnonzero(trig[lo:hi, s_]) + if not len(loc): + continue + li = torch.as_tensor(lo + loc, device=Y.device) + with torch.no_grad(): + sc, _ = bank.predict(Y[li, s_], u0[li], k[li], s=np.full(len(loc), s_)) + scores[:, s_, s0 + loc] = sc.float().cpu().numpy() + if s0 >= live_from: + w = plastic.row_weights(sc, torch.tensor(lines[:, s_], dtype=sc.dtype, device=sc.device)) + keep = np.flatnonzero((w > 0).any(0).cpu().numpy()) # a row no configuration would trade teaches none of them + if len(keep): + kl = loc[keep]; ki = torch.as_tensor(lo + kl, device=Y.device) + pending.push([(int(order[s0 + j]), s_) for j in kl], ts_o[s0 + kl], now + lags[s_], + Y[ki, s_], u0[ki], k[ki], w[:, torch.as_tensor(keep, device=w.device)], s=s_) + if g == 0 or (g1 // 50) != (g // 50): + prog.update("fly replay: learning from the market", int(b), n, day=str(datetime.fromtimestamp(float(ts_o[b - 1]), timezone.utc).date()), + updates=n_updates, pending=len(pending), drift=[round(float(x), 4) for x in bank.drift()], eta_s=(n - b) * (time.time() - t0) / max(b, 1)) + g = g1 + return _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, time.time() - t0, fly, names, holds_min, lines, learn) + + +def _learn_add(learn: dict, now: float, st: dict) -> None: + """Accumulate one update's statistics per UTC day, per configuration. A replay that leaves no record of its own + learning cannot say why learning helped or hurt — which is how a degrading fly went unexplained for two runs.""" + if not st or not st.get("n"): + return + n = int(st["n"]); d = datetime.fromtimestamp(now, timezone.utc).date() + e = learn.setdefault(d, {"n": 0, "delta": None, "abs": None, "step": None, "capped": None, "drift": None}) + e["n"] += n + for key, src, scale in (("delta", "mean_delta", n), ("abs", "mean_abs_delta", n), ("step", "step", 1)): + v = [float(x) * scale for x in st[src]] + e[key] = v if e[key] is None else [a + b for a, b in zip(e[key], v)] + cap = [int(bool(x)) for x in st["capped"]] + e["capped"] = cap if e["capped"] is None else [a + b for a, b in zip(e["capped"], cap)] + e["drift"] = [float(x) for x in st["drift"]] + + +def _learning_series(learn: dict) -> list: + """The per-day series for the verdict: mean delta (how wrong it was), step size, and drift from the bootstrap.""" + out = [] + for d in sorted(learn): + e = learn[d]; n = max(e["n"], 1) + out.append({"day": str(d), "n": e["n"], + "mean_delta": [round(x / n, 6) for x in (e["delta"] or [])], + "mean_abs_delta": [round(x / n, 6) for x in (e["abs"] or [])], + "step": [round(x, 8) for x in (e["step"] or [])], + "capped": e["capped"] or [], "drift": [round(x, 5) for x in (e["drift"] or [])]}) + return out + + +def _window(ts_o: np.ndarray, now: float, lag: float, span_s: float) -> tuple[int, int]: + """Positions of the rows whose labels resolved in (now − span, now].""" + return int(np.searchsorted(ts_o, now - span_s - lag, side="right")), int(np.searchsorted(ts_o, now - lag, side="right")) + + +def _line_at(line_log: dict, key, ts: np.ndarray, default: float) -> np.ndarray: + """The line (configuration, strategy) ``key`` had in force when each row was scored.""" + log_c = line_log.get(key) or [] + if not log_c: + return np.full(len(ts), default) + at = np.array([t for t, _ in log_c]); vals = np.array([v for _, v in log_c]) + i = np.searchsorted(at, ts, side="right") - 1 + return np.where(i >= 0, vals[np.clip(i, 0, None)], default) + + +def _recalibrate(ds, order, ts_o, scores, lines, sizings, now, lags, holds_min, line_log) -> None: + """One line and sizing per strategy, calibrated on the frozen fly's scores and shared by every configuration. + + Until 2026-09-22 each configuration calibrated its own. With the plastic and frozen weights no more than 5 % apart, + the frozen arm still took 4,216 selection-half trades to the chosen arm's 2,609: the arms traded different slices + because of their lines, so "plastic beats frozen" measured line calibration, not plasticity. A shared line leaves + the weights as the only difference between arms, which is the question the replay exists to answer.""" + for s_, (lag, H) in enumerate(zip(lags, holds_min)): + a, b = _window(ts_o, now, lag, fly_calibrate.WINDOW_DAYS * DAY_S) + rows = order[a:b]; y = _labels(ds, H)[rows] + sc = scores[0, s_, a:b]; has = np.isfinite(sc) + cal = fly_calibrate.calibrate(ds.ts[rows[has]], ds.mint[rows[has]], H * 60.0, sc[has], y[has], prev_line=float(lines[0, s_]), prev_sizing=sizings[0][s_]) + for c in range(lines.shape[0]): + key = (c, s_) + if key not in line_log: + line_log[key] = [(-math.inf, float(lines[c, s_]))] + lines[c, s_] = cal.line; sizings[c][s_] = cal.sizing + line_log[key].append((now, float(cal.line))) + + +def _governance(ds, order, ts_o, scores, lines, line_log, bank, now, lags, holds_min, counts) -> None: + for s_, (lag, H) in enumerate(zip(lags, holds_min)): + a3, b3 = _window(ts_o, now, lag, gov.SHADOW_DAYS * DAY_S); a1, b1 = _window(ts_o, now, lag, gov.IC_HOURS * HOUR_S) + r3 = order[a3:b3]; r1 = order[a1:b1]; y = _labels(ds, H) + lf = _line_at(line_log, (0, s_), ts_o[a3:b3], float(lines[0, s_])) + sf = scores[0, s_, a3:b3] + rf = gov.picks_returns(ds.ts[r3], ds.mint[r3], H * 60.0, np.nan_to_num(sf, nan=-np.inf), lf, y[r3]) + drift = bank.drift(s_).tolist() if bank.learn.shape[0] > 1 else bank.drift().tolist() + for c in range(1, lines.shape[0]): + lc = _line_at(line_log, (c, s_), ts_o[a3:b3], float(lines[c, s_])) + sc3 = np.nan_to_num(scores[c, s_, a3:b3], nan=-np.inf) + rp = gov.picks_returns(ds.ts[r3], ds.mint[r3], H * 60.0, sc3, lc, y[r3]) + s1 = scores[c, s_, a1:b1]; has = np.isfinite(s1) + out = gov.check((rp, rf), (s1[has], y[r1][has]), drift[c]) + counts[(c, s_)]["checks"] += 1 + for t in out["triggers"]: + counts[(c, s_)][t] += 1 + counts[(c, s_)]["any"] += bool(out["triggers"]) + + +def _arm(ds, order, ts_o, scores_cs, line_log, key, default_line, mask_pos) -> tuple[np.ndarray, np.ndarray]: + """Full-length (N) scores and per-row lines of (configuration, strategy) ``key`` on ``mask_pos`` (NaN elsewhere).""" + S_full = np.full(len(ds.y), np.nan); L_full = np.full(len(ds.y), np.inf) + pos = np.flatnonzero(mask_pos & np.isfinite(scores_cs)) + S_full[order[pos]] = scores_cs[pos]; L_full[order[pos]] = _line_at(line_log, key, ts_o[pos], default_line) + return S_full, L_full + + +def _book(ds, order, ts_o, scores, line_log, choice: list[int], boot_lines: list[float], holds_min, mask_pos): + """The combined book of the per-strategy choices: per row, among the strategies at/above their line in force, the one + with the largest margin; returns (pick, per-row hold s, per-row return).""" + N = len(ds.y); key = np.full(N, -np.inf); pick = np.zeros(N, bool); hold = np.zeros(N); ret = np.full(N, np.nan, np.float32) + for s_, c in enumerate(choice): + Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], mask_pos) + lab = _labels(ds, holds_min[s_]) + ok = (Sf >= Lf) & np.isfinite(lab) # a row whose hold runs past the corpus has no label: not a trade + m = np.where(ok, Sf - Lf, -np.inf); take = ok & (m > key) + key = np.where(take, m, key); pick |= ok + hold = np.where(take, holds_min[s_] * 60.0, hold); ret = np.where(take, lab, ret) + return pick, hold, ret + + +def _verdict(ds, order, ts_o, live_from, scores, line_log, configs, bank, gov_counts, boot, S, save_verdict, secs, fly, names, holds_min, lines, learn=None) -> dict: + days_r = sorted(set(ds.day[order[live_from:]].tolist())) + sel_days, ev_days = days_r[: len(days_r) // 2], days_r[len(days_r) // 2:] + day_o = ds.day[order] + live_pos = np.arange(len(order)) >= live_from + sel_pos = live_pos & np.isin(day_o, sel_days); ev_pos = live_pos & np.isin(day_o, ev_days) + boot_lines = [fly.lines[k] for k in names] + per_strategy = {}; choice = [] + for s_, name in enumerate(names): + y = _labels(ds, holds_min[s_]); table = [] + for c, (alpha, hl) in enumerate(configs): + Sf, Lf = _arm(ds, order, ts_o, scores[c, s_], line_log, (c, s_), boot_lines[s_], sel_pos) + tr = taken_idx(ds.ts, ds.mint, holds_min[s_] * 60.0, np.flatnonzero(Sf >= Lf)); r = y[tr] + table.append({"config": c, "alpha": alpha, "half_life_days": hl if math.isfinite(hl) else None, "trades": int(len(r)), "total": float(np.nansum(r)), + "mean": float(np.nanmean(r)) if len(r) else None, "governance": gov_counts.get((c, s_))}) + ok = [t for t in table if t["alpha"] > 0 and t["trades"] >= selector.MIN_LINE_TRADES] + best = max(ok, key=lambda t: t["total"])["config"] if ok else 0 + choice.append(best) + per_strategy[name] = {"configs": table, "chosen": table[best] if ok else None, "alpha": table[best]["alpha"], "half_life_days": table[best]["half_life_days"], + "hold_min": holds_min[s_], "plastic": bool(ok)} + + def judge(ch): + pick, hold, ret = _book(ds, order, ts_o, scores, line_log, ch, boot_lines, holds_min, ev_pos) + test = np.zeros(len(ds.y), bool); test[order[ev_pos]] = True + ev = evaluate(ds, np.where(pick, 1.0, np.nan), test & pick, 0.5, "fly", hold_s=hold, returns=ret) + rnd = summarize(random_trades(ds, test, max(1, int(ev["pooled"]["n"] or 1)))) + return ev, rnd + + out = {"S": str(S), "selection_days": [str(sel_days[0]), str(sel_days[-1])] if sel_days else None, "evaluation_days": [str(ev_days[0]), str(ev_days[-1])] if ev_days else None, + "strategies": names, "per_strategy": per_strategy, "configs": per_strategy[names[0]]["configs"], + "bootstrap": {k: boot.get(k) for k in ("lines", "calibration", "diagnostics", "gates_ok", "gate_failures")}, "data": fly_selector.FLY_VERSION, + "secs": secs, "finished_at": datetime.now(timezone.utc).isoformat(), "learning": _learning_series(learn or {})} + ev0, rnd0 = judge([0] * len(names)) + out["frozen"] = {**ev0["pooled"], "random": rnd0} + if not any(v["plastic"] for v in per_strategy.values()): + out.update(passed=False, reason="no strategy's plastic configuration made 100 trades on the selection half") + else: + ev, rnd = judge(choice) + passed, why = selector.deploy_decision(ev["pooled"], rnd) + out.update(passed=bool(passed), reason=why, chosen=per_strategy[names[0]]["chosen"], evaluation=ev["pooled"], random=rnd, per_day=ev["per_day"], + alpha=per_strategy[names[0]]["alpha"], half_life_days=per_strategy[names[0]]["half_life_days"], + plastic_beats_frozen=bool((ev["pooled"]["mean"] or -1) > (ev0["pooled"]["mean"] or -1))) + log.info("fly replay from %s: %s — %s | frozen %s", S, "PASSED" if out.get("passed") else "FAILED", out.get("reason"), out["frozen"]) + log.info("fly replay verdict: %s", {k: out.get(k) for k in ("passed", "reason", "evaluation", "frozen", "random")}) + if save_verdict: + record_event("info" if out.get("passed") else "warning", "fly_replay", "fly replay " + ("passed" if out.get("passed") else "failed"), + {k: out.get(k) for k in ("S", "passed", "reason", "evaluation", "random", "frozen", "plastic_beats_frozen")} | + {"per_strategy": {k: {x: v[x] for x in ("alpha", "half_life_days", "hold_min", "plastic")} for k, v in per_strategy.items()}}) + try: # NaN is valid Python and invalid JSON: never let the store discard a finished replay + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (KEY, json.dumps(prog._finite(out), default=str))) + except Exception: + log.exception("could not store the replay verdict; it is in the log above and the fly cannot be armed until it is stored") + try: # the chosen configuration's learning, where the console already looks for it + c0 = choice[0] if choice else 0 + with transaction() as conn: + for rec in out["learning"]: + d = date.fromisoformat(rec["day"]); pick = lambda k: (rec[k][c0] if len(rec.get(k) or []) > c0 else None) + conn.execute("INSERT INTO fly_updates (hour, n, mean_delta, mean_abs_delta, step, capped, drift, ic, detail) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (hour) DO NOTHING", + (datetime(d.year, d.month, d.day, tzinfo=timezone.utc), rec["n"], pick("mean_delta"), pick("mean_abs_delta"), + pick("step"), pick("capped"), pick("drift"), None, + json.dumps({"source": "replay", "config": c0, "per_config_drift": rec["drift"]}, default=str))) + except Exception: + log.exception("could not store the replay's learning statistics (the verdict is stored)") + prog.update("fly replay: done", 1, 1, force=True, passed=out.get("passed"), reason=out.get("reason")) + return out + + +def verdict() -> dict | None: + """The stored replay verdict for the current fly definitions, or None.""" + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() + v = (r["value"] if isinstance(r["value"], dict) else json.loads(r["value"] or "{}")) if r else None + return v if v and v.get("data") == fly_selector.FLY_VERSION else None + + +def main(days: int | None = None, start_day: int = START_DAY, stop_event: threading.Event | None = None) -> dict: + """The replay; when it passes and no fly may trade yet, the live fly is bootstrapped at once (taught on the whole corpus).""" + prog.set_stop_event(stop_event); prog.clear() + out = run(days=days, start_day=start_day, stop=stop_event) + if out.get("passed") and not (stop_event is not None and stop_event.is_set()): + with transaction() as conn: + have = fly_selector.latest_deployable(conn) + if have is None: + log.info("replay passed: bootstrapping the live fly") + out["live_bootstrap"] = {k: v for k, v in fly_selector.main(days=days, stop_event=stop_event).items() if k in ("snapshot_id", "lines", "gates_ok", "gate_failures", "calibration")} + return out diff --git a/fly_trader/train/fly_selector.py b/fly_trader/train/fly_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..3b0f0a59590f4782287b6c1373ce7b6e8e2717c2 --- /dev/null +++ b/fly_trader/train/fly_selector.py @@ -0,0 +1,662 @@ +"""The fly: the FlyWire central brain, bootstrapped once by imitating the selector, then left to learn from the market +through its mushroom body (``brain/plastic.py``). + +Network (``FlyNet``, flynet-2-mb): the selector's robust-scaled features (``train/scaling.py``) are injected into the +5,460 sensory (afferent) neurons through a learned projection; activity propagates ``K_STEPS`` steps along the real, +signed synapses (leaky tanh rate units, magnitudes initialised from synapse counts, signs and topology fixed, scale.json's +pathway gains applied) and along the mushroom body's KC→MBON synapses, which the connectome build keeps apart from the +sparse graph (the pairs that exist in FlyWire; magnitudes learnable, starting at ``KM_INIT_SCALE`` per synapse so the +MBONs start unsaturated). Kenyon cells keep only their ``KC_ACTIVE`` most active cells at every step (APL-like +winner-take-all; Dasgupta 2017, MacKenzie 2025). Prediction of the net return over the hold (× ``SCALE``) = the +decoder on the normalised descending neurons + a signed linear readout of the normalised MBONs (approach +, avoid −, +the others learned): the path through which plasticity at KC→MBON reaches the output. + +``forward_parts`` gives what the plasticity rule needs — the descending-neuron output, the MBONs' pre-activation at the +last step and the KC code that drove it; ``readout`` completes the prediction with a plastic change ``D`` of the KC→MBON +weights at that last step (``D`` = None is the frozen fly). The descending neurons at the last step are computed from +the previous step, so ``D`` reaches the output only through the MBON readout. + +Bootstrap (``bootstrap``, the one time the selector teaches): the teacher is the selector the live book trades +(``deployed_teacher``); only when none is deployed is a stack refit on the days before ``S − 8`` (one purge day before +the calibration week), which is also what the replay uses, since there the refit is the honest out-of-sample teacher. +The fly imitates it on the tradable minutes of those days (the teacher's top ``DISTIL_TOP`` plus a random +``DISTIL_SAMPLE``; squared error weighted ``TOP_WEIGHT`` × on the teacher's top 1 %; passes stop when a held-out +slice stops improving by more than its own standard error, so the epoch count comes from the data), both +normalisations are recomputed over ``RECAL_ROWS`` training rows and frozen, and the fly's own buy line and sizing bands +are calibrated (``train/fly_calibrate.py``) on its scores for the seven days before ``S`` whose labels were known by +then — never the selector's line. Reports: the MBONs' share of the score variance, MBON saturation, the slope and rank +correlation of the fly's scores against the teacher's. Snapshots: ``data/brain/policies/fly_.pt`` + ``brain_snapshots`` +kind 'fly_selector'. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as Fn + +from .. import config +from ..agent import sizing +from ..brain.connectome import AFFERENT_POPS, EFFERENT_POP, Connectome, SubConnectome +from ..brain.plastic import assign_channels +from ..db.apilog import record_event +from ..db.connection import transaction +from . import fly_calibrate +from . import progress as prog +from . import selector +from .decisions import HOLD_MIN, DecisionSet, build, rank_corr, taken_idx +from .scaling import RobustScaler + +log = logging.getLogger(__name__) + +SCALE = 20.0 # the head predicts the net return × 20 (±5 % ≈ ±1) +K_STEPS, LEAK, HIDDEN = 4, 0.5, 128 +KC_ACTIVE = 0.10 # share of Kenyon cells left active at every step +KM_INIT_SCALE = 0.006 # KC→MBON weight per synapse: ~22 active KC inputs × ~8 synapses ≈ 1, MBONs unsaturated +C_INIT = 0.1 # MBON readout coefficient at start (SCALE units per standard deviation of an MBON) +EPOCHS, BATCH = 8, 512 # EPOCHS only caps the runtime; early stopping below picks the count the data supports +TOP_WEIGHT = 10.0 # loss weight of the teacher's top 1 % of training minutes +DISTIL_TOP, DISTIL_SAMPLE = 0.05, 3_000_000 +RECAL_ROWS = 200_000 +VAL_ROWS, VAL_MIN_ROWS = 200_000, 100_000 # distillation rows held out to judge the fit; a smaller slice is too noisy to stop on +MIN_EPOCHS = 2 # never stop on one epoch: it has nothing to be compared against +LR_GRAPH, LR_HEAD = 1e-4, 1e-3 +PURGE_DAYS, CALIB_DAYS = 1, 7 +DIAG_ROWS = 50_000 +GATES = {"mbon_share_min": 0.10, "mbon_saturated_max": 0.20, "slope_min": 0.6, "slope_max": 1.4} +TEACHER_NOTE = "a stack refit on the fly's training days (no deployed selector to copy)" +# the definitions a fly was trained on (the selector's, as its teacher, + the fly's network and plasticity design) +FLY_VERSION = {**selector.DATA_VERSION, "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1"} + + +def _inv_softplus(x: torch.Tensor) -> torch.Tensor: + return torch.log(torch.expm1(x.clamp(min=1e-6))) + + +class FlyNet(nn.Module): + """Scorer on the connectome: features → afferent neurons → K steps along the synapses and the KC→MBON block → + normalised descending neurons → decoder, + signed readout of the normalised MBONs → predicted net return (× ``SCALE``).""" + + def __init__(self, graph, obs_dim: int, k_steps: int = K_STEPS, leak: float = LEAK, hidden: int = HIDDEN, kc_active: float = KC_ACTIVE, device=None, + n_strategies: int = 1): + super().__init__() + dev = torch.device(device) if device else graph.device + self.N, self.k_steps, self.leak, self.obs_dim, self.hidden, self.kc_active = graph.N, k_steps, leak, obs_dim, hidden, kc_active + self.register_buffer("pre", graph.indices[1].to(dev)); self.register_buffer("post", graph.indices[0].to(dev)) + vals = graph.values_raw.float(); self.register_buffer("sign", torch.sign(vals).to(dev)) + s = float(getattr(graph, "s", 0.0) or 0.0) or 0.99 / float(getattr(graph, "spectral_radius_raw", None) or 2788.0) + self.theta = nn.Parameter(_inv_softplus(vals.abs() * s).to(dev)) # softplus^-1(|w|) + self.gain = nn.Parameter(torch.ones(self.N, device=dev)); self.bias = nn.Parameter(torch.zeros(self.N, device=dev)) + aff = [torch.arange(*graph.pop_ranges[p]) for p in AFFERENT_POPS if p in graph.pop_ranges] + self.register_buffer("aff_rows", torch.cat(aff).to(dev)); self.register_buffer("eff_rows", torch.arange(*graph.pop_ranges[EFFERENT_POP]).to(dev)) + # the mushroom body: KC→MBON pairs of the connectome (MBON columns: approach, avoid, other) + (self.kc0, self.kc1), self.mb0, self.mb1 = graph.pop_ranges["KC"], graph.pop_ranges["MBON_APP"][0], graph.pop_ranges["MBON_OTHER"][1] + self.n_kc, self.n_mbon = self.kc1 - self.kc0, self.mb1 - self.mb0 + self.k_active = max(1, int(round(kc_active * self.n_kc))) + M = graph.M_KM.bool(); ki, mj = torch.nonzero(M, as_tuple=True) + self.register_buffer("mask", M.to(dev)); self.register_buffer("km_kc", ki.to(dev)); self.register_buffer("km_mbon", mj.to(dev)) + self.theta_km = nn.Parameter(_inv_softplus(graph.W_KM0.float()[ki, mj] * KM_INIT_SCALE).to(dev)) + n_app = graph.pop_ranges["MBON_APP"][1] - graph.pop_ranges["MBON_APP"][0]; n_av = graph.pop_ranges["MBON_AV"][1] - graph.pop_ranges["MBON_AV"][0] + c_sign = torch.zeros(self.n_mbon); c_sign[:n_app] = 1.0; c_sign[n_app:n_app + n_av] = -1.0 + self.register_buffer("c_sign", c_sign.to(dev)) + self.rho = nn.Parameter(torch.full((self.n_mbon,), float(_inv_softplus(torch.tensor(C_INIT))), device=dev)) + self.c_free = nn.Parameter(torch.zeros(self.n_mbon, device=dev)) + n_aff, n_eff = len(self.aff_rows), len(self.eff_rows) + self.w_in = nn.Parameter(torch.randn(n_aff, obs_dim, device=dev) / obs_dim ** 0.5) # every feature reaches every sensory neuron + self.eff_norm = nn.BatchNorm1d(n_eff).to(dev) + self.mb_norm = nn.BatchNorm1d(self.n_mbon, affine=False).to(dev) + self.dec = nn.Sequential(nn.Linear(n_eff, hidden), nn.GELU(), nn.Linear(hidden, hidden), nn.GELU()).to(dev) + # one head per strategy over the shared decoder; strategy s reads its own dopamine channel (mushroom-body + # compartments, brain/plastic.assign_channels) and the compartments no strategy claims + self.n_strategies = int(n_strategies) + self.heads = nn.ModuleList([nn.Linear(hidden, 1) for _ in range(self.n_strategies)]).to(dev) + comp = getattr(graph, "mbon_compartment", None) + learn, read = assign_channels(comp if comp is not None else None, graph.W_KM0.float().sum(0).numpy(), c_sign.numpy(), self.n_strategies) + self.register_buffer("learn", torch.as_tensor(learn).to(dev)); self.register_buffer("read", torch.as_tensor(read).to(dev)) + self.dev = dev + + @property + def head(self) -> nn.Linear: + """The first strategy's head (the only one with one strategy).""" + return self.heads[0] + + @property + def c(self) -> torch.Tensor: + """MBON readout coefficients: approach +, avoid −, the others free.""" + return torch.where(self.c_sign != 0, self.c_sign * Fn.softplus(self.rho), self.c_free) + + def w_km(self) -> torch.Tensor: + """The KC→MBON weights as a dense [n_KC, n_MBON] matrix (0 where the connectome has no synapse).""" + W = torch.zeros(self.n_kc, self.n_mbon, device=self.dev) + W[self.km_kc, self.km_mbon] = Fn.softplus(self.theta_km) + return W + + def _kwta(self, h: torch.Tensor) -> torch.Tensor: + kc = torch.relu(h[self.kc0:self.kc1]) + thr = kc.topk(self.k_active, dim=0).values[-1:] + kc = torch.where(kc >= thr, kc, torch.zeros_like(kc)) + return torch.cat([h[:self.kc0], kc, h[self.kc1:]], 0) + + def _propagate(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """(activity after the last step [N, B], KC code that drove it [n_KC, B], MBON pre-activation of it [n_MBON, B]).""" + B = x.shape[0] + I_in = torch.zeros(self.N, B, device=self.dev).index_copy(0, self.aff_rows, self.w_in @ x.T) + w = self.sign * Fn.softplus(self.theta); wkm = Fn.softplus(self.theta_km) + mb_rows, kc_rows = self.mb0 + self.km_mbon, self.kc0 + self.km_kc + h = torch.zeros(self.N, B, device=self.dev); k = u = None + for step in range(self.k_steps): + msg = torch.zeros_like(h).index_add(0, self.post, h.index_select(0, self.pre) * w[:, None]) + msg = msg.index_add(0, mb_rows, h.index_select(0, kc_rows) * wkm[:, None]) + pre_act = self.leak * h + self.gain[:, None] * msg + self.bias[:, None] + I_in + if step == self.k_steps - 1: + k, u = h[self.kc0:self.kc1], pre_act[self.mb0:self.mb1] + h = self._kwta(torch.tanh(pre_act)) + return h, k, u + + def forward_parts_all(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """(every strategy head's decoder output [B, S] × SCALE, MBON pre-activation at the last step [B, n_MBON], KC code [B, n_KC]).""" + h, k, u = self._propagate(x) + z = self.dec(self.eff_norm(h.index_select(0, self.eff_rows).T)) + return torch.cat([hd(z) for hd in self.heads], dim=1), u.T, k.T + + def forward_parts(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """(the first head's decoder output [B] × SCALE, MBON pre-activation at the last step [B, n_MBON], KC code [B, n_KC]).""" + y, u, k = self.forward_parts_all(x) + return y[:, 0], u, k + + def readout(self, y_dn: torch.Tensor, u0: torch.Tensor, k: torch.Tensor, D: torch.Tensor | None = None, s: int = 0) -> torch.Tensor: + """Strategy ``s``'s prediction × SCALE from the parts; ``D`` [n_KC, n_MBON] is a plastic change of the KC→MBON + weights at the last step; the strategy reads its own channel's MBONs and the unclaimed ones.""" + u = u0 if D is None else u0 + self.gain[self.mb0:self.mb1] * (k @ (D * self.mask)) + return y_dn + (self.mb_norm(torch.tanh(u)) * self.read[s].float()) @ self.c + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.readout(*self.forward_parts(x)) + + def forward_all(self, x: torch.Tensor) -> torch.Tensor: + """Every strategy's prediction × SCALE [B, S].""" + y, u, k = self.forward_parts_all(x) + z = self.mb_norm(torch.tanh(u)) + return y + (z[:, None, :] * self.read.float()[None]) @ self.c + + def mbon_stats(self) -> tuple[torch.Tensor, torch.Tensor]: + """The frozen MBON normalisation (mean, standard deviation) the readout uses in eval mode.""" + return self.mb_norm.running_mean, torch.sqrt(self.mb_norm.running_var + self.mb_norm.eps) + + def param_groups(self, lr_graph: float, lr_head: float) -> list[dict]: + graph = [self.theta, self.gain, self.bias, self.theta_km]; ids = {id(p) for p in graph} + return [{"params": graph, "lr": lr_graph}, {"params": [p for p in self.parameters() if id(p) not in ids], "lr": lr_head}] + + +class FlyModel: + """A trained fly as a trading model: one value per strategy of the selector's stack (``rules``: its fitted trigger, + high window and hold), each on its own dopamine channel, with the fly's own line and sizing per strategy. The + single-strategy interface of ``selector.SelectorModel`` (score, threshold, sizing, universe) is the first strategy's.""" + + def __init__(self, net: FlyNet, scaler: RobustScaler, cols: list[str], horizon_min: int, threshold: float | None = None, sizing: list | None = None, + rules: dict | None = None, lines: dict | None = None, sizings: dict | None = None, combine: str = "score"): + self.net, self.scaler, self.cols, self.horizon_min = net, scaler, list(cols), horizon_min + self.rules = dict(rules or {"ev": {"thr": {}, "high": None, "hold_min": int(horizon_min)}}) + self.strategies = list(self.rules) + base = threshold if threshold is not None else selector.MIN_EV + self.lines = {k: float((lines or {}).get(k, base)) for k in self.strategies} + self.sizings = {k: list((sizings or {}).get(k, sizing or [])) for k in self.strategies} + self.combine = combine + + @property + def threshold(self) -> float: + return self.lines[self.strategies[0]] + + @threshold.setter + def threshold(self, v: float) -> None: + self.lines[self.strategies[0]] = float(v) + + @property + def sizing(self) -> list: + return self.sizings[self.strategies[0]] + + @sizing.setter + def sizing(self, v: list) -> None: + self.sizings[self.strategies[0]] = list(v or []) + + def hold_s(self, s: int) -> float: + return float(self.rules[self.strategies[s]]["hold_min"]) * 60.0 + + def _x(self, X: np.ndarray) -> torch.Tensor: + return torch.tensor(self.scaler.transform(X), device=self.net.dev) + + @torch.no_grad() + def score_all(self, X: np.ndarray, batch: int = 2048) -> np.ndarray: + """[B, S] every strategy's predicted net return over its hold (the frozen fly).""" + self.net.eval(); out = np.empty((len(X), len(self.strategies)), np.float64) + for i in range(0, len(X), batch): + out[i:i + batch] = (self.net.forward_all(self._x(X[i:i + batch])) / SCALE).float().cpu().numpy() + return out + + def score(self, X: np.ndarray, batch: int = 2048) -> np.ndarray: + """The first strategy's predicted net return over its hold (the frozen fly).""" + return self.score_all(X, batch)[:, 0] + + @torch.no_grad() + def parts(self, X: np.ndarray) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """``FlyNet.forward_parts`` on raw feature rows (one batch; eval mode).""" + self.net.eval() + return self.net.forward_parts(self._x(X)) + + @torch.no_grad() + def parts_all(self, X: np.ndarray) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """``FlyNet.forward_parts_all``: every head's decoder output [B, S], MBON pre-activation, KC code.""" + self.net.eval() + return self.net.forward_parts_all(self._x(X)) + + def triggers(self, X: np.ndarray, cols: list[str]) -> np.ndarray: + """[B, S] where each strategy's fitted trigger (the selector's) fires — its candidates.""" + from . import strategies + X = np.atleast_2d(X); out = np.zeros((len(X), len(self.strategies)), bool) + for j, name in enumerate(self.strategies): + r = self.rules[name] + out[:, j] = (strategies.base_mask(name, X, cols, r.get("high")) & strategies.trigger_mask(name, r.get("thr") or {}, X, cols)) if name in strategies.STRATEGIES else True + return out + + def universe(self, X: np.ndarray) -> np.ndarray: + return selector.in_universe(X, self.cols) + + +def _device() -> torch.device: + return torch.device("mps" if torch.backends.mps.is_available() else "cpu") + + +def _graph(): + return SubConnectome(Connectome.load(), exclude=("VISUAL",)) + + +def distil_rows(idx: np.ndarray, tgt: np.ndarray, rng: np.random.Generator, top: float = DISTIL_TOP, sample: int = DISTIL_SAMPLE) -> np.ndarray: + """Positions (into ``idx``) the fly trains on: every row in the teacher's top ``top`` plus a random ``sample`` of the rest.""" + if len(idx) <= sample: + return np.arange(len(idx)) + hi = np.flatnonzero(tgt >= np.quantile(tgt, 1.0 - top)); rest = np.setdiff1d(np.arange(len(idx)), hi, assume_unique=True) + return np.sort(np.r_[hi, rng.choice(rest, min(sample, len(rest)), replace=False)]) + + +@torch.no_grad() +def recalibrate(net: FlyNet, X: np.ndarray, scaler: RobustScaler, batch: int = BATCH) -> None: + """Both normalisations re-estimated as plain averages over ``X`` (not the last few training batches), then frozen.""" + norms = (net.eff_norm, net.mb_norm) + for bn in norms: + bn.reset_running_stats(); bn.momentum = None + net.train() + for i in range(0, len(X), batch): + net(torch.tensor(scaler.transform(X[i:i + batch]), device=net.dev)) + net.eval() + + +class StackTeacher: + """The selector's strategy stack (train/strategies.final_models) as the fly's teacher: per strategy, on its candidates, + the net return it predicts where the stack would trade, and one typical score-spread (MAD) below that strategy's + line where a filter (win filter, gates, dump veto) blocks it — so the fly learns the final decision, not the raw score.""" + + def __init__(self, models: dict, scaler: RobustScaler, cols: list[str]): + self.models, self.scaler, self.cols = models, scaler, list(cols) + self.strategies = list(models.get("strategies") or {}) + self.rules = {k: {"thr": dict(v["thr"]), "high": v["high"], "hold_min": int(v["hold_min"])} for k, v in models["strategies"].items()} + self.lines = {k: float(v["line"]) for k, v in models["strategies"].items()} + self.threshold = self.lines[self.strategies[0]] if self.strategies else selector.MIN_EV + self.combine = models.get("combine", "score"); self.eps: dict = {} + + def targets(self, X: np.ndarray, ts: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """(targets [B, S] with NaN off each strategy's candidates, allowed [B, S]).""" + from . import strategies + per = strategies.decide_all(self.models, X, self.cols, ts); T = np.full((len(X), len(self.strategies)), np.nan); A = np.zeros_like(T, dtype=bool) + for j, k in enumerate(self.strategies): + d = per[k]; sc = d["score"]; trig = d["trig"] + if k not in self.eps: + al = sc[d["allow"] & np.isfinite(sc)] + self.eps[k] = float(np.median(np.abs(al - np.median(al)))) if len(al) else 0.0 + T[trig, j] = np.where(d["allow"][trig], sc[trig], np.minimum(sc[trig], self.lines[k] - self.eps[k])); A[:, j] = d["allow"] + return T, A + + +def _teacher_targets(teacher, ds: DecisionSet, idx: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """[n, S] targets (NaN: not a candidate of that strategy) and loss weights for the rows ``idx``.""" + if isinstance(teacher, StackTeacher): + T = np.full((len(idx), len(teacher.strategies)), np.nan, np.float32); W = np.ones_like(T) + for i in range(0, len(idx), 200_000): + t, a = teacher.targets(ds.X[idx[i:i + 200_000]], ds.ts[idx[i:i + 200_000]]) + T[i:i + 200_000] = t; W[i:i + 200_000] = np.where(a, TOP_WEIGHT, 1.0) + return np.clip(T, -1.0, 1.0), W + tgt = np.empty(len(idx), np.float32) + for i in range(0, len(idx), 500_000): + tgt[i:i + 500_000] = teacher.score(ds.X[idx[i:i + 500_000]]) + tgt = np.clip(tgt, -1.0, 1.0) + return tgt[:, None], np.where(tgt >= np.quantile(tgt, 0.99), TOP_WEIGHT, 1.0).astype(np.float32)[:, None] + + +@torch.no_grad() +def _val_mse(net: FlyNet, scaler: RobustScaler, ds: DecisionSet, idx: np.ndarray, T: np.ndarray, W: np.ndarray, val: np.ndarray, + dev: torch.device, batch: int) -> tuple[float, float]: + """Weighted squared error on distillation rows held out of training, and the standard error of that mean. An epoch + that buys less than the noise in this estimate has stopped teaching the fly anything, which is where training ends: + the count comes from the data instead of being chosen. Until 2026-09-21 the bootstrap ran exactly one epoch, leaving + a per-row error of ~3.9 % net return against a 4.6 % buy line — the ordering near the line was mostly noise.""" + net.eval(); parts = [] + for i in range(0, len(val), batch): + sl = val[i:i + batch] + x = torch.tensor(scaler.transform(ds.X[idx[sl]]), device=dev) + t = torch.tensor(np.nan_to_num(T[sl]) * SCALE, device=dev); m = torch.tensor(np.isfinite(T[sl]), device=dev).float() + w = torch.tensor(W[sl], device=dev) * m + parts.append(((w * (net.forward_all(x) - t) ** 2).sum(1) / w.sum(1).clamp(min=1e-9)).cpu()) + net.train() + e = torch.cat(parts).double() if parts else torch.zeros(1, dtype=torch.float64) + return float(e.mean()), float(e.std() / max(len(e) ** 0.5, 1.0)) + + +def train_fly(ds: DecisionSet, rows: np.ndarray, teacher, epochs: int = EPOCHS, batch: int = BATCH, stop: threading.Event | None = None, + seed: int = 0, graph=None, device=None) -> tuple[FlyModel | None, dict]: + """Distil ``teacher`` (a StackTeacher, or anything with ``score`` and ``scaler``) into a FlyNet over the training rows + ``rows`` (mask): one head per strategy, each learning its strategy's targets on that strategy's candidates.""" + torch.manual_seed(seed); rng = np.random.default_rng(seed); dev = torch.device(device) if device else _device() + n_s = len(teacher.strategies) if isinstance(teacher, StackTeacher) else 1 + net = FlyNet(graph if graph is not None else _graph(), obs_dim=len(ds.cols), device=dev, n_strategies=n_s) + idx = np.flatnonzero(rows); T, W = _teacher_targets(teacher, ds, idx) + keep = np.isfinite(T).any(1); idx, T, W = idx[keep], T[keep], W[keep] + if not len(idx): + return None, {"stopped": False, "reason": "no teacher candidates"} + use = distil_rows(idx, np.nanmax(np.where(np.isfinite(T), T, -np.inf), axis=1), rng) + val = np.empty(0, use.dtype) + if epochs > 1 and len(use) // 10 >= VAL_MIN_ROWS: # a smaller slice cannot tell a real improvement from noise + n_v = min(VAL_ROWS, len(use) // 10); p0 = rng.permutation(len(use)); val, use = np.sort(use[p0[:n_v]]), np.sort(use[p0[n_v:]]) + with torch.no_grad(): + for j, hd in enumerate(net.heads): + col = T[:, j]; hd.bias.fill_(float(np.nanmean(col)) * SCALE if np.isfinite(col).any() else 0.0); hd.weight.mul_(0.1) + opt = torch.optim.Adam(net.param_groups(LR_GRAPH, LR_HEAD)); n_b = max(1, len(use) // batch); total = max(1, epochs * n_b); hist = []; t0 = time.time() + best = best_v = prev_v = early = None + for ep in range(epochs): + perm = use[rng.permutation(len(use))]; tot = 0.0; net.train() + for b in range(n_b): + if stop is not None and stop.is_set(): + return None, {"epochs": hist, "stopped": True} + sel = perm[b * batch:(b + 1) * batch] + x = torch.tensor(teacher.scaler.transform(ds.X[idx[sel]]), device=dev) + t = torch.tensor(np.nan_to_num(T[sel]) * SCALE, device=dev); m = torch.tensor(np.isfinite(T[sel]), device=dev).float() + w = torch.tensor(W[sel], device=dev) * m + loss = (w * (net.forward_all(x) - t) ** 2).sum() / w.sum().clamp(min=1e-9) + opt.zero_grad(); loss.backward(); torch.nn.utils.clip_grad_norm_(net.parameters(), 5.0); opt.step(); tot += loss.item() + if b % 200 == 0: + done = ep * n_b + b + 1; el = time.time() - t0 + prog.update("fly: learning from the selector", done, total, epoch=ep, loss=tot / (b + 1), rows_per_s=done * batch / max(el, 1e-9), + eta_s=(total - done) * el / done) + hist.append({"epoch": ep, "mse": tot / n_b, "secs": time.time() - t0, "rows": min(len(use), n_b * batch)}) + log.info("fly epoch %d: weighted mse %.4f over %d rows (%.0fs)", ep, tot / n_b, min(len(use), n_b * batch), time.time() - t0) + if not len(val): + continue + vm, vse = _val_mse(net, teacher.scaler, ds, idx, T, W, val, dev, batch) + hist[-1]["val_mse"], hist[-1]["val_se"] = vm, vse + log.info("fly epoch %d: held-out weighted mse %.4f +/- %.4f over %d rows", ep, vm, vse, len(val)) + if best_v is None or vm < best_v: + best_v, best = vm, {k: v.detach().clone() for k, v in net.state_dict().items()} + gain = None if prev_v is None else prev_v - vm + stop_now = ep + 1 >= MIN_EPOCHS and gain is not None and gain <= vse + prev_v = vm + if stop_now: + early = f"epoch {ep} moved the held-out error by {gain:+.4f}, inside its standard error {vse:.4f}" + log.info("fly: stopping early - %s", early) + break + if best is not None: + net.load_state_dict(best) # the epoch that generalised best, not merely the last one + rec = idx[rng.choice(len(idx), min(RECAL_ROWS, len(idx)), replace=False)] + recalibrate(net, ds.X[np.sort(rec)], teacher.scaler, batch) + if isinstance(teacher, StackTeacher): + fly = FlyModel(net, teacher.scaler, ds.cols, int(teacher.rules[teacher.strategies[0]]["hold_min"]), rules=teacher.rules, combine=teacher.combine) + else: + fly = FlyModel(net, teacher.scaler, ds.cols, int(ds.horizon_s // 60)) + return fly, {"epochs": hist, "stopped": False, "rows": int(len(use)), "training_rows": int(len(idx)), "top_weight": TOP_WEIGHT, + "strategies": fly.strategies, "epochs_run": len(hist), "epoch_cap": int(epochs), "val_rows": int(len(val)), + "val_mse": best_v, "early_stop": early} + + +@torch.no_grad() +def diagnose(fly: FlyModel, X: np.ndarray, teacher_scores: np.ndarray | None = None, batch: int = 2048) -> dict: + """How much of the score runs through the mushroom body, how saturated the MBONs are, how sparse the KC code is, + and (given the teacher's scores) how the fly's scores line up with them.""" + net = fly.net; net.eval(); mb, tot, sat, act = [], [], [], [] + for i in range(0, len(X), batch): + y_dn, u0, k = fly.parts(X[i:i + batch]) + m = net.mb_norm(torch.tanh(u0)) @ net.c + mb.append(m.cpu()); tot.append((y_dn + m).cpu()); sat.append(u0.abs().cpu()); act.append((k > 0).float().mean(1).cpu()) + mb, tot, sat, act = torch.cat(mb).double(), torch.cat(tot).double(), torch.cat(sat), torch.cat(act) + out = {"mbon_share": float(mb.var() / tot.var()) if tot.var() > 0 else 0.0, "mbon_saturated": float((sat.median(0).values > 2.0).float().mean()), + "kc_active": float(act.mean()), "rows": int(len(X))} + if teacher_scores is not None: + s = (tot / SCALE).numpy() + out["slope"] = float(np.polyfit(np.asarray(teacher_scores, dtype=np.float64), s, 1)[0]) if np.std(teacher_scores) > 0 else None + out["rank_corr"] = rank_corr(s, teacher_scores) + return out + + +def gate_check(diag: dict) -> tuple[bool, list[str]]: + why = [] + if diag["mbon_share"] < GATES["mbon_share_min"]: + why.append(f"MBONs carry {diag['mbon_share']:.1%} of the score (< {GATES['mbon_share_min']:.0%}): plasticity would move little") + if diag["mbon_saturated"] > GATES["mbon_saturated_max"]: + why.append(f"{diag['mbon_saturated']:.0%} of MBONs saturated (> {GATES['mbon_saturated_max']:.0%})") + sl = diag.get("slope") + if sl is not None and not (GATES["slope_min"] <= sl <= GATES["slope_max"]): + why.append(f"score slope {sl:.2f} against the teacher outside {GATES['slope_min']}–{GATES['slope_max']}") + return not why, why + + +def _stack_teacher(ds: DecisionSet, train: np.ndarray, stop=None): + """The selector's strategy stack fitted on the training days only (an honest teacher for the calibration week).""" + from . import strategies + sub = ds.subset(train) + stack = strategies.fit_stack(sub, stop, holdout_days_n=0) # already only days before S - 8; the replay is its out-of-sample proof + if not stack.fits: + return None, stack + return StackTeacher(strategies.final_models(sub, stack), RobustScaler.fit(sub.X, seed=7), ds.cols), stack + + +def deployed_teacher(ds: DecisionSet, train: np.ndarray) -> tuple[StackTeacher | None, str, int | None]: + """The selector the live book actually trades, wrapped as the fly's teacher, with the reason and the snapshot id. + + Until 2026-09-21 the bootstrap always refit its own stack on the days before ``S − 8`` and distilled that instead. + Component selection is unstable enough that the refit was a different trading system: the deployed stack (#60, fit + through 2026-09-18) traded ``ev`` at line 0.046 over 120 minutes, while the fly's own teacher (through 2026-09-11) + traded ``capitulation`` at 0.010 over 240 — five of eight components flipped by one extra week of corpus. The fly + was never a copy of the selector it was being judged against. None (with the reason) falls back to that refit. + + The deployed selector is fit on the whole corpus, the fly's calibration week included, so that week is not + out-of-sample for the teacher; the fly's own line is still calibrated on realised labels, never on the teacher's.""" + from ..agent.selector_session import pinned_snapshot + sid = pinned_snapshot() + if sid is None: + with transaction() as conn: + r = selector.latest_current(conn) + sid = int(r["id"]) if r else None + if sid is None: + return None, TEACHER_NOTE, None + m = selector.load_snapshot(sid) + if m is None: + return None, f"selector #{sid} could not be loaded; {TEACHER_NOTE}", None + st = getattr(m, "stack", None) or {} + if not st.get("strategies"): + return None, f"selector #{sid} has no strategy stack; {TEACHER_NOTE}", None + missing = sorted({c for v in st["strategies"].values() for c in v["cols"]} - set(ds.cols)) + if missing: + return None, f"selector #{sid} wants columns the corpus lacks ({', '.join(missing[:4])}); {TEACHER_NOTE}", None + names = ", ".join(f"{k} (line {v['line']:.4f}, {v['hold_min']} min)" for k, v in st["strategies"].items()) + return StackTeacher(st, RobustScaler.fit(ds.X[np.flatnonzero(train)], seed=7), list(ds.cols)), f"deployed selector #{sid}: {names}", sid + + +def fly_decide(fly: FlyModel, X: np.ndarray, cols: list[str], values: np.ndarray | None = None) -> dict: + """The fly's own per-row decision: among the strategies whose trigger fires and whose value clears the fly's line, + the one with the larger sizing-band Kelly fraction or margin (the teacher's combination rule).""" + V = fly.score_all(X) if values is None else values; trig = fly.triggers(X, cols); n = len(X) + strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n); thr = np.full(n, np.inf); tables = [[] for _ in range(n)] + key = np.full(n, -np.inf) + for j, name in enumerate(fly.strategies): + v = V[:, j]; line = fly.lines[name]; ok = trig[:, j] & (v >= line) + kv = (np.array([((sizing.band_for(fly.sizings[name], m) or {}).get("kelly") or 0.0) for m in v - line]) + 1e-9 * v + if fly.combine == "kelly" and fly.sizings[name] else v - line) + take = ok & (kv > key) + key = np.where(take, kv, key) + for i in np.flatnonzero(take): + strat[i] = name; score[i] = v[i]; hold[i] = fly.rules[name]["hold_min"] * 60.0; thr[i] = line; tables[i] = fly.sizings[name] + return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables} + + +def bootstrap(ds: DecisionSet, S: date, epochs: int = EPOCHS, stop: threading.Event | None = None, seed: int = 0, graph=None, + device=None, teacher=None, teacher_note: str | None = None) -> tuple[FlyModel | None, dict]: + """The one time the selector teaches: a fly ready to trade from day ``S`` on, with its own line and sizing per strategy. + ``teacher``: the live path passes the deployed selector (``deployed_teacher``); left None, the stack is refit on the + days before ``S − 8`` — which is what the replay wants, since there the refit is the honest out-of-sample teacher (a + set without the stack's inputs or per-hold labels — synthetic tests — gets the single-model selector). + ``teacher_note`` records in the snapshot which selector taught this fly.""" + from .decisions import LEGACY_COLS + train_end = S - timedelta(days=CALIB_DAYS + PURGE_DAYS) + train = ds.day < train_end; uni = selector.in_universe(ds.X, ds.cols) + if not train.any(): + raise RuntimeError(f"no training days before {train_end}") + stack_info = None + if teacher is None: + prog.update("fly: fitting its teacher (the selector's strategy stack, training days only)", 0, 1, force=True) + if ds.fwd_h and set(LEGACY_COLS) <= set(ds.cols): + teacher, stack = _stack_teacher(ds, train, stop) + stack_info = {"components": stack.components, "deployable": stack.deployable, "reason": stack.reason} + if teacher is None: + return None, {"S": str(S), "gates_ok": False, "gate_failures": ["the teacher stack has no strategy on the training days"], "teacher_stack": stack_info} + else: + teacher = selector.fit(ds, train, seed=7) + fly, fit_info = train_fly(ds, train & uni, teacher, epochs=epochs, stop=stop, seed=seed, graph=graph, device=device) + if fly is None: + return None, {"stopped": True, **fit_info} + s_epoch = datetime(S.year, S.month, S.day, tzinfo=timezone.utc).timestamp() + week = (ds.day >= S - timedelta(days=CALIB_DAYS)) & (ds.day < S) & uni + prog.update("fly: calibrating its own buy lines", 0, 1, force=True) + wi = np.flatnonzero(week); V = fly.score_all(ds.X[wi]) if len(wi) else np.zeros((0, len(fly.strategies))); trig = fly.triggers(ds.X[wi], ds.cols) + per = {} + for j, name in enumerate(fly.strategies): + H = fly.rules[name]["hold_min"]; y = ds.fwd_h.get(H, ds.fwd_pess) + known = trig[:, j] & (ds.ts[wi] + H * 60.0 + 60.0 <= s_epoch) # labels known at S + ci = wi[known] + cal = fly_calibrate.calibrate(ds.ts[ci], ds.mint[ci], H * 60.0, V[known, j], y[ci]) + fly.lines[name], fly.sizings[name] = cal.line, cal.sizing + per[name] = {"line": cal.line, "trades": cal.trades, "mean": cal.mean, "total": cal.total, "hold_min": H} + d = fly_decide(fly, ds.X[wi], ds.cols, V) if len(wi) else {"strategy": np.array([], dtype=object), "hold_s": np.array([])} + picked = np.array([x is not None for x in d["strategy"]], bool) + ret = np.array([ds.fwd_h.get(fly.rules[x]["hold_min"], ds.fwd_pess)[i] if x is not None else np.nan for i, x in zip(wi, d["strategy"])]) + known = picked & (ds.ts[wi] + d["hold_s"] + 60.0 <= s_epoch) + tr = taken_idx(ds.ts[wi], ds.mint[wi], d["hold_s"], np.flatnonzero(known)); r = ret[tr] + comb = {"trades": int(len(r)), "mean": float(np.mean(r)) if len(r) else None, "total": float(np.sum(r)) if len(r) else 0.0, "per_strategy": per} + ci = np.flatnonzero(week & (ds.ts + ds.horizon_s + 60.0 <= s_epoch)) + di = ci if len(ci) <= DIAG_ROWS else np.sort(np.random.default_rng(seed).choice(ci, DIAG_ROWS, replace=False)) + t_sc = teacher.score(ds.X[di]) if hasattr(teacher, "score") else (teacher.targets(ds.X[di], ds.ts[di])[0][:, 0] if len(di) else None) + if t_sc is not None and len(di): + ok_t = np.isfinite(t_sc); di, t_sc = di[ok_t], t_sc[ok_t] + diag = diagnose(fly, ds.X[di], t_sc) if len(di) else {"mbon_share": 0.0, "mbon_saturated": 1.0, "kc_active": 0.0, "rows": 0} + ok, why = gate_check(diag) + info = {"S": str(S), "train_through": str(train_end - timedelta(days=1)), "calibration_days": [str(S - timedelta(days=CALIB_DAYS)), str(S - timedelta(days=1))], + "line": fly.threshold, "sizing": fly.sizing, "lines": fly.lines, "calibration": comb, "strategies": fly.strategies, "rules": fly.rules, + "diagnostics": diag, "gates_ok": ok, "gate_failures": why, "fit": fit_info, "teacher": teacher_note or TEACHER_NOTE, "teacher_line": getattr(teacher, "threshold", None), + "teacher_stack": stack_info} + log.info("fly bootstrap for %s: strategies %s, lines %s | calibration %d trades, mean %s | MBON share %.1f%% saturated %.0f%% | gates %s", + S, fly.strategies, {k: round(v, 4) for k, v in fly.lines.items()}, comb["trades"], f"{comb['mean'] * 100:+.2f}%" if comb["mean"] is not None else "-", + diag["mbon_share"] * 100, diag["mbon_saturated"] * 100, "ok" if ok else "; ".join(why)) + return fly, info + + +def is_current(meta: dict | None) -> bool: + return bool(meta) and meta.get("data") == FLY_VERSION + + +def save(fly: FlyModel, metrics: dict, run_id: str | None = None) -> tuple[Path, int]: + root = config.BRAIN_DIR / "policies"; root.mkdir(parents=True, exist_ok=True) + path = root / f"fly_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.pt" + n = fly.net + torch.save({"state_dict": n.state_dict(), "scaler": fly.scaler.state(), "cols": fly.cols, "threshold": fly.threshold, "sizing": fly.sizing, + "horizon_min": fly.horizon_min, "rules": fly.rules, "lines": fly.lines, "sizings": fly.sizings, "combine": fly.combine, + "config": {"k_steps": n.k_steps, "leak": n.leak, "hidden": n.hidden, "obs_dim": n.obs_dim, "kc_active": n.kc_active, "scale": SCALE, + "n_strategies": n.n_strategies}, "metrics": metrics}, path) + sha = hashlib.sha256(path.read_bytes()).hexdigest() + with transaction() as conn: + row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,'fly_selector',%s) RETURNING id", + (run_id, str(path), sha, json.dumps({**metrics, "data": FLY_VERSION}, default=str))).fetchone() + return path, int(row["id"]) + + +def load(path: str | Path, graph=None, device=None) -> FlyModel: + d = torch.load(path, map_location="cpu", weights_only=False); c = d["config"] + net = FlyNet(graph if graph is not None else _graph(), obs_dim=c["obs_dim"], k_steps=c["k_steps"], leak=c["leak"], hidden=c["hidden"], + kc_active=c["kc_active"], device=device or _device(), n_strategies=c.get("n_strategies", 1)) + net.load_state_dict(d["state_dict"]); net.eval() + for bn in (net.eff_norm, net.mb_norm): + bn.momentum = None + return FlyModel(net, RobustScaler.from_state(d["scaler"]), d["cols"], d["horizon_min"], d["threshold"], d.get("sizing"), rules=d.get("rules"), + lines=d.get("lines"), sizings=d.get("sizings"), combine=d.get("combine", "score")) + + +def latest_current(conn) -> dict | None: + """The newest fly bootstrap trained on the current definitions (``FLY_VERSION``) whose file exists, or None.""" + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = 'fly_selector' ORDER BY id DESC").fetchall(): + try: + meta = json.loads(r["note"] or "{}") + except ValueError: + continue + if is_current(meta) and Path(r["path"]).exists(): + return {**dict(r), "meta": meta} + return None + + +def deployable(meta: dict | None) -> tuple[bool, str]: + """A bootstrap may trade: current definitions, healthy network (``GATES``) and its own line made money over at least + 100 trades on its calibration week.""" + if not is_current(meta): + return False, "trained on other definitions" + if not meta.get("gates_ok"): + return False, "; ".join(meta.get("gate_failures") or ["network gates failed"]) + cal = meta.get("calibration") or {} + if (cal.get("trades") or 0) < selector.MIN_LINE_TRADES: + return False, f"too few calibration trades ({cal.get('trades') or 0})" + if (cal.get("mean") or 0) <= 0: + return False, f"its calibration week lost money ({(cal.get('mean') or 0) * 100:+.2f}% per trade)" + return True, f"its line made {cal['mean'] * 100:+.2f}% per trade over {cal['trades']} calibration trades" + + +def latest_deployable(conn) -> dict | None: + """The newest bootstrap that may trade (``deployable``), or None.""" + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = 'fly_selector' ORDER BY id DESC").fetchall(): + try: + meta = json.loads(r["note"] or "{}") + except ValueError: + continue + if deployable(meta)[0] and Path(r["path"]).exists(): + return {**dict(r), "meta": meta} + return None + + +def main(days: int | None = None, epochs: int | None = None, stop_event: threading.Event | None = None) -> dict: + """Bootstrap a fly to trade from tomorrow on: taught by the selector the live book trades, on every corpus day but + the last eight, calibrated on the last seven. ``epochs``: a cap; None leaves the count to the held-out slice.""" + from ..ops.reset import reset_training_stats + reset_training_stats("fly_selector", reason="fly bootstrap") + prog.set_stop_event(stop_event); prog.clear() + prog.update("fly: building decision points", 0, 1, force=True) + from .strategies import HOLDS_MIN + ds = build(days=days, horizon_min=HOLD_MIN, holds=HOLDS_MIN); S = ds.days[-1] + timedelta(days=1) + teacher, note, taught_by = deployed_teacher(ds, ds.day < S - timedelta(days=CALIB_DAYS + PURGE_DAYS)) + log.info("fly bootstrap teacher: %s", note) + fly, info = bootstrap(ds, S, epochs=EPOCHS if epochs is None else epochs, stop=stop_event, teacher=teacher, teacher_note=note) + if fly is None: + return {"stopped": True} + info["teacher_snapshot"] = taught_by + record_event("info", "fly_selector", "fly bootstrap", {k: info.get(k) for k in ("S", "lines", "calibration", "diagnostics", "gates_ok", "gate_failures", "teacher")}) + path, sid = save(fly, info) + prog.update("fly: done", 1, 1, force=True, snapshot_id=sid, line=info["line"], gates_ok=info["gates_ok"], diagnostics=info["diagnostics"]) + return {**info, "snapshot_id": sid, "path": str(path)} diff --git a/fly_trader/train/mature.py b/fly_trader/train/mature.py new file mode 100644 index 0000000000000000000000000000000000000000..50824b6683f60171980ad24ee63a350750a5186e --- /dev/null +++ b/fly_trader/train/mature.py @@ -0,0 +1,424 @@ +"""The mature-token universe: every PumpSwap token that traded on a day, at any age, as 1-minute rich candles and +as feature rows — the population the live bot actually meets on Jupiter's lists, as opposed to the first 12 hours +after graduation covered by ``replay_assemble``. + +Stage 1 (``aggregate_day``): all ``pump-amm`` trade legs of pump.fun-origin mints (suffix ``pump``, SOL-quoted) of a UTC day, +except legs in pools created directly rather than by a pump.fun migration (``pump_pools``: their owner can pull the unburned +liquidity, and the withdrawal is not in the trade data, so the price just stops after a pump), +with causal filters only (the live stream applies the same ones): quote reserve 0.001–100,000 SOL, a leg more than 50× away +from the median of the mint's previous 200 legs that day is dropped, and each (mint, minute) keeps the pool with the most +legs → per (mint, minute): open/high/low/close, +SOL volume split by side, buy/sell counts, distinct traders in the minute, and the pool's quote reserve at the +minute's last trade → ``data/corpus/mature/.parquet`` (registry ``mature_days``). + +Stage 2 (``build_day``): feature rows for day D using D−1 as lookback. Each minute feeds ``TokenState`` as two +synthetic trades (the minute's buy volume and sell volume at the close), so imbalance and volume features are +exact and price features match the live engine; trade counts come from the candle counts (``logn_*``), +signer counts are replaced by per-minute-distinct trader sums (``logsigners_*``, an upper bound, flagged in the +mask as absent), Jupiter stats stay masked. Age and price-vs-graduation come from ``corpus_meta`` when the +graduation is inside the archive (``corpus_meta.graduated_at``, the source live uses too); older tokens carry NaN age. +Output: ``data/corpus/features_mature//part.parquet``. Both outputs carry a version in their Parquet metadata +(``AGG_VERSION``, ``market.features.FEATURE_VERSION``); ``loop_once`` rebuilds any file from another version and moves +the old one to ``data/corpus/_mature_stale/``. Feature parts also record how many of their mints had a known graduation +(``fly_known``); a part is rebuilt once ``corpus_meta`` dates more of them (older days are backfilled after newer ones). +""" +from __future__ import annotations + +import fcntl +import logging +import math +import os +import shutil +import threading +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import duckdb +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config, logging_setup +from ..db.connection import transaction +from ..market.exit_cost import PUMP_SUPPLY +from ..market.features import FEATURE_VERSION, FIDX, TokenMeta, TokenState +from .corpus_features import SCHEMA as FEAT_SCHEMA, PRE_COLS, _epoch_s, _row +from .corpus_meta import blocked_pool_ids +from .flow import FLOW_COLS, MARKET_COLS, N_SKILL, SKILL_COLS, UNKNOWN, FlowWindow, MarketWindow + +log = logging.getLogger(__name__) +MATURE_DIR = config.CORPUS_DIR / "mature" +MATURE_FEAT_DIR = config.CORPUS_DIR / "features_mature" +STALE_DIR = config.CORPUS_DIR / "_mature_stale" +AGG_VERSION = 4 # 2: causal price band and per-minute dominant pool; 3: directly created (custom) PumpSwap pools excluded; 4: wallet flow columns (train/flow.py) +KNOWN_REBUILD_FRAC, KNOWN_REBUILD_MIN = 0.01, 25 # a part is rebuilt once corpus_meta dates this many more of its mints +MASKED = ["logsigners_15m", "logsigners_1h", "hawkes", "log_since_last", "vpin_15m"] +EXTRA = ["traders_15m", "traders_1h", "n_trades_1m"] + FLOW_COLS + SKILL_COLS + MARKET_COLS +SCHEMA = FEAT_SCHEMA +for _c in EXTRA: + SCHEMA = SCHEMA.append(pa.field(_c, pa.float32())) + + +def part_version(path) -> int: + md = pq.read_schema(path).metadata or {} + try: + return int(md.get(b"fly_version", b"0")) + except ValueError: + return 0 + + +def agg_skill(path) -> str: + """The wallet-skill version an aggregate's skill buckets came from ('none': no table for its day).""" + return (pq.read_schema(path).metadata or {}).get(b"fly_skill", b"none").decode() + + +def _skill_version() -> str: + from .wallet_skill import skill_version + return skill_version() + + +def agg_stale(path) -> bool: + """An aggregate to rebuild: another aggregation version, or skill buckets from another skill version while its day now + has a table of the version in force (days without a table keep 'none').""" + p = Path(path) + if part_version(p) != AGG_VERSION: + return True + want = _skill_version() + return want != "none" and (SKILL_DIR / f"{p.stem}.parquet").exists() and agg_skill(p) != want + + +def part_agg(path) -> int: + """The aggregation version a feature part was built from (parts from before the key: 0 = unknown, rebuilt).""" + md = pq.read_schema(path).metadata or {} + try: + return int(md.get(b"fly_agg", b"0")) + except ValueError: + return 0 + + +def part_current(path) -> bool: + p = Path(path) + return p.exists() and part_version(p) == FEATURE_VERSION and part_agg(p) == AGG_VERSION + + +def build_complete() -> tuple[bool, str]: + """Is the training data complete: every aggregate from the current version, no ingested day left unaggregated, and + a current feature part for every non-empty aggregate day that has a previous day? (reason when not)""" + aggs = sorted(MATURE_DIR.glob("*.parquet")) + if not aggs: + return False, "no aggregated days yet" + old = [f for f in aggs if agg_stale(f)] + if old: + return False, f"{len(old)} day(s) still to re-aggregate" + pending = days_ready() + if pending: + return False, f"{len(pending)} ingested day(s) not aggregated yet" + no_pid = days_missing_pool_id() + if no_pid: + return False, f"{len(no_pid)} day(s) await re-downloaded pool ids (fly-trader refetch-pool-ids; e.g. {no_pid[0]})" + missing = [f.stem for f in aggs[1:] if pq.read_metadata(f).num_rows and not part_current(MATURE_FEAT_DIR / f.stem / "part.parquet")] + if missing: + return False, f"{len(missing)} day(s) of features still to build (e.g. {missing[0]})" + return True, f"{len(aggs)} days ready" + + +def part_known(path) -> int: + """Mints of a feature part whose graduation time was known when it was built (parts from before the key: from age_h).""" + md = pq.read_schema(path).metadata or {} + if b"fly_known" in md: + return int(md[b"fly_known"]) + t = pq.read_table(path, columns=["mint", "age_h"]).to_pandas() + return int(t.loc[t["age_h"].notna(), "mint"].nunique()) + + +def write_part(table: pa.Table, path, version: int, extra: dict[str, str] | None = None) -> None: + """Atomic write with the version (and any ``extra`` keys) in the Parquet metadata.""" + path = Path(path); path.parent.mkdir(parents=True, exist_ok=True) + md = {**(table.schema.metadata or {}), b"fly_version": str(version).encode(), **{k.encode(): str(v).encode() for k, v in (extra or {}).items()}} + table = table.replace_schema_metadata(md) + tmp = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") + pq.write_table(table, tmp, compression="zstd"); tmp.replace(path) + + +def _retire(path, tag: str) -> None: + """Move a stale output aside (data is never deleted).""" + path = Path(path) + if path.exists(): + STALE_DIR.mkdir(parents=True, exist_ok=True) + path.replace(STALE_DIR / f"{tag}_v{part_version(path)}_{int(time.time())}.parquet") + + +def _archive(path, tag: str) -> None: + """Keep a copy of an output about to be replaced (data is never deleted); unlike ``_retire`` the original stays in place, + so a concurrent reader (training) never sees the day missing.""" + path = Path(path) + if path.exists(): + STALE_DIR.mkdir(parents=True, exist_ok=True) + dst = STALE_DIR / f"{tag}_v{part_version(path)}_{int(time.time())}.parquet" + try: + os.link(path, dst) # the atomic replace gives the part a new inode; the link keeps the old one + except OSError: + shutil.copy2(path, dst) + + +def _hour_files(d: date) -> list[str]: + day = config.REPLAY_DIR / d.isoformat() + return [str(day / f"{h:02d}_trades.parquet") for h in range(24) if (day / f"{h:02d}_trades.parquet").exists()] + + +def _lacks_pool_id(files: list[str]) -> list[str]: + return [f for f in files if "pool_id" not in pq.read_schema(f).names] + + +def days_missing_pool_id() -> list[str]: + """Days with an hour file written before the parser kept ``pool_id``: aggregating them would skip the blocked-pool + filter and the dominant-pool choice the live stream applies, so they wait for the re-download.""" + out = [] + for day in sorted(p for p in config.REPLAY_DIR.glob("*") if p.is_dir()): + try: + d = date.fromisoformat(day.name) + except ValueError: + continue + if _lacks_pool_id(_hour_files(d)): + out.append(day.name) + return out + + +def days_ready() -> list[date]: + """Days with all 24 replay hours ingested and no mature aggregate yet, newest first.""" + with transaction() as conn: + done = {r["hour"].astimezone(timezone.utc) for r in conn.execute("SELECT hour FROM replay_hours WHERE status IN ('done','missing')").fetchall()} + built = {r["day"] for r in conn.execute("SELECT day FROM mature_days").fetchall()} + out = [] + for d in sorted({h.date() for h in done}, reverse=True): + if d in built: + continue + if all(datetime(d.year, d.month, d.day, h, tzinfo=timezone.utc) in done for h in range(24)): + out.append(d) + return out + + +def aggregate_table(files: list[str], skill, blocked: list[str] | None = None, insiders: dict | None = None) -> pa.Table: + """A day's minute aggregates from its hour files, with ``skill`` (wallet → decile table, or None) as the day's skill + table — ``aggregate_day``'s SQL, also used to rebuild the skill inputs under a candidate table (train/wallet_skill.fit). + ``blocked`` (pool ids) and ``insiders`` (mint -> wallets) are read from Postgres when not given; a caller that aggregates + many days hands them in once, rather than opening two connections per day.""" + con = duckdb.connect() + cols = [c[0] for c in con.execute("SELECT * FROM read_parquet(?, union_by_name = true) LIMIT 0", [files]).description] + quote_filter = "AND (quote_mint IS NULL OR quote_mint = 'So11111111111111111111111111111111111111112')" if "quote_mint" in cols else "" + pool_sel = "pool_id" if "pool_id" in cols else "NULL AS pool_id" + if blocked is None: + with transaction() as conn: + blocked = blocked_pool_ids(conn) + con.register("blocked", pa.table({"pool_id": pa.array(blocked, pa.string())})) + pool_filter = "AND (pool_id IS NULL OR pool_id NOT IN (SELECT pool_id FROM blocked))" if "pool_id" in cols else "" + # causal filters only (nothing later in the day decides which earlier trades exist); the live stream applies the same: + # no directly created pool, reserve band, a leg within 50x of the median of the mint's previous 200 legs, and per (mint, minute) the pool with the most legs + con.execute(f"""CREATE TEMP TABLE raw AS SELECT mint, ts, slot, trader, side, sol, price, quote_in_pool, {pool_sel} + FROM read_parquet(?, union_by_name = true) + WHERE pool = 'pump-amm' AND price > 0 AND mint LIKE '%pump' AND quote_in_pool BETWEEN 0.001 AND 100000 {quote_filter} {pool_filter}""", [files]) + con.execute("""CREATE TEMP TABLE banded AS SELECT * EXCLUDE (pref) FROM ( + SELECT *, median(price) OVER (PARTITION BY mint ORDER BY ts, slot ROWS BETWEEN 200 PRECEDING AND 1 PRECEDING) AS pref FROM raw) + WHERE pref IS NULL OR price BETWEEN pref / 50 AND pref * 50""") + con.execute("""CREATE TEMP TABLE dom AS SELECT mint, mb, arg_max(pool_id, n) AS pid FROM ( + SELECT mint, time_bucket(INTERVAL 1 MINUTE, ts) AS mb, pool_id, count(*) AS n FROM banded GROUP BY ALL) GROUP BY mint, mb""") + con.execute("""CREATE TEMP TABLE clean AS SELECT b.* FROM banded b JOIN dom d ON d.mint = b.mint AND d.mb = time_bucket(INTERVAL 1 MINUTE, b.ts) + WHERE b.pool_id IS NULL OR d.pid IS NULL OR b.pool_id = d.pid""") + con.execute("""CREATE TEMP TABLE cand AS + SELECT mint, time_bucket(INTERVAL 1 MINUTE, ts) AS ts, + first(price ORDER BY ts, slot) AS open, max(price) AS high, min(price) AS low, last(price ORDER BY ts, slot) AS close, + sum(CASE WHEN side = 1 THEN sol ELSE 0 END) AS buy_sol, sum(CASE WHEN side = -1 THEN sol ELSE 0 END) AS sell_sol, + count(*) FILTER (WHERE side = 1) AS n_buys, count(*) FILTER (WHERE side = -1) AS n_sells, + count(DISTINCT trader) AS n_traders, last(quote_in_pool ORDER BY ts, slot) AS resq_sol + FROM clean GROUP BY mint, time_bucket(INTERVAL 1 MINUTE, ts)""") + # wallet flow (train/flow.py, the live stream's minute_wallet_cols): per (mint, minute, trader) bought and sold SOL + con.execute("""CREATE TEMP TABLE w AS SELECT mint, time_bucket(INTERVAL 1 MINUTE, ts) AS mb, trader, + sum(CASE WHEN side = 1 THEN sol ELSE 0 END) AS b, sum(CASE WHEN side = -1 THEN sol ELSE 0 END) AS s + FROM clean WHERE trader IS NOT NULL GROUP BY ALL""") + mints = [r[0] for r in con.execute("SELECT DISTINCT mint FROM w").fetchall()] + if insiders is None: + with transaction() as conn: + rows = conn.execute("SELECT mint, wallet FROM token_insiders WHERE mint = ANY(%s)", (mints,)).fetchall() if mints else [] + ins = [(r["mint"], r["wallet"]) for r in rows] + else: + ins = [(m, w) for m in mints for w in insiders.get(m, ())] + con.register("ins", pa.table({"mint": pa.array([m for m, _ in ins], pa.string()), "wallet": pa.array([w for _, w in ins], pa.string())})) + con.register("skill", skill if skill is not None else pa.table({"wallet": pa.array([], pa.string()), "bucket": pa.array([], pa.int8())})) + sk_expr = ("[" + ", ".join(f"sum(CASE WHEN coalesce(k.bucket, {UNKNOWN}) = {q} THEN w.b ELSE 0 END)" for q in range(N_SKILL)) + "]") if skill is not None else "NULL::DOUBLE[]" + con.execute(f"""CREATE TEMP TABLE wc AS SELECT w.mint, w.mb AS ts, count(*) FILTER (WHERE w.b > 0) AS n_buyers, + sum(CASE WHEN w.b > 0 AND w.s > 0 THEN w.b + w.s ELSE 0 END) AS wash_sol, + sum(CASE WHEN w.b > 0 AND w.s > 0 THEN w.b ELSE 0 END) AS wash_buy_sol, max(w.s) AS top_sell_sol, + sum(CASE WHEN i.wallet IS NOT NULL THEN w.s ELSE 0 END) AS insider_sell_sol, {sk_expr} AS skill_buy + FROM w LEFT JOIN ins i ON i.mint = w.mint AND i.wallet = w.trader LEFT JOIN skill k ON k.wallet = w.trader + GROUP BY w.mint, w.mb""") + tab = con.execute("""SELECT c.*, coalesce(wc.n_buyers, 0) AS n_buyers, coalesce(wc.wash_sol, 0) AS wash_sol, coalesce(wc.wash_buy_sol, 0) AS wash_buy_sol, + coalesce(wc.top_sell_sol, 0) AS top_sell_sol, coalesce(wc.insider_sell_sol, 0) AS insider_sell_sol, wc.skill_buy + FROM cand c LEFT JOIN wc USING (mint, ts) ORDER BY c.mint, c.ts""").fetch_arrow_table() + con.close() + return tab + + +def aggregate_day(d: date) -> int: + t0 = time.time(); files = _hour_files(d) + if not files: + return 0 + if _lacks_pool_id(files): # exact parity with the live stream needs every leg's pool (refetch-pool-ids) + log.warning("mature %s: %d hour file(s) lack pool_id; not aggregated until they are downloaded again", d, len(_lacks_pool_id(files))) + return 0 + MATURE_DIR.mkdir(parents=True, exist_ok=True) + skill = skill_table_for(d) + tab = aggregate_table(files, skill) + write_part(tab, MATURE_DIR / f"{d.isoformat()}.parquet", AGG_VERSION, {"fly_skill": _skill_version() if skill is not None else "none"}) + n_mints = len(pa.compute.unique(tab["mint"])) + with transaction() as conn: + conn.execute("INSERT INTO mature_days (day, mints, rows, took_s) VALUES (%s,%s,%s,%s) ON CONFLICT (day) DO UPDATE SET mints = EXCLUDED.mints, rows = EXCLUDED.rows, took_s = EXCLUDED.took_s, built_at = now()", + (d, n_mints, tab.num_rows, time.time() - t0)) + log.info("mature %s: %d mints, %d minute rows in %.0fs", d, n_mints, tab.num_rows, time.time() - t0) + return tab.num_rows + + +SKILL_DIR = config.CORPUS_DIR / "wallet_skill" + + +def skill_table_for(d: date): + """The wallet-skill table in force on day D (``train/wallet_skill.py``: wallet → decile, from earlier days only), + as a pyarrow table, or None when the day has none (skill buckets are then NULL, as live without a table).""" + f = SKILL_DIR / f"{d.isoformat()}.parquet" + return pq.read_table(f, columns=["wallet", "bucket"]) if f.exists() else None + + +def _graduations() -> dict: + with transaction() as conn: + return {r["mint"]: r["graduated_at"] for r in conn.execute("SELECT mint, graduated_at FROM corpus_meta WHERE graduated_at IS NOT NULL").fetchall()} + + +def _supplies() -> dict: + """mint → token supply from the create event (mayhem tokens: 2 billion); tokens not in corpus_meta use PUMP_SUPPLY.""" + with transaction() as conn: + return {r["mint"]: float(r["supply"]) for r in conn.execute("SELECT mint, supply FROM corpus_meta WHERE supply > 0").fetchall()} + + +def build_day(d: date, lookback_days: int = 1, grads: dict | None = None, supplies: dict | None = None) -> int: + """Feature rows for every minute of day D, warmed up on the previous day's candles. ``grads``: mint → graduated_at + (default: read from ``corpus_meta``); the part records how many of its mints had one (metadata ``fly_known``).""" + t0 = time.time() + frames = [] + for k in range(lookback_days, -1, -1): + f = MATURE_DIR / f"{(d - timedelta(days=k)).isoformat()}.parquet" + if f.exists(): + frames.append(pq.read_table(f).to_pandas()) + if not frames: + return 0 + cd = pd.concat(frames, ignore_index=True).sort_values(["mint", "ts"]) + day_start = datetime(d.year, d.month, d.day, tzinfo=timezone.utc).timestamp(); day_end = day_start + 86400 + grads = _graduations() if grads is None else grads + supplies = _supplies() if supplies is None else supplies + mkt = _market_windows(cd) + out: list[dict] = []; n_mints = 0; n_known = 0 + for mint, x in cd.groupby("mint", sort=False): + ts_s = _epoch_s(x["ts"]) + if not (ts_s >= day_start).any(): + continue + n_mints += 1 + g = grads[mint].timestamp() if grads.get(mint) else None + n_known += g is not None + st = TokenState(mint); tm = TokenMeta(mint=mint, program_label="Pump.fun Amm", graduated_at=g, supply=supplies.get(mint) or PUMP_SUPPLY) + pre = {k: float("nan") for k in PRE_COLS} + o, h, l, c = x["open"].to_numpy(), x["high"].to_numpy(), x["low"].to_numpy(), x["close"].to_numpy() + bs, ss, nb, ns, nt, rq = x["buy_sol"].to_numpy(), x["sell_sol"].to_numpy(), x["n_buys"].to_numpy(), x["n_sells"].to_numpy(), x["n_traders"].to_numpy(), x["resq_sol"].to_numpy() + hist_ts, hist_n, hist_tr = [], [], [] + fw = FlowWindow(); wcols = [x[c].to_numpy() if c in x else np.zeros(len(x)) for c in ("n_buyers", "wash_sol", "wash_buy_sol", "top_sell_sol", "insider_sell_sol")] + skb = x["skill_buy"].to_numpy() if "skill_buy" in x else np.full(len(x), None, dtype=object) + for i in range(len(ts_s)): + t_end = float(ts_s[i]) + 60.0; price = float(c[i]); resq = float(rq[i]) if np.isfinite(rq[i]) and rq[i] > 0 else None + if bs[i] > 0: + st.append(t_end - 2e-3, price, float(bs[i]), True, None, resq) + if ss[i] > 0 or bs[i] <= 0: + st.append(t_end - 1e-3, price, float(ss[i]), False, None, resq) + hist_ts.append(t_end); hist_n.append(float(nb[i] + ns[i])); hist_tr.append(float(nt[i])) + fw.append(t_end, bs[i], ss[i], wcols[0][i], wcols[1][i], wcols[2][i], wcols[3][i], wcols[4][i], None if skb[i] is None else list(skb[i])) + if t_end <= day_start: + continue + f, mask = st.features(t_end, tm) + ht = np.asarray(hist_ts); hn = np.asarray(hist_n); htr = np.asarray(hist_tr) + for k, w in (("1m", 60), ("5m", 300), ("15m", 900), ("1h", 3600)): + sel = ht > t_end - w + f[FIDX[f"logn_{k}"]] = math.log1p(float(hn[sel].sum())) + for name in MASKED: + f[FIDX[name]] = 0.0; mask &= ~(1 << FIDX[name]) + if g is None: + f[FIDX["log_age_h"]] = 0.0; mask &= ~(1 << FIDX["log_age_h"]) + row = _row(mint, float(ts_s[i]), g if g is not None else float(ts_s[i]) - 1e9, float(o[i]), float(h[i]), float(l[i]), price, float(bs[i] + ss[i]), resq, False, f, mask, pre) + row["traders_15m"] = float(htr[ht > t_end - 900].sum()); row["traders_1h"] = float(htr[ht > t_end - 3600].sum()); row["n_trades_1m"] = float(nb[i] + ns[i]) + row.update(fw.features(t_end)); row["mkt_vol_1h"] = mkt.get(t_end, 0.0) + if g is None: + row["age_h"] = float("nan"); row["t_rel_min"] = -1; row["phase"] = "amm" + out.append(row) + if not out: + return 0 + write_part(pa.Table.from_pylist(out, schema=SCHEMA), MATURE_FEAT_DIR / d.isoformat() / "part.parquet", FEATURE_VERSION, {"fly_known": n_known, "fly_agg": AGG_VERSION}) + log.info("mature features %s: %d mints (%d with graduation), %d rows in %.0fs", d, n_mints, n_known, len(out), time.time() - t0) + return len(out) + + +def _market_windows(cd: pd.DataFrame) -> dict: + """minute end → ``mkt_vol_1h`` over every mint of the frame (the live engine's MarketWindow over its minute rows).""" + tot = (cd["buy_sol"].fillna(0.0) + cd["sell_sol"].fillna(0.0)).groupby(_epoch_s(cd["ts"])).sum().sort_index() + mw = MarketWindow(); out = {} + for t, v in tot.items(): + t_end = float(t) + 60.0; mw.add(t_end, float(v)); out[t_end] = mw.value(t_end) + return out + + +def _knows_more(part: Path, grads: dict) -> bool: + """corpus_meta now dates enough more of the part's mints than when it was built (≥ KNOWN_REBUILD_FRAC of them, at least + KNOWN_REBUILD_MIN): old tokens keep trading for months, so any lower bar rebuilds every part on each backfilled day.""" + mints = pa.compute.unique(pq.read_table(part, columns=["mint"])["mint"]).to_pylist() + grew = sum(1 for m in mints if m in grads) - part_known(part) + return grew >= max(KNOWN_REBUILD_MIN, KNOWN_REBUILD_FRAC * len(mints)) + + +def loop_once() -> int: + """One build round. A file lock serialises rounds across processes (the console's replay worker and the CLI).""" + MATURE_DIR.mkdir(parents=True, exist_ok=True) + with open(MATURE_DIR / ".build.lock", "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + return _loop_once() + + +def _loop_once() -> int: + n = 0 + from . import wallet_skill + n += wallet_skill.daily() # wallet days and the tables in force first: a new table re-aggregates its day below + # aggregates from another version are rebuilt; their day's and the next day's features depend on them + for f in sorted(MATURE_DIR.glob("*.parquet"), reverse=True): + if agg_stale(f) and _hour_files(date.fromisoformat(f.stem)): + d = date.fromisoformat(f.stem) + _retire(f, f"agg_{d}") + for dd in (d, d + timedelta(days=1)): + _retire(MATURE_FEAT_DIR / dd.isoformat() / "part.parquet", f"feat_{dd}") + with transaction() as conn: + conn.execute("DELETE FROM mature_days WHERE day = %s", (d,)) + for d in days_ready(): + aggregate_day(d); n += 1 + grads = _graduations() # corpus_meta.rebuild reads the day's events before every round, so its graduations are known + for f in sorted(MATURE_DIR.glob("*.parquet"), reverse=True): + d = date.fromisoformat(f.stem) + part = MATURE_FEAT_DIR / d.isoformat() / "part.parquet" + # a current part is rebuilt when graduations assembled since (the backfill runs newest-first) date more of its mints + if part_current(part) and not _knows_more(part, grads): + continue + if not (MATURE_DIR / f"{(d - timedelta(days=1)).isoformat()}.parquet").exists(): + continue # the previous day is the lookback + _archive(part, f"feat_{d}") # the old part stays readable until the new one replaces it atomically + if not build_day(d, grads=grads) and part.exists(): + part.unlink() # nothing to write: the (archived) old part must not linger + n += 1 + return n + + +def main() -> None: + logging_setup.setup("replay") + print(f"processed {loop_once()} day steps") diff --git a/fly_trader/train/pipeline.py b/fly_trader/train/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..f7d872ef0d713cf4dad0a5486a1306307f37f3d2 --- /dev/null +++ b/fly_trader/train/pipeline.py @@ -0,0 +1,126 @@ +"""The training pipeline — the ``train`` worker. Every ``INTERVAL_DAYS`` days, or at once when the operator asks: + +1. The selector retrains on the whole corpus (``train/selector.main``): every day after the warm-up is backtested by a + model that never saw it, then the model is fit on every day and saved. It is put to work only if its backtest made + money after costs and beat random picks (``deployable``); the trading engine switches to it without a restart. +2. The fly is bootstrapped (``train/fly_selector.main``: the selector teaches it once) only when no fly exists for the + current definitions or the operator (or an automatic re-bootstrap) asks for one. A weekly run never replaces a fly: + after its bootstrap the fly learns from the market through its mushroom body (``brain/plastic.py``). + +A run starts only when the feature build is complete (every day aggregated and built with the current definitions); +otherwise it waits and checks again every hour. State for the console: ``ui_settings['training_pipeline']``. +""" +from __future__ import annotations + +import json +import logging +import threading +from datetime import datetime, timedelta, timezone + +from ..db.apilog import record_event +from ..db.connection import transaction +from ..logging_setup import setup + +log = logging.getLogger(__name__) +INTERVAL_DAYS = 7 +RETRY_S = 3600 +KEY = "training_pipeline" + + +class NotReady(RuntimeError): + pass + + +def state() -> dict: + with transaction() as conn: + r = conn.execute("SELECT value FROM ui_settings WHERE key = %s", (KEY,)).fetchone() + v = r["value"] if r else {} + return v if isinstance(v, dict) else json.loads(v or "{}") + + +def _save(**kv) -> dict: + v = {**state(), **kv} + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (KEY, json.dumps(v, default=str))) + return v + + +def request_run(fly: bool = False, reason: str | None = None) -> None: + """Run as soon as the worker sees it (the console's Retrain now); ``fly`` also bootstraps a new fly.""" + kv = {"run_requested": True, "retry_after": None} + if fly: + kv.update(fly_requested=True, fly_reason=reason or "requested") + _save(**kv) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def due(st: dict, now: datetime | None = None, handed_over: bool = False) -> bool: + """Every ``INTERVAL_DAYS`` or on request; once the fly holds the selector's seat (agent/handover.py), on request only.""" + now = now or _now() + if st.get("retry_after") and now < datetime.fromisoformat(st["retry_after"]): + return False + if handed_over: + return bool(st.get("run_requested")) + last = st.get("last_run_at") + return bool(st.get("run_requested")) or last is None or now >= datetime.fromisoformat(last) + timedelta(days=INTERVAL_DAYS) + + +def _handed_over() -> bool: + from ..agent import handover + with transaction() as conn: + return handover.state(conn) is not None + + +def _current_fly() -> dict | None: + from . import fly_selector + with transaction() as conn: + return fly_selector.latest_current(conn) + + +def run(stop_event: threading.Event | None = None) -> dict: + from . import fly_selector, mature, selector + ok, why = mature.build_complete() + if not ok: + raise NotReady(why) + st = state() + _save(stage="training the selector", started_at=_now().isoformat(), run_requested=False, retry_after=None, last_error=None, waiting=None) + wf = selector.main(stop_event=stop_event) + if stop_event is not None and stop_event.is_set(): + return _save(stage="stopped") + _save(selector_snapshot=wf.get("snapshot_id"), selector_deployable=wf.get("deployable")) + fly_snapshot = st.get("fly_snapshot") + if st.get("fly_requested") or _current_fly() is None: + _save(stage="bootstrapping the fly (the selector teaches it once)") + boot = fly_selector.main(stop_event=stop_event) + if stop_event is not None and stop_event.is_set(): + return _save(stage="stopped") + fly_snapshot = boot.get("snapshot_id") + now = _now() + out = _save(stage="done", last_run_at=now.isoformat(), next_run_at=(now + timedelta(days=INTERVAL_DAYS)).isoformat(), fly_snapshot=fly_snapshot, + fly_requested=False, fly_reason=None) + record_event("info", "pipeline", "training pipeline finished", {k: out.get(k) for k in ("selector_snapshot", "selector_deployable", "fly_snapshot")}) + return out + + +def main(stop_event: threading.Event | None = None) -> None: + setup("train") + record_event("info", "pipeline", "training pipeline worker started", {"interval_days": INTERVAL_DAYS}) + while not (stop_event is not None and stop_event.is_set()): + if due(state(), handed_over=_handed_over()): + try: + run(stop_event) + except NotReady as e: + log.info("training waits for data: %s", e) + _save(stage="waiting for data", waiting=str(e), retry_after=(_now() + timedelta(seconds=RETRY_S)).isoformat()) + except Exception as e: + log.exception("training pipeline failed") + record_event("error", "pipeline", f"training pipeline failed: {type(e).__name__}: {e}") + _save(stage="failed", last_error=f"{type(e).__name__}: {e}"[:300], retry_after=(_now() + timedelta(seconds=RETRY_S)).isoformat()) + if stop_event is not None: + stop_event.wait(30.0) + else: + threading.Event().wait(30.0) diff --git a/fly_trader/train/progress.py b/fly_trader/train/progress.py new file mode 100644 index 0000000000000000000000000000000000000000..9890c3299ef6db4eb27962b09f793e6281ed4aec --- /dev/null +++ b/fly_trader/train/progress.py @@ -0,0 +1,65 @@ +"""Real-time training status for the console: one JSON row in ui_settings['training_status'], updated +by the trainer every few seconds (stage, step/total, ETA, latest metrics), plus a cooperative stop event.""" +from __future__ import annotations + +import json +import threading +import time +from datetime import datetime, timezone + +from ..db.connection import transaction + +_state: dict = {} +_lock = threading.Lock() +_last_write = 0.0 +STOP: threading.Event | None = None + + +def _finite(x): + """JSON (and Postgres jsonb) have no inf/nan: map them to null recursively.""" + if isinstance(x, float) and (x != x or x in (float("inf"), float("-inf"))): + return None + if isinstance(x, dict): + return {k: _finite(v) for k, v in x.items()} + if isinstance(x, (list, tuple)): + return [_finite(v) for v in x] + return x + + +def set_stop_event(ev: threading.Event | None) -> None: + global STOP + STOP = ev + + +def should_stop() -> bool: + return STOP is not None and STOP.is_set() + + +def update(stage: str, step: int | None = None, total: int | None = None, force: bool = False, **extra) -> None: + global _last_write + with _lock: + _state.update({"stage": stage, "step": step, "total": total, "updated_at": datetime.now(timezone.utc).isoformat()}) + if "started_at" not in _state: + _state["started_at"] = _state["updated_at"] + if "stage_started_at" not in _state or _state.get("_stage") != stage: + _state["stage_started_at"] = time.time(); _state["_stage"] = stage + if step and total: + el = time.time() - _state["stage_started_at"] + _state["eta_s"] = el / max(step, 1) * (total - step) + _state.update(extra) + now = time.time() + if not force and now - _last_write < 3.0: + return + _last_write = now + payload = _finite({k: v for k, v in _state.items() if not k.startswith("_")}) + try: + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('training_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + (json.dumps(payload, default=str),)) + except Exception: + pass + + +def clear() -> None: + with _lock: + _state.clear() diff --git a/fly_trader/train/replay_assemble.py b/fly_trader/train/replay_assemble.py new file mode 100644 index 0000000000000000000000000000000000000000..bb43f6247edf39edcf39bf9ab2e58fb0df518653 --- /dev/null +++ b/fly_trader/train/replay_assemble.py @@ -0,0 +1,178 @@ +"""Replay days → per-token corpus files (thread inside the ``replay`` worker; also ``fly-trader assemble-replay``). + +A day D is assembled once every hour of D and the first 12 hours of D+1 are in ``replay_hours``. For each +pump.fun graduation on D (``migrate`` on pool ``pump-amm`` in the lifecycle rows) it writes the same files the +swap-api puller writes, so ``train/corpus_features.py`` needs no changes: + +* ``data/corpus/trades/.parquet``: every curve and PumpSwap trade leg in [graduation − 30 min, + CORPUS_TRADES_H] + (wallet, side, SOL, tokens, price) — for every token, not a sample; +* ``data/corpus/candles/.parquet``: 1-minute OHLCV in SOL for the first CORPUS_CANDLE_1M_H hours and 5-minute + candles for the rest of the 36 h scanned, plus ``resq_sol`` (the pool's real quote reserve at the last trade of the + candle), which the feature builder prefers over its sqrt(price) approximation when present; +* a ``corpus_tokens`` row (source 'replay', status 'done') with the graduation time and slot from the migrate event. + +DuckDB scans the day's hourly trade files with the graduated-mint list pushed down; a day is ~2–3 GB on disk. +""" +from __future__ import annotations + +import logging +import threading +import time +from datetime import date, datetime, timedelta, timezone + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config, logging_setup +from ..db.connection import transaction +from ..ingest.corpus_pull import CANDLE_SCHEMA, TRADE_SCHEMA +from . import corpus_meta, mature + +log = logging.getLogger(__name__) +CANDLE_SCHEMA_R = CANDLE_SCHEMA.append(pa.field("resq_sol", pa.float64())) + + +def _hours_done() -> set[datetime]: + with transaction() as conn: + return {r["hour"] for r in conn.execute("SELECT hour FROM replay_hours WHERE status IN ('done', 'missing')").fetchall()} + + +def assemblable_days() -> list[date]: + """Days whose 24 hours and the next day's first 12 hours are ingested, newest first, not yet assembled.""" + done = _hours_done() + if not done: + return [] + with transaction() as conn: + did = {r["day"] for r in conn.execute("SELECT day FROM replay_days").fetchall()} + days = sorted({h.astimezone(timezone.utc).date() for h in done}, reverse=True); out = [] + for d in days: + if d in did: + continue + need = [datetime(d.year, d.month, d.day, h, tzinfo=timezone.utc) for h in range(24)] + nxt = d + timedelta(days=1) + need += [datetime(nxt.year, nxt.month, nxt.day, h, tzinfo=timezone.utc) for h in range(12)] + if all(h in done for h in need): + out.append(d) + return out + + +def _files(d: date, kind: str, hours: range) -> list[str]: + day = config.REPLAY_DIR / d.isoformat() + return [str(day / f"{h:02d}_{kind}.parquet") for h in hours if (day / f"{h:02d}_{kind}.parquet").exists()] + + +def band_amm(con, src: str, dst: str) -> None: + """One price scale per mint, causally (as ``mature.aggregate_day`` and the live stream): a PumpSwap leg more than 50x away + from the median of the mint's previous 200 PumpSwap legs is dropped (secondary pools in other quotes); other legs pass.""" + con.execute(f"""CREATE TEMP TABLE {dst} AS SELECT * EXCLUDE (pref) FROM ( + SELECT *, median(price) OVER (PARTITION BY mint ORDER BY ts, slot ROWS BETWEEN 200 PRECEDING AND 1 PRECEDING) AS pref + FROM {src} WHERE pool = 'pump-amm') + WHERE pref IS NULL OR price BETWEEN pref / 50 AND pref * 50 + UNION ALL SELECT * FROM {src} WHERE pool IS DISTINCT FROM 'pump-amm'""") + con.execute(f"DROP TABLE {src}") + + +def assemble_day(d: date) -> tuple[int, int]: + t0 = time.time(); nxt = d + timedelta(days=1) + con = duckdb.connect() + ev_files = _files(d, "events", range(24)) + grads = con.execute("SELECT mint, min(ts) AS g, arg_min(slot, ts) AS slot, arg_min(quote_in_pool, ts) AS rq0 FROM read_parquet(?) " + "WHERE action = 'migrate' AND pool = 'pump-amm' AND mint IS NOT NULL GROUP BY mint", [ev_files]).fetchall() + if not grads: + return 0, 0 + mints = [g[0] for g in grads] + tr_files = _files(d, "trades", range(24)) + _files(nxt, "trades", range(12)) + con.execute("CREATE TEMP TABLE grads AS SELECT * FROM (VALUES " + ",".join("(?, ?)" for _ in grads) + ") t(mint, g)", + [v for g in grads for v in (g[0], g[1])]) + # every trade leg of a graduated mint within [g − 30 min, g + 36 h] + con.execute("CREATE TEMP TABLE tr0 AS SELECT t.mint, t.ts, t.slot, t.pool, t.trader, t.side, t.sol, t.tokens, t.price, t.quote_in_pool, " + "epoch_ms(t.ts) - epoch_ms(g.g) AS rel_ms FROM read_parquet(?) t JOIN grads g USING (mint) " + "WHERE t.ts >= g.g - INTERVAL 30 MINUTE AND t.ts < g.g + INTERVAL 36 HOUR AND t.price > 0 " + "AND (t.pool = 'pump' OR t.quote_in_pool BETWEEN 0.001 AND 100000)", [tr_files]) + band_amm(con, "tr0", "tr") + (cdir := config.CORPUS_DIR / "candles").mkdir(parents=True, exist_ok=True); (tdir := config.CORPUS_DIR / "trades").mkdir(parents=True, exist_ok=True) + win1 = int(config.CORPUS_CANDLE_1M_H * 3_600_000); trades_ms = int(config.CORPUS_TRADES_H * 3_600_000) + # per-mint trade rows for the first hours + tr_tab = con.execute("SELECT mint, ts, CAST(slot AS VARCHAR) AS slot_index, trader AS wallet, side, " + "CASE WHEN pool = 'pump-amm' THEN 'pump_amm' ELSE 'pump' END AS program, price AS price_sol, sol, tokens " + "FROM tr WHERE rel_ms < ? ORDER BY mint, ts", [trades_ms]).fetch_arrow_table() + # candles: 1-minute inside win1, 5-minute after + cd_tab = con.execute(""" + SELECT mint, bucket AS ts, interval, first(price ORDER BY ts, slot) AS open, max(price) AS high, min(price) AS low, + last(price ORDER BY ts, slot) AS close, sum(sol) AS volume_sol, last(quote_in_pool ORDER BY ts, slot) AS resq_sol + FROM (SELECT *, CASE WHEN rel_ms < ? THEN '1m' ELSE '5m' END AS interval, + CASE WHEN rel_ms < ? THEN time_bucket(INTERVAL 1 MINUTE, ts) ELSE time_bucket(INTERVAL 5 MINUTE, ts) END AS bucket + FROM tr WHERE pool = 'pump-amm' OR rel_ms < 0) + GROUP BY mint, bucket, interval ORDER BY mint, bucket""", [win1, win1]).fetch_arrow_table() + n_written = 0; rows = [] + tr_by = tr_tab.to_pandas().groupby("mint") if tr_tab.num_rows else None + cd_by = cd_tab.to_pandas().groupby("mint") if cd_tab.num_rows else None + for mint, g, slot, rq0 in grads: + g_dt = g if g.tzinfo else g.replace(tzinfo=timezone.utc) + cpath, tpath = cdir / f"{mint}.parquet", tdir / f"{mint}.parquet" + upd = {"life_h": None, "candles_1m": 0, "candles_5m": 0, "trades": 0} + if cd_by is not None and mint in cd_by.groups: + c = cd_by.get_group(mint).drop(columns=["mint"]) + upd["candles_1m"] = int((c["interval"] == "1m").sum()); upd["candles_5m"] = int((c["interval"] == "5m").sum()) + upd["life_h"] = max(0.0, (c["ts"].max().timestamp() - g_dt.timestamp()) / 3600.0) + if not cpath.exists(): + pq.write_table(pa.Table.from_pandas(c, schema=CANDLE_SCHEMA_R, preserve_index=False), cpath, compression="zstd") + if tr_by is not None and mint in tr_by.groups: + t = tr_by.get_group(mint).drop(columns=["mint"]); t.insert(2, "tx", None) + upd["trades"] = len(t) + if not tpath.exists(): + pq.write_table(pa.Table.from_pandas(t, schema=TRADE_SCHEMA, preserve_index=False), tpath, compression="zstd") + rows.append((mint, g_dt, int(slot or 0), "replay", "done" if cpath.exists() else "empty", upd["life_h"], upd["candles_1m"], upd["candles_5m"], upd["trades"], + g_dt + timedelta(milliseconds=trades_ms) if upd["trades"] else None, str(cpath) if cpath.exists() else None, str(tpath) if tpath.exists() else None)) + n_written += 1 + with transaction() as conn: + conn.cursor().executemany( + "INSERT INTO corpus_tokens (mint, graduated_at, grad_slot, source, status, life_h, candles_1m, candles_5m, trades, trades_through, candle_path, trade_path) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (mint) DO UPDATE SET graduated_at = EXCLUDED.graduated_at, grad_slot = EXCLUDED.grad_slot, " + "source = EXCLUDED.source, status = EXCLUDED.status, life_h = EXCLUDED.life_h, candles_1m = EXCLUDED.candles_1m, candles_5m = EXCLUDED.candles_5m, " + "trades = EXCLUDED.trades, trades_through = EXCLUDED.trades_through, candle_path = EXCLUDED.candle_path, trade_path = EXCLUDED.trade_path, updated_at = now()", rows) + conn.execute("INSERT INTO replay_days (day, graduations, tokens_written, took_s) VALUES (%s,%s,%s,%s) ON CONFLICT (day) DO UPDATE SET graduations = EXCLUDED.graduations, " + "tokens_written = EXCLUDED.tokens_written, took_s = EXCLUDED.took_s, assembled_at = now()", (d, len(grads), n_written, time.time() - t0)) + con.close() + log.info("assembled %s: %d graduations, %d tokens written, %d trade rows, %d candle rows in %.0fs", d, len(grads), n_written, tr_tab.num_rows, cd_tab.num_rows, time.time() - t0) + return len(grads), n_written + + +_last_meta = [0.0] + + +def assemble_loop(stop_event: threading.Event | None = None, idle_s: float = 60.0) -> None: + while not (stop_event is not None and stop_event.is_set()): + try: + days = assemblable_days() + except Exception: + log.exception("assemble: planning failed"); days = [] + for d in days: + if stop_event is not None and stop_event.is_set(): + return + try: + assemble_day(d) + except Exception: + log.exception("assemble %s failed", d) + if days or time.time() - _last_meta[0] > 1800: + try: + corpus_meta.rebuild(); _last_meta[0] = time.time() + except Exception: + log.exception("corpus_meta rebuild failed; mature build waits for the next round") + else: + try: + mature.loop_once() + except Exception: + log.exception("mature universe build failed") + end = time.time() + idle_s + while time.time() < end and not (stop_event is not None and stop_event.is_set()): + time.sleep(1.0) + + +def main() -> None: + logging_setup.setup("replay") + days = assemblable_days() + print(f"{len(days)} days ready") + for d in days: + g, n = assemble_day(d); print(f"{d}: {g} graduations, {n} tokens") diff --git a/fly_trader/train/scaling.py b/fly_trader/train/scaling.py new file mode 100644 index 0000000000000000000000000000000000000000..ad869a1f9666bdf6ecda491176c5124d65b0ec84 --- /dev/null +++ b/fly_trader/train/scaling.py @@ -0,0 +1,46 @@ +"""Robust feature scaling, shared verbatim by the selector and the fly, so both models receive the same signals. + +Minute features are fat-tailed (a pool's reserve at graduation or a creator's first buy sits thousands of standard +deviations out); mean/std standardisation leaves such values at hundreds or thousands, which swamps a neural network's +units while leaving a tree model unaffected. Each feature is centred on its median, divided by a robust spread (the +inter-quartile range / 1.349, the standard deviation of a normal; the plain standard deviation when the IQR is zero, +1 for a constant feature) and compressed with asinh: linear near the centre, logarithmic in the tails. The transform +is monotone per feature, so tree splits are unchanged; it is fit on the training rows only. +""" +from __future__ import annotations + +import numpy as np + +FIT_SAMPLE = 2_000_000 + + +class RobustScaler: + def __init__(self, center: np.ndarray, scale: np.ndarray): + self.center = np.asarray(center, dtype=np.float32) + self.scale = np.asarray(scale, dtype=np.float32) + + @classmethod + def fit(cls, X: np.ndarray, sample: int = FIT_SAMPLE, seed: int = 0) -> "RobustScaler": + idx = np.arange(len(X)) if len(X) <= sample else np.random.default_rng(seed).choice(len(X), sample, replace=False) + Xs = np.asarray(X[idx], dtype=np.float64) + med = np.median(Xs, axis=0); q75, q25 = np.percentile(Xs, [75, 25], axis=0); iqr = (q75 - q25) / 1.349; sd = Xs.std(axis=0) + return cls(med, np.where(iqr > 1e-12, iqr, np.where(sd > 1e-12, sd, 1.0))) + + def transform(self, X: np.ndarray) -> np.ndarray: + """A scaled copy of ``X``, never touching it. One allocation: the arithmetic then runs in place, so a walk-forward + block of tens of millions of rows costs one array instead of four (subtract, divide, asinh and cast each used to + allocate their own).""" + return self.transform_into(np.array(X, dtype=np.float32, order="C")) + + def transform_into(self, Z: np.ndarray) -> np.ndarray: + """Scale ``Z`` in place and return it. Only for an array the caller owns — a block copy from fancy indexing, never + a view of the corpus, which this would corrupt.""" + Z -= self.center; Z /= self.scale; np.arcsinh(Z, out=Z) + return Z + + def state(self) -> dict: + return {"center": self.center.tolist(), "scale": self.scale.tolist()} + + @classmethod + def from_state(cls, s: dict) -> "RobustScaler": + return cls(np.asarray(s["center"]), np.asarray(s["scale"])) diff --git a/fly_trader/train/selector.py b/fly_trader/train/selector.py new file mode 100644 index 0000000000000000000000000000000000000000..6eda96237cdcfb155a50e9071f3d4d9ae25ac8b4 --- /dev/null +++ b/fly_trader/train/selector.py @@ -0,0 +1,347 @@ +"""The selector: a gradient-boosted model of each minute's net return over the hold (``train/decisions.HOLD_MIN``). + +Strategy: the model predicts the net return of buying a token now and selling ``HOLD_MIN`` minutes later, after the +real fees and price impact; it trains on every eligible minute. It trades only tokens at least ``MIN_AGE_H`` hours +past graduation (or graduated before the archive began) — random entries into younger tokens lose 4–28 % per trade at +real costs. ``in_universe`` is the one definition of that universe for training, backtest and the live engine. + +Trading configuration, shared verbatim with the fly (``train/fly_selector.py``): features robust-scaled +(``train/scaling.py``); buy when the predicted net return clears the buy line; the line is the one (of +``LINE_CANDIDATES``) with the most total net profit on the first half of the walk-forward's out-of-sample days, and it +is judged on the second half only; same universe, gates, one position per token, hold, pessimistic fills, paper costs +and sizing procedure (``agent/sizing.py`` bands from the out-of-sample trades). + +Walk-forward protocol (the strategy gate): every day after a 21-day warm-up is a test day, scored by a model fit on +every eligible minute of the days at least two days earlier (refit every 7 days). A model goes to work only if the +evaluation half made money after costs, beat random picks from the same universe and has at least 100 trades. The +deployed model is then fit on every day and saved with its line, scaler and sizing table +(``data/brain/selectors/selector_.joblib`` + ``brain_snapshots`` kind 'selector'). Runs inside the training +pipeline (``train/pipeline.py``) and as ``fly-trader train-selector``. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading +import time +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path + +import joblib +import numpy as np +from sklearn.ensemble import HistGradientBoostingRegressor + +from .. import config +from ..agent import sizing +from ..db.apilog import record_event +from ..db.connection import transaction +from ..market.features import FEATURE_VERSION +from . import progress as prog +from .decisions import HOLD_MIN, MIN_RESQ_SOL, MIN_VOL_15M_SOL, X_COLS, DecisionSet, build, evaluate, random_trades, rank_corr, summarize, taken_rows +from .mature import AGG_VERSION +from .scaling import RobustScaler + +log = logging.getLogger(__name__) +SELECTOR_DIR = config.BRAIN_DIR / "selectors" +# the data and model definitions a selector was trained on (the input columns by hash, so dropping or adding one retires +# old models); a model from other definitions is never loaded or shown. The fly has its own (train/fly_selector.FLY_VERSION), +# so a change to the fly never retires a selector. +def _skill_version() -> str: + from .wallet_skill import skill_version + return skill_version() + + +DATA_VERSION = {"agg": AGG_VERSION, "features": FEATURE_VERSION, "costs": "real-fees-sized-1", "selector": "multi-1", + "cols": hashlib.sha1(",".join(X_COLS).encode()).hexdigest()[:8], "skill": _skill_version(), "meta": "rug-1"} +WARMUP_DAYS, BLOCK_DAYS = 21, 7 # walk-forward: the first 21 days only train; every later day is tested, refit every 7 days +MIN_AGE_H = 6.0 # trade only tokens at least this long past graduation (unknown age = graduated before the archive); fitted, see decisions.HOLD_MIN +MIN_EV = 0.01 # the buy line of a model before its walk-forward chose one +LINE_CANDIDATES = (0.0, 0.0025, 0.005, 0.0075, 0.01, 0.015, 0.02, 0.03) +MIN_LINE_TRADES = 100 +GBM_PARAMS = dict(max_iter=200, learning_rate=0.06, max_leaf_nodes=63, min_samples_leaf=300, l2_regularization=1.0) +IC_SAMPLE = 200_000 + + +def in_universe(X: np.ndarray, cols: list[str]) -> np.ndarray: + """Rows the models may trade: age unknown (graduated before the archive) or at least ``MIN_AGE_H`` hours old.""" + X = np.atleast_2d(X); c = {n: i for i, n in enumerate(cols)} + return (X[:, c["age_known"]] == 0) | (np.expm1(X[:, c["log_age_h"]]) >= MIN_AGE_H) + + +def is_current(meta: dict | None) -> bool: + """Trained on every definition in ``DATA_VERSION``. Keys a model carries that are no longer definitions (the fly's, + which moved to ``FLY_VERSION``) are ignored; a model saved before the column hash existed is judged on the rest.""" + d = (meta or {}).get("data") + return bool(meta) and isinstance(d, dict) and all(d.get(k) == v for k, v in DATA_VERSION.items() if k != "cols" or "cols" in d) + + +def is_deployable(meta: dict | None) -> bool: + """Trained on the current definitions AND its evaluation half made money after costs and beat random picks.""" + return is_current(meta) and bool(meta.get("deployable")) + + +def deploy_decision(p: dict, rb: dict) -> tuple[bool, str]: + if (p.get("n") or 0) < 100: + return False, f"too few backtest trades ({p.get('n') or 0})" + if p.get("mean") is None or p["mean"] <= 0: + return False, f"its backtest lost money ({(p.get('mean') or 0) * 100:+.2f}% per trade after costs)" + if rb.get("mean") is not None and p["mean"] <= rb["mean"]: + return False, "it did not beat random picks" + return True, f"its backtest made {p['mean'] * 100:+.2f}% per trade after costs (random {(rb.get('mean') or 0) * 100:+.2f}%)" + + +@dataclass +class SelectorModel: + gbm: HistGradientBoostingRegressor + scaler: RobustScaler + cols: list[str] + threshold: float # the buy line: predicted net return + horizon_min: int + trained_through: str + metrics: dict = field(default_factory=dict) + sizing: list = field(default_factory=list) # agent/sizing.py table from the backtest's out-of-sample trades + stack: dict = field(default_factory=dict) # train/strategies.final_models: strategies, veto, combination rule (empty: single ev model) + + def score(self, X: np.ndarray) -> np.ndarray: + """Predicted net return over the hold.""" + return self.gbm.predict(self.scaler.transform(X)) + + def decide(self, X: np.ndarray, cols: list[str], t_start: float) -> dict: + """Per row: the strategy that trades it, score, hold, line, sizing table, allowed or the filter's reason + (train/strategies.decide); a model without a stack decides with its single ev model.""" + from . import strategies + st = getattr(self, "stack", None) or {} + if st.get("strategies"): + return strategies.decide(st, X, cols, t_start) + idx = [cols.index(c) for c in self.cols]; sc = self.score(np.atleast_2d(X)[:, idx]); n = len(sc) + return {"strategy": np.full(n, "ev", dtype=object), "score": sc, "hold_s": np.full(n, self.horizon_min * 60.0), "threshold": np.full(n, self.threshold), + "tables": [self.sizing] * n, "allow": np.ones(n, bool), "reason": np.full(n, "trade", dtype=object)} + + def universe(self, X: np.ndarray) -> np.ndarray: + return in_universe(X, self.cols) + + +def fit(ds: DecisionSet, train: np.ndarray, seed: int = 0) -> SelectorModel: + """Regression of the net return (pessimistic fill, paper costs, clipped to ±100 %) on every eligible training minute.""" + idx = np.flatnonzero(train) + scaler = RobustScaler.fit(ds.X[idx], seed=seed) + gbm = HistGradientBoostingRegressor(**GBM_PARAMS, random_state=seed) + gbm.fit(scaler.transform(ds.X[idx]), np.clip(ds.fwd_pess[idx], -1.0, 1.0)) + return SelectorModel(gbm=gbm, scaler=scaler, cols=list(ds.cols), threshold=MIN_EV, horizon_min=int(ds.horizon_s // 60), trained_through=str(max(ds.day[train]))) + + +def ic(ds: DecisionSet, scores: np.ndarray, rows: np.ndarray, seed: int = 0) -> float | None: + """Information coefficient: rank correlation of the scores with the realised net returns on (a sample of) ``rows``.""" + idx = np.flatnonzero(rows) + if len(idx) > IC_SAMPLE: + idx = np.random.default_rng(seed).choice(idx, IC_SAMPLE, replace=False) + return rank_corr(scores[idx], ds.fwd_pess[idx]) if len(idx) > 1 else None + + +@dataclass +class Fold: + day: object + model: SelectorModel + test: np.ndarray # [N] bool, the tradable rows of the test days + scores: np.ndarray # [N] float, predicted net return on the test rows (NaN elsewhere) + ic: float | None + + +def fold(ds: DecisionSet, D, seed: int = 0, min_train: int = 50_000, min_test: int = 500) -> Fold | None: + """One walk-forward step: fit on every day before the first test day minus one (one-day purge), score the test + day(s) ``D`` (a date or a list of dates). None when either side is too small.""" + test_days = list(D) if isinstance(D, (list, tuple)) else [D] + train = ds.day < (min(test_days) - timedelta(days=1)); tix = np.flatnonzero(np.isin(ds.day, test_days)) + if train.sum() < min_train or len(tix) < min_test: + return None + m = fit(ds, train, seed=seed) + tix = tix[m.universe(ds.X[tix])]; test = np.zeros(len(ds.y), bool); test[tix] = True + scores = np.full(len(ds.y), np.nan); scores[tix] = m.score(ds.X[tix]) if len(tix) else scores[tix] + return Fold(day=test_days[0], model=m, test=test, scores=scores, ic=ic(ds, scores, test, seed)) + + +def choose_line(ds: DecisionSet, scores: np.ndarray, rows: np.ndarray) -> tuple[float, list[dict]]: + """The buy line with the most total net profit (fixed size, one position per token) on ``rows``.""" + table = [] + for c in LINE_CANDIDATES: + tr = taken_rows(ds, rows & (scores >= c)); r = ds.fwd_pess[tr] + table.append({"line": c, "trades": int(len(r)), "mean": float(r.mean()) if len(r) else None, "total": float(r.sum())}) + ok = [t for t in table if t["trades"] >= MIN_LINE_TRADES] + return (max(ok, key=lambda t: t["total"])["line"] if ok else MIN_EV), table + + +def _pct(v, fmt: str = "+.2f") -> str: + return f"{v*100:{fmt}}%" if v is not None else "-" + + +def walk_forward(ds: DecisionSet, warmup_days: int = WARMUP_DAYS, block_days: int = BLOCK_DAYS, stop: threading.Event | None = None) -> dict: + """Out-of-sample scores for every day after ``warmup_days`` (refit every ``block_days``); the buy line chosen on the + first half of those days, everything judged on the second half.""" + days = ds.days; scores = np.full(len(ds.y), np.nan); tested = np.zeros(len(ds.y), bool); ics: dict[str, float | None] = {} + blocks = [days[k:k + block_days] for k in range(warmup_days, len(days), block_days)] + for k, blk in enumerate(blocks): + if stop is not None and stop.is_set(): + break + prog.update("selector walk-forward", k, len(blocks), day=f"{blk[0]}..{blk[-1]}", force=True) + fo = fold(ds, blk, seed=k) + if fo is None: + continue + scores[fo.test] = fo.scores[fo.test]; tested |= fo.test + for D in blk: + ics[str(D)] = ic(ds, scores, fo.test & (ds.day == D), k) + tdays = sorted(set(ds.day[tested].tolist())) + if len(tdays) < 2: + empty = summarize(np.array([])); empty.update(days_positive=0, days=0) + return {"per_day": {}, "ic": ics, "line": MIN_EV, "lines": [], "pooled": empty, "random": summarize(np.array([])), "sizing": [], "bankroll": {}} + sel_days, ev_days = tdays[: len(tdays) // 2], tdays[len(tdays) // 2:] + sel_mask = tested & np.isin(ds.day, sel_days); ev_mask = tested & np.isin(ds.day, ev_days) + line, lines = choose_line(ds, scores, sel_mask) + log.info("buy line %s chosen on %s..%s (total net profit per candidate: %s)", _pct(line), sel_days[0], sel_days[-1], + ", ".join(f"{t['line'] * 100:.2f}%: {t['total']:+.1f} ({t['trades']})" for t in lines)) + out = {"per_day": {}, "ic": ics, "line": line, "lines": lines, "selection_days": [str(sel_days[0]), str(sel_days[-1])], + "evaluation_days": [str(ev_days[0]), str(ev_days[-1])]} + sel_set = set(sel_days) + for D in tdays: + td = tested & (ds.day == D); r = evaluate(ds, scores, td, line)["pooled"] + rs = summarize(random_trades(ds, td, int((td & (scores >= line)).sum()))); half = "selection" if D in sel_set else "evaluation" + out["per_day"][str(D)] = {**r, "ic": ics.get(str(D)), "line": line, "random_mean": rs["mean"], "half": half} + log.info("selector %s [%s]: IC %s | predicted >= %s: n=%s mean %s median %s win %s PF %s | random %s", D, half, + f"{ics.get(str(D)):.3f}" if ics.get(str(D)) is not None else "-", _pct(line), r["n"], _pct(r["mean"]), _pct(r["median"]), _pct(r["win"], ".0f"), + f"{r['pf']:.2f}" if r["pf"] is not None else "-", _pct(rs["mean"])) + record_event("info", "selector", f"walk-forward {D}", {"day": str(D), "ic": ics.get(str(D)), **r, "random_mean": rs["mean"], "half": half, "line": line}) + ev_rows = taken_rows(ds, ev_mask & (scores >= line)) + pooled = summarize(ds.fwd_pess[ev_rows]); pooled["days_positive"] = sum(1 for D in ev_days if (out["per_day"][str(D)]["mean"] or 0) > 0); pooled["days"] = len(ev_days) + out["pooled"] = pooled + out["random"] = summarize(random_trades(ds, ev_mask, int((ev_mask & (scores >= line)).sum()))) + # position sizing from the out-of-sample trades at the chosen line: bands of margin over the line -> growth-optimal fraction + rows = taken_rows(ds, tested & (scores >= line)); margins = scores[rows] - line; rets = ds.fwd_pess[rows] + resq = np.expm1(ds.X[rows, ds.cols.index("log_liquidity_sol")].astype(np.float64)) / 2.0 + out["sizing"] = sizing.build_table(margins, rets) + out["bankroll"] = {"sized": sizing.simulate_bankroll(ds.ts[rows], ds.horizon_s, margins, rets, out["sizing"], res_quote=resq), + "fixed": sizing.simulate_bankroll(ds.ts[rows], ds.horizon_s, margins, rets, None, res_quote=resq)} + return out + + +def save(m: SelectorModel, run_id: str | None = None) -> tuple[Path, int]: + SELECTOR_DIR.mkdir(parents=True, exist_ok=True) + path = SELECTOR_DIR / f"selector_{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}.joblib" + joblib.dump(m, path); sha = hashlib.sha256(path.read_bytes()).hexdigest() + with transaction() as conn: + row = conn.execute("INSERT INTO brain_snapshots (run_id, path, sha256, kind, note) VALUES (%s,%s,%s,'selector',%s) RETURNING id", + (run_id, str(path), sha, json.dumps({"threshold": m.threshold, "horizon_min": m.horizon_min, "trained_through": m.trained_through, + **m.metrics}, default=str))).fetchone() + return path, int(row["id"]) + + +def latest_current(conn) -> dict | None: + """The newest deployable selector snapshot (current definitions, profitable backtest), or None.""" + for r in conn.execute("SELECT id, path, note FROM brain_snapshots WHERE kind = 'selector' ORDER BY id DESC").fetchall(): + try: + meta = json.loads(r["note"] or "{}") + except ValueError: + continue + if is_deployable(meta) and Path(r["path"]).exists(): + return dict(r) + return None + + +def load_snapshot(snapshot_id: int) -> SelectorModel | None: + """One named selector snapshot, whatever has been deployed since (agent/selector_session.py's pin).""" + with transaction() as conn: + row = conn.execute("SELECT path FROM brain_snapshots WHERE id = %s AND kind = 'selector'", (snapshot_id,)).fetchone() + return joblib.load(row["path"]) if row and Path(row["path"]).exists() else None + + +def load_latest() -> SelectorModel | None: + with transaction() as conn: + r = latest_current(conn) + return joblib.load(r["path"]) if r else None + + +def main(days: int | None = None, horizon_min: int = HOLD_MIN, stop_event: threading.Event | None = None) -> dict: + """The strategy stack (train/strategies.py) fitted, judged and saved; ``days``: None = the whole corpus.""" + from . import strategies + from ..ops.reset import reset_training_stats + reset_training_stats("selector", reason="selector training") + prog.set_stop_event(stop_event); prog.clear() + prog.update("selector: building decision points", 0, 1, force=True) + t0 = time.time(); ds = build(days=days, horizon_min=horizon_min, holds=strategies.HOLDS_MIN) + log.info("decision points: %d rows, %d days, universe mean %+.2f%% (%.0fs)", len(ds.y), len(ds.days), ds.fwd_pess.mean() * 100, time.time() - t0) + stack = strategies.fit_stack(ds, stop_event) + for c in stack.components: + log.info("component %-28s %s — %s", c["name"], "PASSED" if c["passed"] else "dropped", c["reason"]) + record_event("info", "selector", f"component {c['name']}: {'passed' if c['passed'] else 'dropped'}", c) + if stop_event is not None and stop_event.is_set(): + return {"stopped": True} + prog.update("selector: fitting the deployable models on every day", 0, 1, force=True) + models = strategies.final_models(ds, stack) if stack.fits else {"strategies": {}} + # scored with models refit without the holdout: the deployed ones above have seen those days and would score themselves + holdout = strategies.score_holdout(ds, strategies.final_models(ds, stack, exclude_days=stack.holdout_days), stack.holdout_days) if stack.holdout_days else {} + if holdout.get("n"): + log.info("holdout %s..%s (no fit ever saw these days): %d trades, %s winners, PF %s, %s per trade vs random %s", + holdout["days"][0], holdout["days"][1], holdout["n"], + f"{holdout['win'] * 100:.1f}%" if holdout.get("win") is not None else "-", + f"{holdout['pf']:.2f}" if holdout.get("pf") is not None else "-", + f"{holdout['mean'] * 100:+.2f}%" if holdout.get("mean") is not None else "-", + f"{(holdout.get('random_mean') or 0) * 100:+.2f}%") + elif stack.holdout_days: + log.info("holdout %s: the book took no trades there", holdout.get("days")) + ev = (models["strategies"] or {}).get("ev") or next(iter((models["strategies"] or {}).values()), None) + if ev is None: + final = fit(ds, np.ones(len(ds.y), bool), seed=99) + else: + final = SelectorModel(gbm=ev["gbm"], scaler=ev["scaler"], cols=ev["cols"], threshold=ev["line"], horizon_min=ev["hold_min"], trained_through=str(ds.days[-1]), + sizing=ev["sizing"]) + final.stack = models + final.metrics = {"walk_forward": {**stack.evaluation, "days": None}, "selection": stack.selection, "random_baseline": {"mean": stack.evaluation.get("random_mean")}, + "components": stack.components, "strategies": {k: {x: v[x] for x in ("line", "hold_min", "thr", "high", "hours", "regimes", "sizing", "selection", "evaluation")} + for k, v in models["strategies"].items()}, + "groups": stack.groups, "combine": stack.combine, "veto": {k: v for k, v in (stack.veto or {}).items() if k not in ("p",)} or None, + "fallback": stack.fallback, "holdout": holdout, "line": final.threshold, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", + "data": DATA_VERSION, "deployable": stack.deployable, "deploy_reason": stack.reason, "sizing": final.sizing, + "model": {"kind": "stack", "min_age_h": MIN_AGE_H, "min_resq_sol": MIN_RESQ_SOL, "min_vol_15m_sol": MIN_VOL_15M_SOL}, + "rows": int(len(ds.y)), "days": len(ds.days), "first_day": str(ds.days[0]), "last_day": str(ds.days[-1])} + path, sid = save(final) + record_event("info", "selector", f"selector saved (snapshot {sid})", {"path": str(path), "deployable": stack.deployable, "reason": stack.reason, + "fallback": stack.fallback, **stack.evaluation}) + prog.update("selector: saved", 1, 1, force=True, snapshot_id=sid, deployable=stack.deployable, deploy_reason=stack.reason) + log.info("selector saved: %s (snapshot %d) — %s: %s", path, sid, "put to work" if stack.deployable else "NOT put to work", stack.reason) + return {"snapshot_id": sid, "deployable": stack.deployable, "deploy_reason": stack.reason, "line": final.threshold, "components": stack.components} + + +def main_single(days: int | None = None, horizon_min: int = HOLD_MIN, stop_event: threading.Event | None = None) -> dict: + """The single-strategy selector (before 2026-09-15); kept for comparison runs. ``days``: None = the whole corpus.""" + from ..ops.reset import reset_training_stats + reset_training_stats("selector", reason="selector training") + prog.set_stop_event(stop_event); prog.clear() + prog.update("selector: building decision points", 0, 1, force=True) + t0 = time.time(); ds = build(days=days, horizon_min=horizon_min) + prog.update("selector: decision points ready", 1, 1, force=True, rows=int(len(ds.y)), tokens=int(len(set(ds.mint.tolist()))), days=len(ds.days), + universe_mean=float(ds.fwd_pess.mean())) + log.info("decision points: %d rows, %d days, universe mean %+.2f%% (%.0fs)", len(ds.y), len(ds.days), ds.fwd_pess.mean() * 100, time.time() - t0) + wf = walk_forward(ds, stop=stop_event) + p = wf["pooled"]; rb = wf["random"] + log.info("selector walk-forward, evaluation half at line %s: n=%s mean %s median %s win %s PF %s days positive %s/%s | random %s", _pct(wf["line"]), p["n"], + _pct(p["mean"]), _pct(p["median"]), _pct(p["win"], ".0f"), f"{p['pf']:.2f}" if p["pf"] else "-", p["days_positive"], p["days"], _pct(rb["mean"])) + prog.update("selector: walk-forward done", 1, 1, force=True, walk_forward=p, random_baseline=rb, line=wf["line"]) + if stop_event is not None and stop_event.is_set(): + return wf + prog.update("selector: fitting the deployable model on every day", 0, 1, force=True) + deployable, why = deploy_decision(p, rb) + final = fit(ds, np.ones(len(ds.y), bool), seed=99); final.threshold = wf["line"]; final.sizing = wf["sizing"] + bk = wf["bankroll"] + if bk: + log.info("bankroll replay of the backtest trades: sized %.2fx (worst drawdown %.0f%%) vs fixed %g SOL %.2fx (worst drawdown %.0f%%)", + bk["sized"]["multiple"] or 0, bk["sized"]["max_drawdown"] * 100, config.MAX_POSITION_SOL, bk["fixed"]["multiple"] or 0, bk["fixed"]["max_drawdown"] * 100) + final.metrics = {"walk_forward": p, "random_baseline": rb, "line": wf["line"], "lines": wf["lines"], "selection_days": wf.get("selection_days"), + "evaluation_days": wf.get("evaluation_days"), "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", "data": DATA_VERSION, "deployable": deployable, "deploy_reason": why, + "sizing": wf["sizing"], "bankroll": bk, "model": {"kind": "ev", "line": wf["line"], "min_age_h": MIN_AGE_H, + "min_resq_sol": MIN_RESQ_SOL, "min_vol_15m_sol": MIN_VOL_15M_SOL}, + "ic_by_day": wf["ic"], "rows": int(len(ds.y)), "days": len(ds.days), "first_day": str(ds.days[0]), "last_day": str(ds.days[-1])} + path, sid = save(final) + record_event("info", "selector", f"selector saved (snapshot {sid})", {"path": str(path), "line": final.threshold, "deployable": deployable, "reason": why, **p}) + prog.update("selector: saved", 1, 1, force=True, snapshot_id=sid, path=str(path), line=final.threshold, deployable=deployable, deploy_reason=why) + log.info("selector saved: %s (snapshot %d, line %s) — %s: %s", path, sid, _pct(final.threshold), "put to work" if deployable else "NOT put to work", why) + return {**wf, "snapshot_id": sid, "deployable": deployable, "deploy_reason": why} diff --git a/fly_trader/train/selector_eval.py b/fly_trader/train/selector_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..22dd407f8e9109e7a611fdeed8e58e9233d29372 --- /dev/null +++ b/fly_trader/train/selector_eval.py @@ -0,0 +1,52 @@ +"""Diagnostic walk-forward of the selector (``fly-trader selector-eval``): the same folds as the strategy gate +(``train/selector.fold``: decision points, purge, size guards, scaler and model), plus close-fill vs pessimistic-fill +returns at the default buy line, what the picks look like, and permutation importance on the last fold.""" +import time + +import numpy as np +import pandas as pd +from sklearn.inspection import permutation_importance + +from fly_trader.train.decisions import build, summarize, trades_from_picks +from fly_trader.train.selector import fold + + +def _pct(v) -> str: + return f"{v*100:+.2f}%" if v is not None else "-" + + +def main(days: int | None = None, test_days: int = 21) -> None: + t0 = time.time() + ds = build(days=days) + print(f"decision points {len(ds.y):,} over {len(ds.days)} days, universe mean {ds.fwd_pess.mean()*100:+.2f}% ({time.time()-t0:.0f}s)", flush=True) + close_r, pess_r, picked = [], [], []; last = None + for k, D in enumerate(ds.days[-test_days:]): + fo = fold(ds, D, seed=k) + if fo is None: + print(f"{D}: skipped (train or test too small)", flush=True); continue + last = fo; pick = fo.test & (fo.scores >= fo.model.threshold) + rc = trades_from_picks(ds, pick, returns=ds.fwd); rp = trades_from_picks(ds, pick) + close_r.append(rc); pess_r.append(rp); picked.append(np.flatnonzero(pick)) + sm = summarize(rp) + print(f"{D}: IC {fo.ic if fo.ic is not None else float('nan'):.3f} | n={sm['n']:4d} close-fill mean {_pct(summarize(rc)['mean'])} | " + f"next-open fill mean {_pct(sm['mean'])} median {_pct(sm['median'])} win {_pct(sm['win'])}", flush=True) + for name, rs in (("close fill", close_r), ("next-open fill", pess_r)): + r = np.concatenate(rs) if rs else np.array([]); sm = summarize(r) + if not sm["n"]: + print(f"\nALL [{name}]: no trades"); continue + print(f"\nALL [{name}]: n={sm['n']} mean {_pct(sm['mean'])} median {_pct(sm['median'])} win {_pct(sm['win'])} PF {sm['pf'] if sm['pf'] is not None else float('inf'):.2f} " + f"| p10 {np.percentile(r, 10)*100:+.1f}% p90 {np.percentile(r, 90)*100:+.1f}%") + X = pd.DataFrame(ds.X, columns=ds.cols); sel = np.concatenate(picked) if picked else np.array([], dtype=int) + show = [c for c in ["ret_1m", "ret_5m", "ret_15m", "ret_1h", "imb_5m", "imb_15m", "logvol_15m", "logn_15m", "traders_15m", "dd_1h", "overext_1h", "log_liquidity_sol", "log_age_h", "n_trades_1m"] if c in ds.cols] + if len(sel): + print("\npicked minutes vs all decision points (medians):"); print(pd.DataFrame({"picked": X.iloc[sel][show].median(), "all": X[show].median()}).round(4).to_string()) + idx = np.flatnonzero(last.test)[:30000] if last is not None else np.array([], dtype=int) + if len(idx): + m = last.model + imp = permutation_importance(m.gbm, m.scaler.transform(ds.X[idx]), np.clip(ds.fwd_pess[idx], -1, 1), n_repeats=3, random_state=0) + print(f"\npermutation importance (R² drop, last fold {last.day}):"); print(pd.Series(imp.importances_mean, index=ds.cols).sort_values(ascending=False).head(12).round(4).to_string()) + print(f"\n({time.time()-t0:.0f}s)") + + +if __name__ == "__main__": + main() diff --git a/fly_trader/train/strategies.py b/fly_trader/train/strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..410859acf55f840bdbf3663d320af4badbeea916 --- /dev/null +++ b/fly_trader/train/strategies.py @@ -0,0 +1,749 @@ +"""The selector's strategy stack (2026-09-15): three sub-strategies, a per-strategy win filter (meta-label), a dump +veto and hour/market gates, every parameter chosen by the walk-forward and one objective. + +Objective (``Score``): on the walk-forward's selection half, among settings with ≥ ``WIN_MIN`` winning trades, profit +factor ≥ ``PF_MIN`` and ≥ ``selector.MIN_LINE_TRADES`` trades, the one with the +most total net profit (fixed size, one position per token, each position's own hold). When no setting reaches the win +rate, the most profitable one that meets the other conditions is kept and flagged (``fallback``). Only the chosen +setting of a component is ever scored on the evaluation half, where it must meet the same bars; a filter (meta-label, +veto, gates) must also beat removing the same number of trades at random (``RANDOM_SEEDS`` draws). Search ranges come +from the data: thresholds over deciles and tail percentiles (``QUANTILES``) of the variable among the selection half's +candidates, holds over ``HOLDS_MIN``. + +Strategies (each: a definitional candidate set, fitted trigger thresholds, its own GBM of the net return over its own +hold, its own line): +- ``ev``: every tradable minute (today's selector), hold ``EV_HOLD``; +- ``capitulation``: a falling minute without insider selling (ret_1m < 0, insider_sell_share_1m = 0); thresholds on the + drop size, the top seller's share and returning buyers (small-coin short-term reversal: Fičura & Colak 2023); +- ``breakout``: organic buying pressure at a new high (org_imb_5m > 0, buyers_slope > 0, at the 1 h or 3 h high); + thresholds on organic imbalance, buyer growth and organic volume (order-flow and attention evidence, wash removed). +A per-strategy classifier of P(net return > 0), trained only on that strategy's out-of-sample candidates (nested +walk-forward; the primary score is one of its inputs), filters trades (Joubert 2022 meta-labeling). A multiclass +GBM of the loss tail gives P(return ≤ d) for dump levels d at tail quantiles of the selection half's pick returns +and vetoes buys (MemeTrans/MELT: rug risk). Hours and market-volume deciles whose selection-half profit factor is below 1 +are blocked. Where several strategies fire on a token-minute, the one with the larger measured Kelly fraction (its +sizing band) or the higher score wins — whichever rule made more on the selection half. +""" +from __future__ import annotations + +import hashlib +import logging +import threading +from dataclasses import dataclass, field +from pathlib import Path +from datetime import timedelta + +import numpy as np +from scipy.stats import binom +from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor + +from .. import config +from ..agent import sizing +from . import progress as prog +from .decisions import GROUPS, LEGACY_COLS, DecisionSet, taken_idx +from .scaling import RobustScaler + +log = logging.getLogger(__name__) +HOLDS_MIN = (10, 20, 30, 45, 60, 90, 120, 180, 240) +EV_HOLD = 120 +WIN_MIN, PF_MIN, SIGN_ALPHA = 0.65, 1.3, 0.05 +# The bars a setting must clear, as a key: results fitted under another objective (train/wallet_skill.py's saved candidates) +# are discarded rather than compared with these. The weekly sign test is recorded but no longer required — with 9 weeks in +# each walk-forward half it demanded 8 winning weeks (89 %), a stricter bar than the 65 % winners it was meant to protect, +# and nothing in the corpus passed it (operator decision, 2026-09-16). +OBJECTIVE = f"win{WIN_MIN}-pf{PF_MIN}-trades-noweekly" +QUANTILES = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.975, 0.99, 0.995, 0.999) +LOSS_QUANTILES = (0.05, 0.10, 0.20, 0.30) +RANDOM_SEEDS = 20 +PASSES = 2 +PREDICT_CHUNK = 2_000_000 # rows scored at once out of sample: the block's own copy stays small +# The last three weeks are never fitted on: no strategy, filter, gate or input group may see them. The finished book is +# scored on them exactly once, for the record — never to decide anything, or they would stop being a holdout. At the +# book's observed rate (~48 trades a day) that is ~1,000 trades, an order of magnitude above MIN_LINE_TRADES. +HOLDOUT_DAYS = 21 +CACHE_DIR = config.CORPUS_DIR / "stack_cache" + + +def _rss_gb() -> float: + """The process's resident size now (ru_maxrss is a high-water mark and hides a climb).""" + import psutil + return psutil.Process().memory_info().rss / 2**30 + + +def _gbm_params(): + from .selector import GBM_PARAMS + return GBM_PARAMS + + +def _min_trades() -> int: + from .selector import MIN_LINE_TRADES + return MIN_LINE_TRADES + + +# ---------------------------------------------------------------- objective +@dataclass +class Score: + n: int = 0 + win: float | None = None + pf: float | None = None + total: float = 0.0 + mean: float | None = None + weeks: int = 0 + weeks_pos: int = 0 + sign_ok: bool = False + + @property + def base_ok(self) -> bool: + return self.n >= _min_trades() and self.pf is not None and self.pf >= PF_MIN + + @property + def admissible(self) -> bool: + return self.base_ok and self.win is not None and self.win >= WIN_MIN + + def dict(self) -> dict: + return {"n": self.n, "win": self.win, "pf": self.pf, "total": self.total, "mean": self.mean, "weeks": self.weeks, "weeks_pos": self.weeks_pos, + "sign_ok": self.sign_ok, "admissible": self.admissible, "base_ok": self.base_ok} + + +def score_trades(ds: DecisionSet, tr: np.ndarray, returns: np.ndarray, day0) -> Score: + r = np.asarray(returns[tr], dtype=np.float64) + ok = np.isfinite(r); r = r[ok]; tr = np.asarray(tr)[ok] + if len(r) == 0: + return Score() + g, l = r[r > 0].sum(), -r[r <= 0].sum() + wk = np.array([(d - day0).days // 7 for d in ds.day[tr]]) + tot = np.bincount(wk - wk.min(), weights=r) if len(wk) else np.array([]) + has = np.bincount(wk - wk.min()) > 0 + weeks = int(has.sum()); pos = int(((tot > 0) & has).sum()) + p = float(binom.sf(pos - 1, weeks, 0.5)) if weeks else 1.0 # P(X ≥ pos) under no edge + return Score(n=len(r), win=float((r > 0).mean()), pf=float(g / l) if l > 0 else float("inf"), total=float(r.sum()), mean=float(r.mean()), + weeks=weeks, weeks_pos=pos, sign_ok=p <= SIGN_ALPHA) + + +def score_pick(ds: DecisionSet, pick: np.ndarray, hold_s, returns: np.ndarray, day0) -> Score: + return score_trades(ds, taken_idx(ds.ts, ds.mint, hold_s, np.flatnonzero(pick)), returns, day0) + + +def better(a: Score, b: Score | None) -> bool: + """a beats b under the objective: admissible first, then the fallback conditions, then total profit.""" + if b is None: + return a.base_ok + ka = (a.admissible, a.base_ok, a.total); kb = (b.admissible, b.base_ok, b.total) + return ka > kb + + +def passes(s: Score, fallback: bool) -> bool: + return s.admissible or (fallback and s.base_ok) + + +def beats_random_removal(ds: DecisionSet, pick_before: np.ndarray, pick_after: np.ndarray, hold_s, returns, day0, + hold_before=None, returns_before=None) -> tuple[bool, dict]: + """A filter must beat dropping the same number of trades at random from the unfiltered trades. Each book is scored + with its own holds and returns: the filtered book's arrays hold nothing for a row the filter removed — a zero there + would let a token re-enter within its own hold, and its return would count as zero profit rather than what it made.""" + hb = hold_s if hold_before is None else hold_before + rb = returns if returns_before is None else returns_before + t0 = taken_idx(ds.ts, ds.mint, hb, np.flatnonzero(pick_before)); t1 = taken_idx(ds.ts, ds.mint, hold_s, np.flatnonzero(pick_after)) + r0 = np.nan_to_num(np.asarray(rb[t0], dtype=np.float64)); after = float(np.nansum(returns[t1])) + k = max(0, len(t0) - len(t1)) + if k == 0 or len(t0) == 0: + return after >= float(r0.sum()), {"filtered_total": after, "random_total": float(r0.sum()), "removed": 0} + rnd = [float(np.delete(r0, np.random.default_rng(s).choice(len(r0), k, replace=False)).sum()) for s in range(RANDOM_SEEDS)] + return after > float(np.mean(rnd)), {"filtered_total": after, "random_total": float(np.mean(rnd)), "removed": k} + + +# ---------------------------------------------------------------- walk-forward primitives +def _cache_path(ds: DecisionSet, cols: list[str], tag: str) -> "Path": + from .selector import DATA_VERSION + k = repr((str(ds.days[0]), str(ds.days[-1]), int(len(ds.y)), float(ds.horizon_s), list(cols), tag, + DATA_VERSION.get("agg"), DATA_VERSION.get("features"), DATA_VERSION.get("costs"), OBJECTIVE)) + return CACHE_DIR / f"{hashlib.sha256(k.encode()).hexdigest()[:20]}.npy" + + +def _cached(ds: DecisionSet, cols: list[str], tag: str, stop, compute): + """One strategy-hold walk-forward is ~1.5 h of the fit; keep its out-of-sample scores on disk, keyed by the corpus, + the inputs and the hold, so a fit that is interrupted resumes in minutes instead of starting again. A run that was + stopped part-way is never written.""" + f = _cache_path(ds, cols, tag) + if f.exists(): + try: + v = np.load(f) + if len(v) == len(ds.y): + log.info("walk-forward %s: reusing cached scores", tag) + return v + log.warning("cached walk-forward %s has %d rows, not %d; recomputing", tag, len(v), len(ds.y)) + except (OSError, ValueError): + log.warning("cached walk-forward %s unreadable; recomputing", tag) + v = compute() + if stop is not None and stop.is_set(): + return v # partial: not worth keeping + CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = f.with_suffix(".tmp.npy"); np.save(tmp, v); tmp.replace(f) + return v + + + +def wf_blocks(days: list) -> list[list]: + from .selector import BLOCK_DAYS, WARMUP_DAYS + return [days[k:k + BLOCK_DAYS] for k in range(WARMUP_DAYS, len(days), BLOCK_DAYS)] + + +def fit_regressor(X: np.ndarray, y: np.ndarray, seed: int): + scaler = RobustScaler.fit(X, seed=seed) + m = HistGradientBoostingRegressor(**_gbm_params(), random_state=seed); m.fit(scaler.transform(X), np.clip(y, -1.0, 1.0)) + return m, scaler + + +def fit_classifier(X: np.ndarray, y: np.ndarray, seed: int): + scaler = RobustScaler.fit(X, seed=seed) + m = HistGradientBoostingClassifier(**_gbm_params(), random_state=seed); m.fit(scaler.transform(X), y) + return m, scaler + + +def wf_regress(ds: DecisionSet, rows: np.ndarray, y: np.ndarray, cols: np.ndarray, stop: threading.Event | None = None, label: str = "") -> np.ndarray: + """Out-of-sample predictions of y on ``rows`` (NaN elsewhere): each 7-day block by a model fit on rows at least a day earlier.""" + out = np.full(len(ds.y), np.nan, np.float32); fin = rows & np.isfinite(y); blocks = wf_blocks(ds.days) + for k, blk in enumerate(blocks): + if stop is not None and stop.is_set(): + break + tr = np.flatnonzero(fin & (ds.day < (min(blk) - timedelta(days=1)))); te = np.flatnonzero(rows & np.isin(ds.day, blk)) + if len(tr) < 10 * _gbm_params()["min_samples_leaf"] or len(te) == 0: + continue + Xtr = ds.X[np.ix_(tr, cols)] # this block's own copy: scaled in place, freed before the next + sc = RobustScaler.fit(Xtr, seed=k) + m = HistGradientBoostingRegressor(**_gbm_params(), random_state=k); m.fit(sc.transform_into(Xtr), np.clip(y[tr], -1.0, 1.0)) + del Xtr + for i in range(0, len(te), PREDICT_CHUNK): + ci = te[i:i + PREDICT_CHUNK] + out[ci] = m.predict(sc.transform_into(ds.X[np.ix_(ci, cols)])) + prog.update(f"selector: {label} walk-forward", k + 1, len(blocks), force=True) + return out + + +def wf_classify(ds: DecisionSet, rows: np.ndarray, feats, y: np.ndarray, n_classes: int = 2, stop: threading.Event | None = None) -> np.ndarray: + """Out-of-sample class probabilities [N, n_classes] on ``rows`` (NaN elsewhere); ``feats(idx)`` builds the inputs.""" + out = np.full((len(ds.y), n_classes), np.nan, np.float32); blocks = wf_blocks(ds.days) + for k, blk in enumerate(blocks): + if stop is not None and stop.is_set(): + break + tr = np.flatnonzero(rows & (ds.day < (min(blk) - timedelta(days=1)))); te = np.flatnonzero(rows & np.isin(ds.day, blk)) + if len(tr) < 10 * _gbm_params()["min_samples_leaf"] or len(te) == 0 or len(np.unique(y[tr])) < 2: + continue + Xtr = np.asarray(feats(tr), dtype=np.float32) # as above: one block-sized array at a time + log.info("classifier block %d: train %d rows (%.2f GB), score %d rows", k, len(tr), Xtr.nbytes / 2**30, len(te)) + sc = RobustScaler.fit(Xtr, seed=k) + m = HistGradientBoostingClassifier(**_gbm_params(), random_state=k); m.fit(sc.transform_into(Xtr), y[tr]) + del Xtr + for i in range(0, len(te), PREDICT_CHUNK): + ci = te[i:i + PREDICT_CHUNK] + pr = m.predict_proba(sc.transform_into(np.asarray(feats(ci), dtype=np.float32))) + full = np.zeros((len(ci), n_classes), np.float32); full[:, m.classes_.astype(int)] = pr + out[ci] = full + return out + + +def quantile_grid(v: np.ndarray, qs=QUANTILES) -> list[float]: + v = np.asarray(v, dtype=np.float64); v = v[np.isfinite(v)] + return sorted({float(x) for x in np.quantile(v, qs)}) if len(v) else [] + + +# ---------------------------------------------------------------- strategies +def _col(ds_cols: list[str], X: np.ndarray, name: str) -> np.ndarray: + return X[:, ds_cols.index(name)] + + +STRATEGIES = { + "ev": {"holds": HOLDS_MIN, "vars": (), "highs": (None,)}, # the hold is fitted; EV_HOLD is only the default label horizon + "capitulation": {"holds": HOLDS_MIN, "vars": ("neg_ret_1m", "top_sell_share_1m", "buyers_ret"), "highs": (None,)}, + "breakout": {"holds": HOLDS_MIN, "vars": ("org_imb_5m", "buyers_slope", "log_org_vol_15m"), "highs": ("dd_1h", "dd_3h")}, +} + + +def base_mask(name: str, X: np.ndarray, cols: list[str], high: str | None = None) -> np.ndarray: + """The definitional candidates of a strategy (no fitted parameter).""" + if name == "ev": + return np.ones(len(X), bool) + if name == "capitulation": + return (_col(cols, X, "ret_1m") < 0) & (_col(cols, X, "insider_sell_share_1m") == 0) + if name == "breakout": + return (_col(cols, X, "org_imb_5m") > 0) & (_col(cols, X, "buyers_slope") > 0) & (_col(cols, X, high or "dd_1h") >= 0) + raise KeyError(name) + + +def trigger_value(name: str, var: str, X: np.ndarray, cols: list[str]) -> np.ndarray: + return -_col(cols, X, "ret_1m") if var == "neg_ret_1m" else _col(cols, X, var) + + +def trigger_mask(name: str, thr: dict, X: np.ndarray, cols: list[str]) -> np.ndarray: + m = np.ones(len(X), bool) + for var, t in thr.items(): + if t is not None and np.isfinite(t): + m &= trigger_value(name, var, X, cols) >= t + return m + + +@dataclass +class StrategyFit: + name: str + hold_min: int + line: float + thr: dict + high: str | None + cols: list[str] + oos: np.ndarray = field(repr=False, default=None) # OOS primary score (chosen hold), NaN off candidates + selection: dict = field(default_factory=dict) + evaluation: dict = field(default_factory=dict) + trials: int = 0 + best_seen: dict = field(default_factory=dict) # the most profitable setting tried, whether or not it cleared the bars + meta_cut: float | None = None + meta_p: np.ndarray = field(repr=False, default=None) + hours: list = field(default_factory=list) + regimes: list = field(default_factory=list) # blocked mkt_vol_1h decile bins [(lo, hi)] + passed: bool = False + reason: str = "" + + def pick(self, ds: DecisionSet, rows: np.ndarray, uni: np.ndarray) -> np.ndarray: + X, c = ds.X, ds.cols + p = rows & uni & base_mask(self.name, X, c, self.high) & trigger_mask(self.name, self.thr, X, c) & (self.oos >= self.line) + if self.meta_cut is not None and self.meta_p is not None: + p &= self.meta_p >= self.meta_cut + if self.hours: + hr = ((ds.ts // 3600) % 24).astype(int); p &= ~np.isin(hr, self.hours) + for lo, hi in self.regimes: + mv = _col(c, X, "mkt_vol_1h"); p &= ~((mv >= lo) & (mv < hi)) + return p + + +def fit_strategy(ds: DecisionSet, name: str, cols: list[str], uni: np.ndarray, sel: np.ndarray, day0, stop=None, holds=None, + cache_tag: str = "") -> StrategyFit | None: + """Coordinate ascent over (hold, line, trigger thresholds, high window) on the selection half; each hold's OOS scores + come from its own walk-forward (models fit on the strategy's candidates, label = net return over that hold). + ``holds``: fit these holds instead of the strategy's whole grid — one walk-forward per hold is the expensive part, so + the input-group comparison runs at a single hold and only the strategies themselves scan the grid. ``cache_tag`` + distinguishes fits whose inputs share their column names but hold different values (the wallet-skill candidates), which + would otherwise collide in the walk-forward cache.""" + spec = STRATEGIES[name]; X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in cols]) + oos = {} + loose = uni & np.zeros(len(ds.y), bool) + for high in spec["highs"]: + loose |= uni & base_mask(name, X, c, high) + for H in (holds or spec["holds"]): + y = ds.fwd_h[H] if H in ds.fwd_h else (ds.fwd_pess if H * 60 == ds.horizon_s else None) + if y is None: + continue + oos[H] = _cached(ds, cols, f"{name}|hold{H}{cache_tag}", stop, lambda H=H, y=y: wf_regress(ds, loose, y, ci, stop, label=f"{name} {H} min")) + if not oos: + return None + ret = lambda H: ds.fwd_h[H] if H in ds.fwd_h else ds.fwd_pess + best = None; best_p = None; trials = 0; seen = None + thr = {v: None for v in spec["vars"]}; high = spec["highs"][0]; H = next(iter(oos)) + g0 = quantile_grid(oos[H][sel & loose]); line = g0[0] if g0 else float("inf") # the loosest line until a trial passes + grids = {v: quantile_grid(trigger_value(name, v, X[sel & loose], c)) for v in spec["vars"]} + + def trial(H_, line_, thr_, high_): + nonlocal best, best_p, trials, seen + trials += 1 + pk = sel & uni & base_mask(name, X, c, high_) & trigger_mask(name, thr_, X, c) & (oos[H_] >= line_) + s = score_pick(ds, pk, H_ * 60.0, ret(H_), day0) + if s.n > 0 and (seen is None or s.total > seen.total): + seen = s # the most profitable setting that actually traded (an empty pick scores 0 and must not win this) + if better(s, best): + best = s; best_p = (H_, line_, dict(thr_), high_) + for _ in range(PASSES): + for H_ in oos: + for line_ in quantile_grid(oos[H_][sel & loose]): + trial(H_, line_, thr, high) + if best_p: + H, line, thr, high = best_p + for v in spec["vars"]: + for t in [None] + grids[v]: + trial(H, line, {**thr, v: t}, high) + if best_p: + H, line, thr, high = best_p + for high_ in spec["highs"]: + trial(H, line, thr, high_) + if best_p: + H, line, thr, high = best_p + if best_p is None: + return StrategyFit(name, H, float("inf"), thr, high, cols, oos=oos[H], trials=trials, best_seen=seen.dict() if seen else {}, + reason="no setting met PF and the trade count on the selection half") + H, line, thr, high = best_p + return StrategyFit(name, H, float(line), thr, high, cols, oos=oos[H], selection=best.dict(), trials=trials, best_seen=seen.dict() if seen else {}) + + +# ---------------------------------------------------------------- meta-label, veto, gates +def fit_meta(ds: DecisionSet, s: StrategyFit, uni: np.ndarray, sel: np.ndarray, day0, stop=None) -> tuple[np.ndarray, float | None, dict]: + """Nested: a classifier of P(net return > 0 over the strategy's hold) on its OOS candidates (primary score ≥ 0 and + the fitted trigger), walk-forward, the primary score among its inputs; the cut-off (deciles among the selection + half's candidates) jointly with the line.""" + X, c = ds.X, ds.cols; ci = np.asarray([c.index(x) for x in s.cols]); y_r = ds.fwd_h.get(s.hold_min, ds.fwd_pess) + rows = uni & base_mask(s.name, X, c, s.high) & trigger_mask(s.name, s.thr, X, c) & np.isfinite(s.oos) & (s.oos >= 0) & np.isfinite(y_r) + log.info("meta-label %s: %d candidate rows of %d, %d inputs, hold %d min", s.name, int(rows.sum()), len(ds.y), len(ci) + 1, s.hold_min) + feats = lambda idx: np.c_[X[np.ix_(idx, ci)], s.oos[idx]] + p = wf_classify(ds, rows, feats, (y_r > 0).astype(np.int8), 2, stop)[:, 1] + log.info("meta-label %s: classifiers done, %.1f GB resident; choosing the line and cut-off", s.name, _rss_gb()) + best = None; best_p = None; trials = 0; seen = None + lines = [s.line] + quantile_grid(s.oos[sel & rows]); cuts = quantile_grid(p[sel & rows]) + log.info("meta-label %s: %d lines x %d cut-offs to try", s.name, len(lines), len(cuts)) + for line in lines: + for cut in cuts: + trials += 1 + if trials % 25 == 0: + log.info("meta-label %s: trial %d/%d, %.1f GB resident", s.name, trials, len(lines) * len(cuts), _rss_gb()) + pk = sel & rows & (s.oos >= line) & (p >= cut) + sc = score_pick(ds, pk, s.hold_min * 60.0, y_r, day0) + if sc.n > 0 and (seen is None or sc.total > seen.total): + seen = sc # the most profitable cut-off that actually traded + if better(sc, best): + best = sc; best_p = (line, cut) + return p, (best_p[1] if best_p else None), {"line": best_p[0] if best_p else None, "selection": best.dict() if best else None, "trials": trials, + "best_seen": seen.dict() if seen else None} + + +def system_pick(ds: DecisionSet, fits: list[StrategyFit], rows: np.ndarray, uni: np.ndarray, combine: str, tables: dict | None = None): + """(pick, hold_s, returns, strategy index) of the combined book: where several strategies fire, ``combine`` decides.""" + N = len(ds.y); pick = np.zeros(N, bool); hold = np.full(N, np.nan); ret = np.full(N, np.nan, np.float32); who = np.full(N, -1) + key = np.full(N, -np.inf) + for k, f in enumerate(fits): + pk = f.pick(ds, rows, uni) + if combine == "kelly" and tables and tables.get(f.name): + v = np.array([((sizing.band_for(tables[f.name], m) or {}).get("kelly") or 0.0) for m in (f.oos[pk] - f.line)]) + kv = np.full(N, -np.inf); kv[pk] = v + 1e-9 * f.oos[pk] + else: + kv = np.where(pk, f.oos, -np.inf) + take = pk & (kv > key) + key = np.where(take, kv, key); who = np.where(take, k, who); pick |= pk + hold = np.where(take, f.hold_min * 60.0, hold); ret = np.where(take, ds.fwd_h.get(f.hold_min, ds.fwd_pess), ret) + return pick, np.where(np.isnan(hold), 0.0, hold), ret, who + + +def fit_veto(ds: DecisionSet, pick: np.ndarray, hold: np.ndarray, ret: np.ndarray, sel: np.ndarray, day0, cols: list[str], stop=None): + """Multiclass GBM of the loss tail on the book's candidate trades (nested walk-forward): P(return ≤ d) for dump levels d + at the selection half's loss-tail quantiles; the level and the cut-off are chosen jointly by the objective.""" + base_r = ret[sel & pick & np.isfinite(ret)] + if len(base_r) < 10 * _gbm_params()["min_samples_leaf"]: + return None + levels = sorted({float(x) for x in np.quantile(base_r, LOSS_QUANTILES)}) + y = np.searchsorted(levels, np.nan_to_num(ret, nan=0.0), side="left").astype(np.int8) # class k: levels[k-1] < r ≤ levels[k] + ci = np.asarray([ds.cols.index(x) for x in cols]) + rows = pick & np.isfinite(ret) + probs = wf_classify(ds, rows, lambda idx: ds.X[np.ix_(idx, ci)], y, len(levels) + 1, stop) + cum = np.cumsum(probs, axis=1) # P(r ≤ levels[k]) + best = None; best_p = None; trials = 0; seen = None + for k, d in enumerate(levels): + pd_ = cum[:, k] + for v in quantile_grid(pd_[sel & rows]): + trials += 1 + pk = sel & rows & ~(pd_ >= v) + sc = score_pick(ds, pk, hold, ret, day0) + if sc.n > 0 and (seen is None or sc.total > seen.total): + seen = sc # the most profitable level and cut-off that actually traded + if better(sc, best): + best = sc; best_p = (k, d, v) + if best_p is None: + return {"cut": None, "levels": levels, "best_seen": seen.dict() if seen else None, "trials": trials} + return {"level_index": best_p[0], "dump": best_p[1], "cut": best_p[2], "levels": levels, "p": cum[:, best_p[0]], "selection": best.dict(), "trials": trials, + "best_seen": seen.dict() if seen else None} + + +# ---------------------------------------------------------------- the whole stack +@dataclass +class Stack: + groups: list + cols: list + fits: list + combine: str + veto: dict | None + components: list + selection: dict + evaluation: dict + deployable: bool + fallback: bool + reason: str + tables: dict = field(default_factory=dict) + holdout_days: list = field(default_factory=list) # withheld from every fit; scored once afterwards by score_holdout + + +def _comp(name, passed, reason, sel=None, ev=None, params=None, trials=0, best=None) -> dict: + """One component's verdict. Logged as it is decided: the stack spends hours per component, and a run that dies must + leave behind what it had already settled.""" + c = {"name": name, "passed": bool(passed), "reason": reason, "selection": sel, "evaluation": ev, "params": params or {}, "trials": int(trials), + "best_seen": best} + log.info("component %s: %s (%s) | selection %s | evaluation %s | best reached %s | %d settings tried", + name, "kept" if c["passed"] else "dropped", reason, sel, ev, best, c["trials"]) + return c + + +def fit_stack(ds: DecisionSet, stop: threading.Event | None = None, holdout_days_n: int | None = None) -> Stack: + """Every component in order, each on top of the accepted ones; see the module docstring. ``holdout_days_n``: how many + of the last days to withhold from every fit (default ``HOLDOUT_DAYS``). A caller whose ``ds`` is already restricted to + days before some later date -- the fly's bootstrap teacher (train/fly_selector.py) -- passes 0: its out-of-sample + proof is the replay that follows, and withholding again would only cost it its most recent three weeks.""" + from .selector import deploy_decision, in_universe + uni = in_universe(ds.X, ds.cols) + n_hold = HOLDOUT_DAYS if holdout_days_n is None else int(holdout_days_n) + holdout_days = list(ds.days[-n_hold:]) if n_hold and len(ds.days) > n_hold + 4 else [] + fit_days = [d for d in ds.days if d not in set(holdout_days)] + tdays = sorted({d for blk in wf_blocks(fit_days) for d in blk}) + if len(tdays) < 4: + raise RuntimeError("too few walk-forward days for the strategy stack") + half = tdays[len(tdays) // 2]; day0 = tdays[0] + tested = np.isin(ds.day, tdays); sel = tested & (ds.day < half); ev = tested & (ds.day >= half) + log.info("fitting on %d days (%s..%s); holdout of %d days (%s..%s) is never fitted on", len(tdays), tdays[0], tdays[-1], + len(holdout_days), holdout_days[0] if holdout_days else "-", holdout_days[-1] if holdout_days else "-") + comps = [] + # 1. input groups, judged on the ev strategy: all groups at once; if that is rejected, each group on its own, accumulating + best = fit_strategy(ds, "ev", LEGACY_COLS, uni, sel, day0, stop, holds=(EV_HOLD,)) + groups, cols = [], list(LEGACY_COLS) + trial_sets = [list(GROUPS)] + [[g] for g in GROUPS] + + def _score_of(f): + return Score(**{k: v for k, v in f.selection.items() if k in Score.__dataclass_fields__}) if f is not None and f.selection else None + + for gs in trial_sets: + if stop is not None and stop.is_set(): + break + if gs != list(GROUPS) and set(groups) == set(GROUPS): + break + add = [g for g in gs if g not in groups] + if not add: + continue + gi = [ds.cols.index(x) for x in (x for g in add for x in GROUPS[g]) if x in ds.cols] + sample = np.flatnonzero(tested)[:200_000] + if gi and len(sample) and not (np.ptp(ds.X[np.ix_(sample, gi)], axis=0) > 0).any(): + comps.append(_comp(f"inputs: {'+'.join(add)}", False, "these inputs are empty in this corpus (every value identical), so there is nothing to learn from", + params={"groups": add})); continue + cand_cols = cols + [x for g in add for x in GROUPS[g]] + f = fit_strategy(ds, "ev", cand_cols, uni, sel, day0, stop, holds=(EV_HOLD,)) + if f is None: + continue + sel_s = _score_of(f) or Score() + if better(sel_s, _score_of(best)): + evs = score_pick(ds, f.pick(ds, ev, uni), f.hold_min * 60.0, ds.fwd_h.get(f.hold_min, ds.fwd_pess), day0) + ok = passes(evs, fallback=not sel_s.admissible) + comps.append(_comp(f"inputs: {'+'.join(add)}", ok, "improves the ev strategy on the selection half" + ("" if ok else ", fails on the evaluation half"), + f.selection, evs.dict(), {"groups": add}, f.trials, best=f.best_seen)) + if ok: + groups, cols, best = groups + add, cand_cols, f + else: + comps.append(_comp(f"inputs: {'+'.join(add)}", False, "does not improve the ev strategy on the selection half", f.selection, None, {"groups": add}, f.trials, + best=f.best_seen)) + # 2. strategies + fits = [] + for name in STRATEGIES: + log.info("fitting strategy %s over holds %s", name, STRATEGIES[name]["holds"]) + f = fit_strategy(ds, name, cols, uni, sel, day0, stop) # the full hold grid: the group step above fitted one hold only + if f is None or not f.selection: + comps.append(_comp(f"strategy: {name}", False, (f.reason if f else "no candidates") or "no admissible setting", trials=f.trials if f else 0, + best=f.best_seen if f else None)); continue + y_r = ds.fwd_h.get(f.hold_min, ds.fwd_pess) + evs = score_pick(ds, f.pick(ds, ev, uni), f.hold_min * 60.0, y_r, day0) + sel_adm = f.selection.get("admissible") + f.evaluation = evs.dict(); f.passed = passes(evs, fallback=not sel_adm) + f.reason = ("meets ≥65 % winners and PF ≥1.3 on both halves" if sel_adm and evs.admissible else + "profit fallback: below 65 % winners, PF ≥1.3 on both halves" if f.passed else "fails on the evaluation half") + comps.append(_comp(f"strategy: {name}", f.passed, f.reason, f.selection, f.evaluation, {"hold_min": f.hold_min, "line": f.line, "thr": f.thr, "high": f.high}, f.trials, + best=f.best_seen)) + if f.passed: + fits.append(f) + if not fits: + return Stack(groups, cols, [], "score", None, comps, {}, {}, False, False, "no strategy passed its walk-forward test") + # sizing tables from each strategy's out-of-sample trades (selection half: used to choose the combination rule) + tables = {} + for f in fits: + y_r = ds.fwd_h.get(f.hold_min, ds.fwd_pess); t = taken_idx(ds.ts, ds.mint, f.hold_min * 60.0, np.flatnonzero(f.pick(ds, sel, uni))) + tables[f.name] = sizing.build_table(f.oos[t] - f.line, y_r[t]) + # 3. combination rule + best_c = None; combine = "score" + for rule in ("kelly", "score"): + pk, hd, rt, _ = system_pick(ds, fits, sel, uni, rule, tables) + s = score_pick(ds, pk, hd, rt, day0) + if better(s, best_c): + best_c = s; combine = rule + comps.append(_comp("combination", True, f"'{combine}' made more on the selection half", best_c.dict() if best_c else None, None, {"rule": combine})) + # 4. meta-label per strategy + for f in fits: + p, cut, info = fit_meta(ds, f, uni, sel, day0, stop) + if cut is None: + comps.append(_comp(f"meta-label: {f.name}", False, "no cut-off cleared the bars on the selection half", trials=info["trials"], best=info.get("best_seen"))); continue + log.info("meta-label %s: scoring the book before the filter (%.1f GB)", f.name, _rss_gb()) + before_s = score_pick(ds, *system_pick(ds, fits, sel, uni, combine, tables)[:3], day0) + old = (f.meta_p, f.meta_cut, f.line); f.meta_p, f.meta_cut = p, cut; f.line = info["line"] if info["line"] is not None else f.line + log.info("meta-label %s: scoring the book after the filter (%.1f GB)", f.name, _rss_gb()) + after_sel = score_pick(ds, *system_pick(ds, fits, sel, uni, combine, tables)[:3], day0) + log.info("meta-label %s: evaluation half, unfiltered book (%.1f GB)", f.name, _rss_gb()) + pk0, hd0, rt0, _ = system_pick(ds, [*(g for g in fits if g is not f), _without_meta(f, old)], ev, uni, combine, tables) + log.info("meta-label %s: evaluation half, filtered book (%.1f GB)", f.name, _rss_gb()) + pk1, hd1, rt1, _ = system_pick(ds, fits, ev, uni, combine, tables) + log.info("meta-label %s: scoring the evaluation half (%.1f GB)", f.name, _rss_gb()) + ev_s = score_pick(ds, pk1, hd1, rt1, day0) + log.info("meta-label %s: random-removal test (%.1f GB)", f.name, _rss_gb()) + rnd_ok, rnd = beats_random_removal(ds, pk0, pk1, hd1, rt1, day0, hold_before=hd0, returns_before=rt0) + ok = better(after_sel, before_s) and passes(ev_s, fallback=not after_sel.admissible) and rnd_ok + comps.append(_comp(f"meta-label: {f.name}", ok, "improves the book on the selection half, passes on the evaluation half and beats random removal" if ok + else "rejected (selection, evaluation or random-removal test)", after_sel.dict(), {**ev_s.dict(), "random_removal": rnd}, {"cut": cut, "line": f.line}, + info["trials"], best=info.get("best_seen"))) + if not ok: + f.meta_p, f.meta_cut, f.line = old + # 5. veto + veto = None + pk, hd, rt, who = system_pick(ds, fits, tested, uni, combine, tables) + v = fit_veto(ds, pk, hd, rt, sel, day0, cols, stop) + if v is not None and v.get("cut") is not None: + before_sel = score_pick(ds, pk & sel, hd, rt, day0); after_sel = score_pick(ds, pk & sel & ~(v["p"] >= v["cut"]), hd, rt, day0) + ev_before = pk & ev; ev_after = ev_before & ~(v["p"] >= v["cut"]) + ev_s = score_pick(ds, ev_after, hd, rt, day0); rnd_ok, rnd = beats_random_removal(ds, ev_before, ev_after, hd, rt, day0) + ok = better(after_sel, before_sel) and passes(ev_s, fallback=not after_sel.admissible) and rnd_ok + comps.append(_comp("dump veto", ok, "improves the book on the selection half, passes on the evaluation half and beats random removal" if ok else "rejected", + after_sel.dict(), {**ev_s.dict(), "random_removal": rnd}, {"dump": v["dump"], "cut": v["cut"]}, v["trials"], best=v.get("best_seen"))) + if ok: + veto = v + elif v is None: + comps.append(_comp("dump veto", False, "too few candidate trades to fit it")) + else: + comps.append(_comp("dump veto", False, "no dump level and cut-off cleared the bars on the selection half", trials=v["trials"], best=v.get("best_seen"))) + # (hour and market-volume gates were removed on 2026-09-20: fitted on the selection half they blocked 8 of 24 hours + # and 2 volume deciles, cutting the book from ~27 trades a day to under 1 on days nothing was fitted on. Strategies + # still carry empty ``hours``/``regimes`` so models saved with gates keep working.) + keep = ~(veto["p"] >= veto["cut"]) if veto else np.ones(len(ds.y), bool) + # the whole book + pk, hd, rt, _ = system_pick(ds, fits, tested, uni, combine, tables); pk &= keep + s_sel = score_pick(ds, pk & sel, hd, rt, day0); s_ev = score_pick(ds, pk & ev, hd, rt, day0) + tr_ev = taken_idx(ds.ts, ds.mint, hd, np.flatnonzero(pk & ev)) + from .decisions import random_trades, summarize + rnd = summarize(random_trades(ds, ev & uni, max(1, len(tr_ev)))) + dep, why = deploy_decision({"n": s_ev.n, "mean": s_ev.mean}, rnd) + strict = s_sel.admissible and s_ev.admissible + fb = (not strict) and s_sel.base_ok and s_ev.base_ok + deployable = dep and (strict or fb) + reason = (why + ("; meets ≥65 % winners and PF ≥1.3 on both halves" if strict else "; profit fallback (below 65 % winners)" if fb else "; fails the stack's bars")) + return Stack(groups, cols, fits, combine, veto, comps, s_sel.dict(), {**s_ev.dict(), "random_mean": rnd.get("mean")}, deployable, fb and deployable, reason, tables, + holdout_days=holdout_days) + + +def _without_meta(f: StrategyFit, old) -> StrategyFit: + import copy + g = copy.copy(f); g.meta_p, g.meta_cut, g.line = old + return g + + +def score_holdout(ds: DecisionSet, models: dict, days: list) -> dict: + """The deployed decision (``decide``: strategies, win filters, dump veto and gates as they would trade) scored on the + days ``fit_stack`` withheld from every fit. It decides nothing — recorded and reported only. Feeding it back into any + choice would make it a second evaluation half rather than a holdout.""" + from .decisions import random_trades, summarize + from .selector import in_universe + if not days or not (models.get("strategies") or {}): + return {} + eligible = np.isin(ds.day, days) & in_universe(ds.X, ds.cols) + rows = np.flatnonzero(eligible) + out = {"days": [str(days[0]), str(days[-1])], "rows": int(len(rows))} + if not len(rows): + return out + d = decide(models, ds.X[rows], ds.cols, ds.ts[rows]) + take = d["allow"] + if not take.any(): + return {**out, "n": 0} + idx = rows[take]; holds = d["hold_s"][take] + hold_full = np.zeros(len(ds.y)); hold_full[idx] = holds + ret_full = np.full(len(ds.y), np.nan, np.float32) + for hm in {int(h // 60) for h in holds}: + r = idx[(holds // 60).astype(int) == hm] + ret_full[r] = ds.fwd_h.get(hm, ds.fwd_pess)[r] + tr = taken_idx(ds.ts, ds.mint, hold_full, idx) + s = score_trades(ds, tr, ret_full, days[0]) + rnd = summarize(random_trades(ds, eligible, max(1, len(tr)))) + return {**out, **s.dict(), "random_mean": rnd.get("mean")} + + +# ---------------------------------------------------------------- deployment: final models and live decisions +def final_models(ds: DecisionSet, stack: Stack, exclude_days: list | None = None) -> dict: + """Every accepted component refit on all days (the walk-forward only chose their settings): per strategy its GBM over + its candidates and hold, its win classifier on all its out-of-sample candidates; the veto on all candidate trades. + ``exclude_days``: leave those days out of every fit, so the result can be scored on days it has never seen. Deployment + refits on everything; only the holdout measurement excludes, or it would be scoring itself in sample.""" + from .selector import in_universe + uni = in_universe(ds.X, ds.cols); out = {"strategies": {}, "veto": None, "combine": stack.combine} + keep = ~np.isin(ds.day, exclude_days) if exclude_days else np.ones(len(ds.y), bool) + for f in stack.fits: + ci = np.asarray([ds.cols.index(x) for x in f.cols]); y = ds.fwd_h.get(f.hold_min, ds.fwd_pess) + loose = np.zeros(len(ds.y), bool) + for high in STRATEGIES[f.name]["highs"]: + loose |= uni & base_mask(f.name, ds.X, ds.cols, high) + rows = np.flatnonzero(loose & np.isfinite(y) & keep) + gbm, scaler = fit_regressor(ds.X[np.ix_(rows, ci)], y[rows], seed=99) + meta = None + if f.meta_cut is not None: + mr = np.flatnonzero(uni & base_mask(f.name, ds.X, ds.cols, f.high) & trigger_mask(f.name, f.thr, ds.X, ds.cols) & np.isfinite(f.oos) & (f.oos >= 0) & np.isfinite(y) & keep) + mc, ms = fit_classifier(np.c_[ds.X[np.ix_(mr, ci)], f.oos[mr]], (y[mr] > 0).astype(np.int8), seed=99) + meta = {"model": mc, "scaler": ms, "cut": f.meta_cut} + out["strategies"][f.name] = {"gbm": gbm, "scaler": scaler, "cols": list(f.cols), "line": f.line, "hold_min": f.hold_min, "thr": dict(f.thr), "high": f.high, + "meta": meta, "hours": list(f.hours), "regimes": list(f.regimes), "sizing": stack.tables.get(f.name) or [], + "selection": f.selection, "evaluation": f.evaluation} + if stack.veto is not None: + pk, hd, rt, _ = system_pick(ds, stack.fits, np.ones(len(ds.y), bool), uni, stack.combine, stack.tables) + rows = np.flatnonzero(pk & np.isfinite(rt) & keep); ci = np.asarray([ds.cols.index(x) for x in stack.cols]) + y = np.searchsorted(stack.veto["levels"], rt[rows], side="left").astype(np.int8) + vm, vs = fit_classifier(ds.X[np.ix_(rows, ci)], y, seed=99) + out["veto"] = {"model": vm, "scaler": vs, "cols": list(stack.cols), "k": stack.veto["level_index"], "cut": stack.veto["cut"], "dump": stack.veto["dump"]} + return out + + +def decide_all(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: + """Every accepted strategy's own view of each row: ``trig`` (its fitted trigger fires), ``score`` (its predicted net + return over its hold, −inf off the trigger), ``allow`` (at/above its line and not blocked by its win filter, its + gates or the dump veto), ``why`` (the blocking filter), plus its line, hold and sizing table. ``t_start``: the minute + start (a scalar, or one per row) — the hour gates use it.""" + X = np.atleast_2d(X); n = len(X) + hour = ((np.broadcast_to(np.asarray(t_start, dtype=np.float64), (n,)) // 3600) % 24).astype(int) + out = {} + for name, m in (models.get("strategies") or {}).items(): + ci = np.asarray([cols.index(x) for x in m["cols"]]) + trig = base_mask(name, X, cols, m["high"]) & trigger_mask(name, m["thr"], X, cols) + sc = np.full(n, -np.inf); why = np.full(n, "", dtype=object) + if trig.any(): + ti = np.flatnonzero(trig); sc[ti] = m["gbm"].predict(m["scaler"].transform(X[np.ix_(ti, ci)])) + ok = trig & (sc >= m["line"]) + if m.get("meta") is not None and ok.any(): + idx = np.flatnonzero(ok) + p = m["meta"]["model"].predict_proba(m["meta"]["scaler"].transform(np.c_[X[np.ix_(idx, ci)], sc[idx]]))[:, list(m["meta"]["model"].classes_).index(1)] + low = np.zeros(n, bool); low[idx] = p < m["meta"]["cut"]; why[low] = "win filter (meta-label)" + gated = ok & np.isin(hour, m.get("hours", [])) & (why == "") + why[gated] = [f"hour {h} UTC gated" for h in hour[gated]] + if m.get("regimes") and "mkt_vol_1h" in cols: + mv = X[:, cols.index("mkt_vol_1h")] + for lo, hi in m["regimes"]: + why[ok & (mv >= lo) & (mv < hi) & (why == "")] = "market-volume regime gated" + out[name] = {"trig": trig, "score": sc, "allow": ok & (why == ""), "why": why, "line": m["line"], "hold_min": m["hold_min"], "sizing": m["sizing"]} + v = models.get("veto") + if v is not None and out: + any_ok = np.zeros(n, bool) + for d in out.values(): + any_ok |= d["allow"] + if any_ok.any(): + idx = np.flatnonzero(any_ok); ci = np.asarray([cols.index(x) for x in v["cols"]]) + pr = v["model"].predict_proba(v["scaler"].transform(X[np.ix_(idx, ci)])) + full = np.zeros((len(idx), int(max(v["model"].classes_)) + 1)); full[:, v["model"].classes_.astype(int)] = pr + bad = idx[np.cumsum(full, axis=1)[:, v["k"]] >= v["cut"]] + for d in out.values(): + hit = bad[d["allow"][bad]] + d["allow"][hit] = False; d["why"][hit] = f"dump veto (P(≤{v['dump'] * 100:.0f} %) ≥ {v['cut']:.2f})" + return out + + +def decide(models: dict, X: np.ndarray, cols: list[str], t_start) -> dict: + """The selector's decision for each row of one minute (``t_start`` = the minute's start): which strategy trades it, + its predicted net return, hold, line and sizing table, and whether a filter blocks it (with the reason).""" + X = np.atleast_2d(X); n = len(X); per = decide_all(models, X, cols, t_start) + best_key = np.full(n, -np.inf); strat = np.full(n, None, dtype=object); score = np.full(n, -1.0); hold = np.zeros(n) + thr = np.full(n, np.inf); tables = [[] for _ in range(n)]; allow = np.zeros(n, bool); reason = np.full(n, "no strategy fires", dtype=object) + for name, d in per.items(): + sc = d["score"]; margin = sc - d["line"] + key = (np.array([((sizing.band_for(d["sizing"], v) or {}).get("kelly") or 0.0) if np.isfinite(v) else -np.inf for v in margin]) + 1e-9 * sc + if models.get("combine") == "kelly" and d["sizing"] else sc) + take = d["trig"] & (sc >= d["line"]) & (key > best_key) + best_key = np.where(take, key, best_key) + for i in np.flatnonzero(take): + strat[i] = name; score[i] = sc[i]; hold[i] = d["hold_min"] * 60.0; thr[i] = d["line"]; tables[i] = d["sizing"] + allow[i] = bool(d["allow"][i]); reason[i] = d["why"][i] or "trade" + return {"strategy": strat, "score": score, "hold_s": hold, "threshold": thr, "tables": tables, "allow": allow, "reason": reason} diff --git a/fly_trader/train/wallet_skill.py b/fly_trader/train/wallet_skill.py new file mode 100644 index 0000000000000000000000000000000000000000..7561d4ed1a67ce934a2172996d532f9405d42046 --- /dev/null +++ b/fly_trader/train/wallet_skill.py @@ -0,0 +1,312 @@ +"""Wallet skill: how well each wallet's past PumpSwap trades turned out, as a daily decile table the archive +aggregation (``mature.aggregate_day``) and the live stream (``pumpstream.load_skill``) both join, so the selector's +``skill_top*_15m`` inputs (``train/flow.py``: the share of the last 15 minutes' buying from wallets in the top deciles) +are the same in training and live. Copying wallets does not work (2026-09-13); skill is used as an input only. + +``build_wallet_day(D)``: for every universe leg of day D (SOL-quoted PumpSwap legs of pump.fun-origin mints) and every +hold h of ``HOLDS_MIN``, the leg's markout — a buy: the net return of holding it h minutes (the close of the last traded +minute at or before t + h over the leg's price, after the one-side exit cost at both ends, ``exit_cost_0p1``); a sell: +minus the gross move it stepped aside from — summed per wallet, SOL-weighted, clipped to ±100 % like the selector's label +→ ``data/corpus/wallet_day/.parquet`` (wallet, h, n, sol, sol_m). Needs D's and D+1's aggregates and feature parts. + +``skill_table(D, h, L)``: the L wallet days ending ``GAP_DAYS`` before D (the exits of those days' holds are known once +the following day is aggregated, so the table for D exists before D begins — live never waits for it), score = +Σ sol·m / (Σ sol + k0) with k0 the median Σ sol of the window's wallets (empirical-Bayes shrinkage: a wallet with little +history is pulled toward 0), wallets cut into score deciles → ``wallet_skill/.parquet`` (wallet, bucket 0–9). +The (h, L) in force is fitted by the selector's profit objective (``train/selector.py``) and frozen in +``wallet_skill/config.json``; ``skill_version()`` names it (part of the selector's data version and of every aggregate). +""" +from __future__ import annotations + +import json +import logging +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import duckdb +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from .. import config +from . import flow, mature + +log = logging.getLogger(__name__) +HOLDS_MIN = (10, 20, 30, 45, 60, 90, 120, 180, 240) +LOOKBACKS_D = (1, 3, 7, 14, 30) +GAP_DAYS = 3 +DAY_DIR = config.CORPUS_DIR / "wallet_day" +CONFIG_FILE = mature.SKILL_DIR / "config.json" +FIT_FILE = mature.SKILL_DIR / "fit.json" + + +def skill_config() -> dict | None: + try: + return json.loads(CONFIG_FILE.read_text()) + except (FileNotFoundError, ValueError): + return None + + +def skill_version() -> str: + c = skill_config() + return f"h{c['h']}-L{c['L']}" if c else "none" + + +def _day_inputs(d: date) -> tuple[list[str], list[str], list[str]] | None: + nxt = d + timedelta(days=1) + aggs = [mature.MATURE_DIR / f"{x.isoformat()}.parquet" for x in (d, nxt)] + parts = [mature.MATURE_FEAT_DIR / x.isoformat() / "part.parquet" for x in (d, nxt)] + trades = mature._hour_files(d) + if not trades or not all(p.exists() for p in aggs + parts): + return None + return trades, [str(p) for p in aggs], [str(p) for p in parts] + + +def build_wallet_day(d: date, out_dir: Path | None = None) -> int: + """Per-wallet markouts of day D for every hold (None inputs → 0 rows written).""" + t0 = time.time(); inp = _day_inputs(d) + if inp is None: + return 0 + trades, aggs, parts = inp + con = duckdb.connect() + con.execute("""CREATE TEMP TABLE legs AS SELECT mint, trader, ts, side, sol, price FROM read_parquet(?, union_by_name = true) + WHERE pool = 'pump-amm' AND mint LIKE '%pump' AND price > 0 AND sol > 0 AND trader IS NOT NULL""", [trades]) + con.execute("CREATE TEMP TABLE closes AS SELECT mint, ts AS mts, close FROM read_parquet(?) ORDER BY mint, mts", [aggs]) + con.execute("CREATE TEMP TABLE ec AS SELECT mint, ts AS mts, exit_cost_0p1 AS ec FROM read_parquet(?)", [parts]) + out = [] + for h in HOLDS_MIN: + r = con.execute(f""" + SELECT l.trader AS wallet, {h} AS h, count(*) AS n, sum(l.sol) AS sol, + sum(l.sol * greatest(-1.0, least(1.0, CASE WHEN l.side = 1 + THEN c.close / l.price * (1 - coalesce(e0.ec, 0)) * (1 - coalesce(e1.ec, 0)) - 1 + ELSE -(c.close / l.price - 1) END))) AS sol_m + FROM legs l ASOF JOIN closes c ON c.mint = l.mint AND c.mts <= l.ts + INTERVAL {h} MINUTE + LEFT JOIN ec e0 ON e0.mint = l.mint AND e0.mts = time_bucket(INTERVAL 1 MINUTE, l.ts) + LEFT JOIN ec e1 ON e1.mint = c.mint AND e1.mts = c.mts + GROUP BY l.trader""").fetch_arrow_table() + out.append(r) + con.close() + tab = pa.concat_tables(out) + root = out_dir or DAY_DIR; root.mkdir(parents=True, exist_ok=True) + tmp = root / f"{d.isoformat()}.parquet.tmp"; pq.write_table(tab, tmp, compression="zstd"); tmp.replace(root / f"{d.isoformat()}.parquet") + log.info("wallet day %s: %d wallet-hold rows in %.0fs", d, tab.num_rows, time.time() - t0) + return tab.num_rows + + +def window_days(d: date, L: int) -> list[date]: + """The L wallet days a table for day D may use: D − GAP_DAYS − L + 1 .. D − GAP_DAYS.""" + end = d - timedelta(days=GAP_DAYS) + return [end - timedelta(days=k) for k in range(L - 1, -1, -1)] + + +def skill_table(d: date, h: int, L: int, day_dir: Path | None = None) -> pa.Table | None: + """wallet → decile 0–9 of shrunk skill over the window, or None when the window has no wallet day.""" + root = day_dir or DAY_DIR + files = [str(root / f"{x.isoformat()}.parquet") for x in window_days(d, L) if (root / f"{x.isoformat()}.parquet").exists()] + if not files: + return None + con = duckdb.connect() + w = con.execute(f"SELECT wallet, sum(sol) AS sol, sum(sol_m) AS sol_m FROM read_parquet(?) WHERE h = {int(h)} GROUP BY wallet HAVING sum(sol) > 0", [files]).df() + con.close() + if w.empty: + return None + k0 = float(np.median(w["sol"])); score = w["sol_m"] / (w["sol"] + k0) + rank = score.rank(method="first") - 1 + bucket = np.minimum((rank * 10 // len(w)).astype(int), 9).astype(np.int8) + return pa.table({"wallet": pa.array(w["wallet"].astype(str).tolist(), pa.string()), "bucket": pa.array(bucket.tolist(), pa.int8())}) + + +def write_table(d: date, h: int, L: int, out_dir: Path | None = None, day_dir: Path | None = None) -> bool: + t = skill_table(d, h, L, day_dir) + if t is None: + return False + root = out_dir or mature.SKILL_DIR; root.mkdir(parents=True, exist_ok=True) + t = t.replace_schema_metadata({b"fly_skill": f"h{int(h)}-L{int(L)}".encode()}) + tmp = root / f"{d.isoformat()}.parquet.tmp"; pq.write_table(t, tmp, compression="zstd"); tmp.replace(root / f"{d.isoformat()}.parquet") + return True + + +def freeze(h: int, L: int, fit: dict | None = None) -> None: + """Record the fitted (h, L); aggregates stamped with another skill version are rebuilt (train/mature.py).""" + mature.SKILL_DIR.mkdir(parents=True, exist_ok=True) + CONFIG_FILE.write_text(json.dumps({"h": int(h), "L": int(L), "fit": fit or {}, "frozen_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())})) + + +def table_version(path) -> str: + """The (h, L) a skill table was written with ('none': before tables were stamped).""" + return (pq.read_schema(path).metadata or {}).get(b"fly_skill", b"none").decode() + + +def build_wallet_days() -> int: + """Wallet days for every aggregated day that has none yet (a day whose next day is not aggregated waits).""" + n = 0 + for f in sorted(mature.MATURE_DIR.glob("*.parquet")): + d = date.fromisoformat(f.stem) + if not (DAY_DIR / f"{d.isoformat()}.parquet").exists() and build_wallet_day(d): + n += 1 + return n + + +def _window_ready(d: date, L: int) -> bool: + """Every wallet day of D's window exists (days before the first wallet day do not count: the corpus starts there).""" + have = sorted(DAY_DIR.glob("*.parquet")) if DAY_DIR.exists() else [] + if not have: + return False + first = date.fromisoformat(have[0].stem.split(".")[0]) + need = [x for x in window_days(d, L) if x >= first] + return bool(need) and all((DAY_DIR / f"{x.isoformat()}.parquet").exists() for x in need) + + +def daily() -> int: + """Run by mature's build loop: wallet days for the aggregated days; with a frozen (h, L), the table of that version for + every day from the first aggregate through tomorrow (UTC) once its whole window exists — so a day's table is there + before the day begins, and archive and live read the same file. A rewritten table re-aggregates its day.""" + n = build_wallet_days() + c = skill_config() + aggs = sorted(mature.MATURE_DIR.glob("*.parquet")) + if not c or not aggs: + return n + v = skill_version(); d = date.fromisoformat(aggs[0].stem); last = datetime.now(timezone.utc).date() + timedelta(days=1) + while d <= last: + f = mature.SKILL_DIR / f"{d.isoformat()}.parquet" + if (not f.exists() or table_version(f) != v) and _window_ready(d, c["L"]) and write_table(d, c["h"], c["L"]): + n += 1 + d += timedelta(days=1) + return n + + +def _lookups() -> tuple[list[str], dict]: + """The blocked pools and token insiders, read once: a fit aggregates 149 days per candidate, and reconnecting for each + one cost thousands of connections and lost a 20-hour run to a connection timeout.""" + from ..db.connection import transaction + from .corpus_meta import blocked_pool_ids + with transaction() as conn: + blocked = blocked_pool_ids(conn) + ins: dict = {} + for r in conn.execute("SELECT mint, wallet FROM token_insiders").fetchall(): + ins.setdefault(r["mint"], []).append(r["wallet"]) + log.info("wallet skill: %d blocked pools, insiders for %d tokens (read once)", len(blocked), len(ins)) + return blocked, ins + + +def _day_skill(d: date, h: int, L: int, day_dir: Path | None = None, lookups=None): + """(mint, minute start s, skill_buy [n, N_SKILL]) of day D's aggregate rows as the archive builds them under candidate + (h, L)'s table (``mature.aggregate_table``: the aggregation's own SQL); None without usable hour files.""" + files = mature._hour_files(d) + if not files or mature._lacks_pool_id(files): + return None + blocked, insiders = lookups if lookups is not None else (None, None) + tab = mature.aggregate_table(files, skill_table(d, h, L, day_dir), blocked=blocked, insiders=insiders) + sb = np.zeros((tab.num_rows, flow.N_SKILL)) + col = tab["skill_buy"].combine_chunks() + if col.null_count < len(col): + valid = col.is_valid().to_numpy(zero_copy_only=False) + sb[valid] = np.asarray(col.flatten().to_numpy(zero_copy_only=False), dtype=np.float64).reshape(-1, flow.N_SKILL) + return tab["mint"].to_numpy(zero_copy_only=False), mature._epoch_s(tab["ts"].to_pandas()), sb + + +def candidate_skill_cols(ds, h: int, L: int, day_dir: Path | None = None, stop=None) -> np.ndarray: + """[rows of ``ds``, SKILL_COLS]: the skill inputs every decision row would have had under candidate (h, L)'s tables — + each day re-aggregated with its candidate table, the previous day as the lookback (``mature.build_day``), FlowWindow's + 15-minute window (``flow.skill_features``).""" + from . import progress as prog + out = np.zeros((len(ds.y), len(flow.SKILL_COLS)), np.float32) + lookups = _lookups() + groups = pd.Series(np.arange(len(ds.y))).groupby(ds.day).indices + days = sorted(groups); prev_d, prev = None, None + for i, d in enumerate(days): + if stop is not None and stop.is_set(): + break + before = prev if prev_d == d - timedelta(days=1) else _day_skill(d - timedelta(days=1), h, L, day_dir, lookups) + cur = _day_skill(d, h, L, day_dir, lookups); prev_d, prev = d, cur + if cur is None: + continue + parts = [x for x in (before, cur) if x is not None] + m = np.concatenate([x[0] for x in parts]); t = np.concatenate([x[1] for x in parts]); sb = np.vstack([x[2] for x in parts]) + f = flow.skill_features(m, t, sb) + keep = np.flatnonzero(t >= datetime(d.year, d.month, d.day, tzinfo=timezone.utc).timestamp()) + idx = groups[d] + j = pd.DataFrame({"mint": ds.mint[idx], "ts": np.asarray(ds.ts[idx], dtype=np.float64), "i": idx}).merge( + pd.DataFrame({"mint": m[keep], "ts": np.asarray(t[keep], dtype=np.float64), "k": keep}), on=["mint", "ts"], how="left") + hit = j["k"].notna().to_numpy() + out[j["i"].to_numpy()[hit]] = f[j["k"].to_numpy()[hit].astype(np.int64)] + prog.update(f"wallet skill: inputs under h{h} L{L}", i + 1, len(days)) + return out + + +def _read_fit() -> dict: + try: + return json.loads(FIT_FILE.read_text()) + except (FileNotFoundError, ValueError): + return {} + + +def _write_fit(state: dict) -> None: + FIT_FILE.parent.mkdir(parents=True, exist_ok=True) + tmp = FIT_FILE.with_suffix(".tmp"); tmp.write_text(json.dumps(state, default=str)); tmp.replace(FIT_FILE) + + +def fit(days: int | None = None, stop=None) -> dict: + """The (h, L) the skill tables use: for every hold of ``HOLDS_MIN`` × lookback of ``LOOKBACKS_D``, the ev strategy fitted + on the legacy inputs + the skill inputs those tables give (``strategies.fit_strategy``: the selector's objective on the + selection half of the stack's walk-forward). The best by that objective is frozen; its evaluation half is read once, + for the record (the stack's input step judges the skill group again). Resumable: finished candidates are kept in + ``wallet_skill/fit.json``. Then the tables of that version are written (``daily``) and the build loop re-aggregates.""" + from . import progress as prog, strategies + from .decisions import LEGACY_COLS, build + from .selector import in_universe + prog.update("wallet skill: wallet days", 0, 1, force=True) + build_wallet_days() + ds = build(days=days, horizon_min=strategies.EV_HOLD, holds=strategies.HOLDS_MIN) # ev fits its hold, like the other strategies + ds.X = np.require(ds.X, requirements=["W"]) # pandas hands back a read-only array; the skill columns are rewritten per candidate + uni = in_universe(ds.X, ds.cols) + tdays = sorted({d for blk in strategies.wf_blocks(ds.days) for d in blk}) + half = tdays[len(tdays) // 2]; day0 = tdays[0] + tested = np.isin(ds.day, tdays); sel = tested & (ds.day < half); ev = tested & (ds.day >= half) + sk = [ds.cols.index(c) for c in flow.SKILL_COLS]; cols = list(LEGACY_COLS) + list(flow.SKILL_COLS) + key = f"{ds.days[0]}..{ds.days[-1]}|{strategies.OBJECTIVE}" # candidates fitted under another objective are re-run, not reused + state = _read_fit() + if state.get("key") != key: + state = {"key": key, "candidates": {}} + if "baseline" not in state: + b = strategies.fit_strategy(ds, "ev", list(LEGACY_COLS), uni, sel, day0, stop, holds=(strategies.EV_HOLD,)) + if stop is not None and stop.is_set(): + return {"stopped": True} + state["baseline"] = b.selection if b else None; _write_fit(state) + grid = [(h, L) for h in HOLDS_MIN for L in LOOKBACKS_D] + for i, (h, L) in enumerate(grid): + name = f"h{h}-L{L}" + if name in state["candidates"]: + continue + prog.update("wallet skill: fitting (hold, lookback)", i, len(grid), force=True, candidate=name) + ds.X[:, sk] = candidate_skill_cols(ds, h, L, stop=stop) + f = strategies.fit_strategy(ds, "ev", cols, uni, sel, day0, stop, holds=(strategies.EV_HOLD,), cache_tag=f"|{name}") + if stop is not None and stop.is_set(): + return {"stopped": True, "candidates": len(state["candidates"])} + state["candidates"][name] = {"h": h, "L": L, "selection": f.selection if f else None, "trials": f.trials if f else 0, + "best_seen": f.best_seen if f else None} + _write_fit(state) + log.info("wallet skill %s: selection %s", name, state["candidates"][name]["selection"]) + + def score(s): + return strategies.Score(**{k: v for k, v in s.items() if k in strategies.Score.__dataclass_fields__}) if s else None + best = None + for c in state["candidates"].values(): + sc = score(c["selection"]) + if sc is not None and strategies.better(sc, score(best["selection"]) if best else None): + best = c + if best is None: + raise RuntimeError("no (hold, lookback) gave the ev strategy a setting that passes the objective on the selection half") + ds.X[:, sk] = candidate_skill_cols(ds, best["h"], best["L"], stop=stop) + f = strategies.fit_strategy(ds, "ev", cols, uni, sel, day0, stop, holds=(strategies.EV_HOLD,), cache_tag=f"|h{best['h']}-L{best['L']}") + evs = strategies.score_pick(ds, f.pick(ds, ev, uni), f.hold_min * 60.0, ds.fwd_h.get(f.hold_min, ds.fwd_pess), day0) if f else strategies.Score() + rec = {"h": best["h"], "L": best["L"], "selection": best["selection"], "evaluation": evs.dict(), "baseline": state.get("baseline"), + "beats_baseline": bool(strategies.better(score(best["selection"]), score(state.get("baseline")))), "candidates": len(state["candidates"]), + "trials": int(sum(c["trials"] for c in state["candidates"].values())), "days": key} + freeze(best["h"], best["L"], rec) + rec["tables"] = daily() + log.info("wallet skill frozen at h%d L%d (beats the inputs without skill: %s); %d tables written", best["h"], best["L"], rec["beats_baseline"], rec["tables"]) + prog.update("wallet skill: done", 1, 1, force=True) + return rec diff --git a/fly_trader/ui/__init__.py b/fly_trader/ui/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/fly_trader/ui/__init__.py @@ -0,0 +1 @@ + diff --git a/fly_trader/ui/app.py b/fly_trader/ui/app.py new file mode 100644 index 0000000000000000000000000000000000000000..9e2718d259f78826f808465858eed971ef60ffc4 --- /dev/null +++ b/fly_trader/ui/app.py @@ -0,0 +1,46 @@ +"""fly-trader operator console (Streamlit): one place to see what the system is doing and to control each process. +Never runs trading logic; every control writes a Postgres row or signals a worker thread. Pages live in ``app_pages/``; +the status strip under the navigation answers, on every page: paper or live, trading or not, which model, training or +not, and whether the market feed is current.""" +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import streamlit as st + +st.set_page_config(page_title="fly-trader", page_icon="🪰", layout="wide") + +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import status_strip + + +def _autostart_once() -> None: + if get_supervisor().autostarted or st.session_state.get("_autostarted"): + return + st.session_state["_autostarted"] = True + try: + get_supervisor().autostart() + except Exception as e: # never block the page on a worker failing to start + st.warning(f"autostart: {e}") + + +_autostart_once() +pages = [ + st.Page("app_pages/overview.py", title="Overview", icon=":material/dashboard:", default=True), + st.Page("app_pages/trades.py", title="Trades", icon=":material/swap_horiz:"), + st.Page("app_pages/model.py", title="Model & training", icon=":material/psychology:"), + st.Page("app_pages/data.py", title="Data pipelines", icon=":material/database:"), + st.Page("app_pages/processes.py", title="Processes", icon=":material/settings_applications:"), + st.Page("app_pages/safety.py", title="Safety & wallet", icon=":material/shield:"), +] +page = st.navigation(pages, position="top") +status_strip() +try: + page.run() +except Exception as e: # keep the console alive on any query error + st.error(f"{type(e).__name__}: {e}") diff --git a/fly_trader/ui/app_pages/data.py b/fly_trader/ui/app_pages/data.py new file mode 100644 index 0000000000000000000000000000000000000000..df98abe2aa707f0c3bc8d9453ce1629d86f45e3b --- /dev/null +++ b/fly_trader/ui/app_pages/data.py @@ -0,0 +1,101 @@ +"""Data pipelines: is each source current, and how far along are the historical builds.""" +import glob +from datetime import datetime, timezone + +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q1 +from fly_trader.market.features import FEATURE_VERSION +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import ago, setting + + +def _state_badge(alive: bool, ok: bool = True) -> None: + st.badge("running" if alive and ok else "stale" if alive else "stopped", color="green" if alive and ok else "orange" if alive else "gray") + + +@st.cache_data(ttl=60, show_spinner=False) +def feature_parts() -> dict: + from fly_trader.train.mature import part_version + files = sorted(glob.glob(str(config.CORPUS_DIR / "features_mature" / "*" / "part.parquet"))) + days = [f.split("/")[-2] for f in files] + return {"n": len(files), "first": days[0] if days else None, "last": days[-1] if days else None, + "current": sum(1 for f in files if part_version(f) == FEATURE_VERSION)} + + +@st.fragment(run_every="10s") +def pipelines() -> None: + sup = get_supervisor(); ws = sup.status(); now = datetime.now(timezone.utc) + ps, ps_at = setting("pumpstream_status"); rs, rs_at = setting("replay_status") + a, b = st.columns(2) + with a: + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/sensors: **Market feed** (live)") + ft = ps.get("flushed_through"); age = (now - datetime.fromisoformat(ft)).total_seconds() - 60 if ft else None + _state_badge(ws["pumpstream"]["alive"], age is not None and age < 180) + with st.container(horizontal=True): + st.metric("Events / s", f"{float(ps.get('events_per_s') or 0):.0f}") + st.metric("Newest minute", f"{age:.0f} s old" if age is not None else "—", help="Seconds since the newest complete minute closed.") + st.metric("Graduations seen", int(ps.get("migrates") or 0)) + st.metric("Reconnects", int(ps.get("reconnects") or 0)) + st.caption(f"PumpSwap trades → 1-minute candles (pump_minutes) · dropped as outliers {int(ps.get('dropped_band') or 0):,} · late {int(ps.get('dropped_late') or 0):,} · updated {ago(ps_at)}") + with b: + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/history: **History archive** (training data)") + _state_badge(ws["replay"]["alive"]) + done, total = int(rs.get("hours_done") or 0), int(rs.get("hours_total") or 0) + days = q1("SELECT count(*) AS n, max(day) AS last FROM replay_days") or {} + with st.container(horizontal=True): + st.metric("Hours ingested", f"{done:,}/{total:,}" if total else f"{done:,}") + st.metric("Days assembled", int(days.get("n") or 0), delta=f"newest {days.get('last')}" if days.get("last") else None, delta_color="off") + st.metric("Errors", int(rs.get("errors") or 0)) + if total: + st.progress(min(1.0, done / total)) + st.caption(f"pumpapi.io hourly archive since {config.REPLAY_START}, newest first · last hour {str(rs.get('last_hour') or '—')[:13]}" + + (f" · about {float(rs['eta_h']):.1f} h left" if rs.get("eta_h") else "") + f" · updated {ago(rs_at)}") + a, b = st.columns(2) + with a: + with st.container(border=True): + st.markdown(":material/table_chart: **Training features**") + fp = feature_parts() + with st.container(horizontal=True): + st.metric("Days built", fp["n"]) + st.metric("Range", f"{fp['first'] or '—'} → {fp['last'] or '—'}") + st.metric("Current version", f"{fp['current']}/{fp['n']}", help="Days built with the live feature engine's current definitions; training refuses a mix.") + st.caption("One row per token-minute of every PumpSwap token, fed through the live feature engine. Built by the history archive worker.") + with b: + with st.container(border=True): + st.markdown(":material/badge: **Token metadata**") + m = q1("SELECT count(*) AS n, count(creator) AS creators, count(own_dd60) AS outcomes FROM corpus_meta") or {} + cr = q1("SELECT count(*) AS n FROM pump_events WHERE action = 'create'") or {} + with st.container(horizontal=True): + st.metric("Graduations", f"{int(m.get('n') or 0):,}") + st.metric("With creator", f"{int(m.get('creators') or 0):,}") + st.metric("Token launches", f"{int(cr.get('n') or 0):,}", help="Creates in pump_events, used for each creator's launch history.") + st.caption("Graduation time, creation facts and each creator's track record, shared by training and live scoring.") + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/query_stats: **Token stats** (Jupiter)") + _state_badge(ws["discover"]["alive"]) + cov = token_stats_coverage() + with st.container(horizontal=True): + st.metric("Tradable tokens with stats", f"{cov['covered']:,}/{cov['universe']:,}", + help="Tokens that traded in the last hour with a pool above the selector's gate, and how many have Jupiter stats from the last hour.") + st.metric("History since", str(cov["first"])[:10] if cov["first"] else "—") + st.caption("Holders, organic score and top-holder share, recorded every 10 minutes for every token the selector can trade. " + "Not a model input yet: Jupiter serves only current values, so these need weeks of recorded history first.") + + +@st.cache_data(ttl=60, show_spinner=False) +def token_stats_coverage() -> dict: + from fly_trader.train.decisions import MIN_RESQ_SOL + c = q1("""SELECT count(*) AS universe, count(*) FILTER (WHERE EXISTS (SELECT 1 FROM token_stats s WHERE s.mint = u.mint AND s.ts > now() - interval '1 hour')) AS covered + FROM (SELECT DISTINCT mint FROM pump_minutes WHERE ts > now() - interval '1 hour' AND resq_sol >= %s) u""", (MIN_RESQ_SOL,)) or {} + first = q1("SELECT min(ts) AS first FROM token_stats") or {} + return {"universe": int(c.get("universe") or 0), "covered": int(c.get("covered") or 0), "first": first.get("first")} + + +pipelines() diff --git a/fly_trader/ui/app_pages/model.py b/fly_trader/ui/app_pages/model.py new file mode 100644 index 0000000000000000000000000000000000000000..74d704889d4a5936cfeb866d1353f9fcc8cbe881 --- /dev/null +++ b/fly_trader/ui/app_pages/model.py @@ -0,0 +1,302 @@ +"""Model & training: which model trades and how it tested against random picks, the automatic retraining pipeline (the +selector every 7 days until the fly holds its seat; the fly bootstrapped once), the plastic fly, and every saved model.""" +import json +from datetime import datetime, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.agent import fly_session +from fly_trader.db.queries import q, q1 +from fly_trader.ops.reset import reset_fly +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.train import fly_selector, pipeline +from fly_trader.train.selector import is_current, is_deployable +from fly_trader.ui.common import ago, backtest_line, jv, latest_snapshot, parse_note, pct, system_state + + +def _pf(v) -> str: + return "∞" if v is None else f"{float(v):.2f}" + + +def _until(iso: str | None) -> str: + if not iso: + return "—" + s = (datetime.fromisoformat(iso) - datetime.now(timezone.utc)).total_seconds() + return "due now" if s <= 0 else f"in {s / 3600:.0f} h" if s < 172800 else f"in {s / 86400:.1f} days" + + +def sizing_panel(meta: dict) -> None: + """How big each buy is: the model's certainty bands from its backtest, and the backtest replayed with that sizing.""" + tbl = meta.get("sizing") or []; bk = meta.get("bankroll") or {} + st.markdown("**Position sizing**") + if not tbl: + st.caption(f"Fixed {config.MAX_POSITION_SOL:g} SOL per buy: this model was trained before sizing and has no certainty bands.") + return + st.caption(f"Each buy's size comes from how certain the model is. Its backtest trades are split into bands by how far the score cleared the buy line; " + f"each band gets {config.KELLY_FRACTION:g} × its growth-optimal share of the bankroll (wealth minus the {config.GAS_RESERVE_SOL:g} SOL gas reserve), " + f"at most {config.MAX_POSITION_FRACTION:.0%} of the bankroll and {config.MAX_POOL_SHARE:.0%} of the pool. Buys under {config.MIN_POSITION_SOL:g} SOL are skipped.") + st.dataframe(pd.DataFrame([{"score above the line": f"+{b['lo']:.3f} and up", "test trades": b["n"], "average per trade": (b["mean"] or 0) * 100, + "winning trades": (b["win"] or 0) * 100, "bet (share of bankroll)": min(config.KELLY_FRACTION * b["kelly"], config.MAX_POSITION_FRACTION) * 100} + for b in tbl]), hide_index=True, + column_config={"average per trade": st.column_config.NumberColumn(format="%+.2f%%"), "winning trades": st.column_config.NumberColumn(format="%.0f%%"), + "bet (share of bankroll)": st.column_config.NumberColumn(format="%.1f%%", help="0 % means that band has no edge: no buy.")}) + s, f = bk.get("sized") or {}, bk.get("fixed") or {} + if s and f: + st.caption(f"The backtest's trades replayed from {s.get('start_sol', 0):g} SOL: with this sizing → {s.get('final_sol', 0):.2f} SOL (worst drawdown {s.get('max_drawdown', 0):.0%}); " + f"at a fixed {config.MAX_POSITION_SOL:g} SOL → {f.get('final_sol', 0):.2f} SOL (worst drawdown {f.get('max_drawdown', 0):.0%}).") + + +def _best(b: dict | None) -> str: + """The most profitable setting a component tried, whether or not it cleared the bars — why a dropped one was dropped.""" + if not b: + return "—" + win = f"{b['win'] * 100:.0f}% winners" if b.get("win") is not None else "—" + pf = f"PF {b['pf']:.2f}" if b.get("pf") is not None else "PF ∞" + return f"{win} · {pf} · {b.get('n', 0)} trades · total {b.get('total', 0):+.1f}" + + +def stack_panel(meta: dict) -> None: + """The selector's strategy stack: every component tried (kept or dropped, and why) and the strategies it trades.""" + comps = meta.get("components") or [] + if not comps: + return + st.markdown("**Components**") + if meta.get("fallback"): + st.warning("No setting reached 65 % winning trades on both halves: this model trades the most profitable system that passes profit factor 1.3 and the deploy rule instead.", + icon=":material/warning:") + + def half(x: dict | None, k: str): + return None if not x or x.get(k) is None else float(x[k]) + st.dataframe(pd.DataFrame([{"component": c["name"], "kept": c["passed"], "why": c["reason"], + "winning (selection)": (half(c.get("selection"), "win") or 0) * 100 if half(c.get("selection"), "win") is not None else None, + "winning (evaluation)": (half(c.get("evaluation"), "win") or 0) * 100 if half(c.get("evaluation"), "win") is not None else None, + "profit factor (evaluation)": half(c.get("evaluation"), "pf"), "total (evaluation)": half(c.get("evaluation"), "total"), + "trades (evaluation)": half(c.get("evaluation"), "n"), "settings tried": c.get("trials"), "best reached": _best(c.get("best_seen")), + "parameters": json.dumps(c.get("params") or {}, default=str)} + for c in comps]), hide_index=True, + column_config={"winning (selection)": st.column_config.NumberColumn(format="%.0f%%"), "winning (evaluation)": st.column_config.NumberColumn(format="%.0f%%"), + "profit factor (evaluation)": st.column_config.NumberColumn(format="%.2f"), "total (evaluation)": st.column_config.NumberColumn(format="%+.2f", + help="Sum of the net returns of the evaluation half's trades (1.00 = one position's cost)."), + "best reached": st.column_config.TextColumn(help="The most profitable setting this component tried on the selection half, whether or not it cleared the bars.")}) + strat = meta.get("strategies") or {} + if strat: + st.markdown("**Strategies**") + st.dataframe(pd.DataFrame([{"strategy": k, "hold (min)": v.get("hold_min"), "buy line": (v.get("line") or 0) * 100, "trigger": json.dumps(v.get("thr") or {}, default=str), + "trades (evaluation)": half(v.get("evaluation"), "n"), "winning (evaluation)": (half(v.get("evaluation"), "win") or 0) * 100 if half(v.get("evaluation"), "win") is not None else None, + "profit factor (evaluation)": half(v.get("evaluation"), "pf")} for k, v in strat.items()]), hide_index=True, + column_config={"buy line": st.column_config.NumberColumn(format="%+.2f%%"), "winning (evaluation)": st.column_config.NumberColumn(format="%.0f%%"), + "profit factor (evaluation)": st.column_config.NumberColumn(format="%.2f")}) + + +@st.fragment(run_every="10s") +def loaded() -> None: + s = system_state(); m, latest = s["model"], s["latest"] + with st.container(border=True): + st.markdown("**Model at work**") + if m and not is_current(m["meta"]): + st.warning(f"Model #{m['id']} is in use but was trained on outdated data (before the data fixes of 2026-09-14). Its backtest is not valid and is hidden.", + icon=":material/warning:") + elif m: + meta = m["meta"]; wf = meta.get("walk_forward") or {}; rb = meta.get("random_baseline") or {}; run = m.get("run") or {} + st.caption(f"Model #{m['id']} · gradient-boosted selector · trained on {meta.get('first_day', '—')} → {meta.get('trained_through', '—')} · saved {ago(m['ts'])} · " + f"at work since {ago(m['since'])}") + with st.container(horizontal=True): + mi = meta.get("model") or {} + st.metric("Buys when predicted return ≥", pct(float(run.get('threshold') or meta.get('threshold') or 0), 1), + help=f"Each minute the model predicts every eligible token's net return over the hold after fees and price impact. It buys tokens at least " + f"{mi.get('min_age_h', '—')} h past graduation (or older than the archive) whose prediction clears this line.") + st.metric("Holds for", f"{run.get('horizon_min') or meta.get('horizon_min') or '—'} min", help="Each position is sold this long after the buy.") + st.metric("Test trades", wf.get("n", "—"), help=f"Trades on the backtest's evaluation half ({wf.get('days', '—')} days), each traded by a model that never saw " + "that day, at a buy line chosen on the earlier selection half.") + st.metric("Average per trade", pct(wf.get("mean")), delta=(f"random {pct(rb['mean'])}" if rb.get("mean") is not None else None), delta_color="off", + help="Net return per trade on the test days, after fees and price impact. 'random' is the same number of random buys under the same rules.") + st.metric("Winning trades", pct(wf.get("win"), 0, False), help="Share of test trades that made money.") + st.metric("Profit factor", _pf(wf.get("pf")), help="Total gains divided by total losses on the test trades. Above 1 makes money.") + st.metric("Profitable test days", f"{wf.get('days_positive', '—')} of {wf.get('days', '—')}", help="Test days whose average trade made money after costs.") + st.caption(backtest_line(meta)) + sizing_panel(meta) + stack_panel(meta) + elif not latest: + st.info("No model has been trained on the current data yet, so nothing is trading. The automatic retraining below runs as soon as the data is ready.", + icon=":material/info:") + else: + st.info(f"No model is at work. The latest, #{latest['id']}, was not put to work: {latest['meta'].get('deploy_reason') or 'it did not qualify'}.", + icon=":material/block:") + if m and latest and latest["id"] > m["id"]: + if is_deployable(latest["meta"]): + st.info(f"Model #{latest['id']} qualified and replaces #{m['id']} within 10 minutes, without a restart.", icon=":material/upgrade:") + else: + st.caption(f"Model #{latest['id']} ({ago(latest['ts'])}) was not put to work: {latest['meta'].get('deploy_reason') or 'it did not qualify'}.") + + +@st.fragment(run_every="3s") +def retraining() -> None: + s = system_state(); sup = get_supervisor(); tr = s["train_status"]; ps = pipeline.state(); alive = sup.alive("train") + running = alive and str(ps.get("stage", "")).startswith("training") + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(f":material/autorenew: **Automatic retraining** · every {pipeline.INTERVAL_DAYS} days" if not s["handover"] else ":material/autorenew: **Retraining** · on request only: the fly holds the seat") + st.space("stretch") + if st.button("Retrain now", icon=":material/play_arrow:", type="primary", key="retrain_now", disabled=running, + help="Runs the whole pipeline now; the 7-day schedule restarts from this run."): + pipeline.request_run() + if not alive: + try: + sup.start("train", started_by="console") + except RuntimeError as e: + st.error(str(e)) + st.rerun() + if running and st.button("Stop", icon=":material/stop:", key="train_stop", help="Stops the current run; the schedule resumes when the trainer is started again."): + sup.request_stop("train"); st.rerun() + st.caption("1 · The selector retrains on every day of history. Every day after the first 21 is backtested by a model that never saw it, against random picks, " + "with real fees and price impact. \n2 · It goes to work only if that backtest made money and beat random picks; the trading engine switches to it " + "without a restart. \n3 · The fly is taught by the selector once — when none exists for the current definitions, or on request — " + "and from then on learns from the market; weekly retraining stops when the fly takes the selector's seat.") + if not alive: + st.warning("Automatic retraining is off because the Trainer is not running. Click Retrain now, or start the Trainer on the Processes page.", icon=":material/warning:") + last = ps.get("last_run_at"); nxt = ps.get("next_run_at") + sel = latest_snapshot("selector") + with st.container(horizontal=True): + st.metric("Status", (ps.get("stage") or "idle").capitalize() if alive else "Off") + st.metric("Last run", ago(last) if last else "never") + st.metric("Next run", _until(nxt) if nxt else ("as soon as the data is ready" if alive else "—")) + st.metric("Latest selector", f"#{sel['id']}" if sel else "—", delta=("at work" if sel and is_deployable(sel["meta"]) else "not at work" if sel else None), + delta_color="off", help=backtest_line(sel["meta"]) if sel else None) + if running and tr.get("step") is not None and tr.get("total"): + st.progress(min(1.0, float(tr["step"]) / max(float(tr["total"]), 1.0)), text=f"{tr.get('stage', '')}" + (f" · about {float(tr['eta_s']) / 60:.0f} min left" if tr.get("eta_s") else "")) + if ps.get("stage") == "waiting for data" and ps.get("waiting"): + st.caption(f"Waiting for the training data: {ps['waiting']}. Checking again {_until(ps.get('retry_after'))}.") + if ps.get("stage") == "failed" and ps.get("last_error"): + st.error(f"The last run failed: {ps['last_error']}. Retrying {_until(ps.get('retry_after'))}.") + + +@st.fragment(run_every="15s") +def results() -> None: + wfd = q("SELECT detail FROM events WHERE source = 'selector' AND message LIKE 'walk-forward%%' ORDER BY id") + with st.container(border=True): + st.markdown("**Latest selector backtest, day by day**") + if wfd and not system_state()["latest"]: + st.caption("The last backtest ran on outdated data and is hidden.") + elif wfd: + df = pd.DataFrame([{"day": d.get("day"), "half": d.get("half"), "IC": d.get("ic"), "trades": d.get("n"), "model": (d["mean"] * 100) if d.get("mean") is not None else None, + "random": (d["random_mean"] * 100) if d.get("random_mean") is not None else None, "winners": (d["win"] * 100) if d.get("win") is not None else None, + "profit factor": d.get("pf")} for d in (jv(r["detail"]) for r in wfd)]) + st.bar_chart(df, x="day", y=[c for c in ("model", "random") if df[c].notna().any()], stack=False, x_label="", y_label="% per trade", height=220) + st.dataframe(df, hide_index=True, column_config={"IC": st.column_config.NumberColumn(format="%.3f", help="Rank correlation of the predictions with the realised returns (0 = no skill)."), + "half": st.column_config.TextColumn(help="selection: the days that chose the buy line; evaluation: the days that judge it."), + "model": st.column_config.NumberColumn("model % / trade", format="%+.2f"), + "random": st.column_config.NumberColumn("random % / trade", format="%+.2f", help="Random picks, same count, costs and fills."), + "winners": st.column_config.NumberColumn("winners %", format="%.0f"), + "profit factor": st.column_config.NumberColumn(format="%.2f")}) + st.caption("Each day is traded by a model trained only on days at least two days earlier, with the paper broker's fees and price impact. " + "The buy line is chosen on the selection half and judged on the evaluation half.") + else: + st.caption("No selector backtest on the current data yet.") + + +def _fly_bootstrap() -> dict | None: + for r in q("SELECT id, ts, note FROM brain_snapshots WHERE kind = 'fly_selector' ORDER BY id DESC LIMIT 10"): + m = parse_note(r["note"]) + if fly_selector.is_current(m): + return {**r, "meta": m} + return None + + +@st.fragment(run_every="15s") +def plastic_fly() -> None: + s = system_state(); fly = s["fly"]; rp = s["replay"]; sup = get_supervisor() + running = bool(fly.get("stage")) and fly.get("stage") != "not trading"; frozen = bool(fly.get("learning_frozen")) + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/neurology: **The plastic fly** · taught once by the selector, then learning from the market") + st.space("stretch") + if st.button("Resume learning" if frozen else "Pause learning", key="fly_learn", disabled=not running, + help="Pause: the fly keeps trading with what it has learned; its mushroom body stops changing."): + fly_session.send_command("resume" if frozen else "pause"); st.toast("Sent: applied at the next minute") + if st.button("Roll back", key="fly_rollback", disabled=not running, + help="Restore the newest good hourly snapshot at least a day old (or the bootstrap). Does not count toward the automatic re-bootstrap."): + fly_session.send_command("rollback"); st.toast("Sent: applied at the next minute") + if st.button("Re-bootstrap", key="fly_reboot", help="The selector retrains and teaches a new fly; it trades at once if it qualifies."): + pipeline.request_run(fly=True, reason="console"); st.toast("Re-bootstrap requested") + with st.popover("Reset fly", disabled=sup.alive("runner"), help="Stop the trading engine first." if sup.alive("runner") else None): + st.markdown("Archive and clear the fly's paper book and everything it learned? It restarts from its bootstrap and the race starts over.") + if st.button("Reset", type="primary", key="fly_reset_confirm"): + st.success(f"Archived to {reset_fly(reason='console button')['archived_to']}") + if not rp: + st.info("The replay has not run on the current definitions. It is the proof required before the fly trades: `fly-trader fly-replay` " + "bootstraps a fly, lets it learn over months of history without retraining, and judges it.", icon=":material/info:") + else: + ev, rnd, fr = rp.get("evaluation") or {}, rp.get("random") or {}, rp.get("frozen") or {} + (st.success if rp.get("passed") else st.warning)(f"Replay {'passed' if rp.get('passed') else 'failed'}: {rp.get('reason')}", + icon=":material/verified:" if rp.get("passed") else ":material/block:") + with st.container(horizontal=True): + st.metric("Replay trades", ev.get("n", "—"), border=True, help="Trades on the evaluation half of the replay.") + st.metric("Average per trade", pct(ev.get("mean")), delta=(f"random {pct(rnd.get('mean'))}" if rnd.get("mean") is not None else None), delta_color="off", border=True) + st.metric("Frozen fly", pct(fr.get("mean")), border=True, help="The same bootstrap without plasticity, on the same days.") + st.metric("Profitable days", f"{ev.get('days_positive', '—')} of {ev.get('days', '—')}", border=True) + st.metric("Learning rate α", f"{rp.get('alpha') or 0:g}", border=True) + st.metric("Forgetting half-life", f"{rp['half_life_days']:g} days" if rp.get("half_life_days") else "none", border=True) + st.caption(f"Bootstrapped for {rp.get('S')} and never retrained: every scored minute taught its mushroom body with the realized return two hours later. " + f"Configuration chosen on {' → '.join(rp.get('selection_days') or ['—'])}, judged on {' → '.join(rp.get('evaluation_days') or ['—'])}.") + if running: + ch = fly.get("checks") or {}; sh = ch.get("shadow") or {} + with st.container(horizontal=True): + st.metric("Buy line", pct(fly.get("line")), delta=f"shadow {pct(fly.get('frozen_line'))}", delta_color="off", + help="Recalibrated every day from its own scores and the realized returns of the last 7 days.") + st.metric("Drift", pct(fly.get("drift"), 1, False), help="Size of the learned KC→MBON change relative to the bootstrap weights; rolled back above 50 %.") + st.metric("Labels pending", fly.get("pending", "—"), help="Scored minutes waiting for their two-hour outcome.") + st.metric("24 h IC", f"{ch['ic']:.3f}" if ch.get("ic") is not None else "—", help="Rank correlation of its scores with the realized returns; rolled back below 0.") + st.metric("Picks vs shadow (3 d)", pct(sh.get("mean_plastic")), delta=f"shadow {pct(sh.get('mean_frozen'))}", delta_color="off", + help="Average realized return of its picks vs the frozen bootstrap's picks; rolled back when it trails by more than one standard error.") + st.metric("Learning", "frozen" if frozen else "on") + upd = q("SELECT hour, n, mean_abs_delta, drift, ic FROM fly_updates ORDER BY hour DESC LIMIT 72") + if upd: + st.line_chart(pd.DataFrame(upd), x="hour", y=["drift", "ic"], x_label="", height=200) + rb = q("SELECT ts, reason, from_snapshot, to_snapshot FROM fly_rollbacks ORDER BY id DESC LIMIT 20") + if rb: + with st.expander(f"Rollbacks ({len(rb)})", icon=":material/history:"): + st.dataframe(pd.DataFrame(rb), hide_index=True, column_config={"ts": st.column_config.DatetimeColumn("when", format="MMM D HH:mm")}) + else: + st.caption(f"Not trading: {fly.get('detail') or 'the trading engine is stopped'}.") + b = _fly_bootstrap() + if b: + d = b["meta"].get("diagnostics") or {}; ok, why = fly_selector.deployable(b["meta"]) + st.caption(f"Bootstrap #{b['id']} ({ago(b['ts'])}): the mushroom body carries {pct(d.get('mbon_share'), 0, False)} of the score, " + f"{pct(d.get('mbon_saturated'), 0, False)} of MBONs saturated, {pct(d.get('kc_active'), 0, False)} of Kenyon cells active, " + f"slope {d.get('slope') or 0:.2f} against its teacher. " + ("May trade: " if ok else "May not trade: ") + why + ".") + + +def history() -> None: + snaps = q("SELECT id, ts, kind, note FROM brain_snapshots WHERE kind IN ('selector', 'fly_selector') ORDER BY id DESC LIMIT 30") + s = system_state(); in_use = (s["model"] or {}).get("id") + with st.expander("Saved models", icon=":material/folder_open:"): + rows = [] + for r in snaps: + m = parse_note(r["note"]); wf = m.get("walk_forward") or m.get("fly") or {}; rb = m.get("random_baseline") or m.get("random") or {} + sel = r["kind"] == "selector"; ok = is_current(m) if sel else fly_selector.is_current(m) + if not sel: + wf, rb = m.get("calibration") or {}, {} + wf = {"n": wf.get("trades"), "mean": wf.get("mean")} + rows.append({"model": f"#{r['id']}", "type": "selector" if sel else "fly bootstrap", "saved": r["ts"], "data": "current" if ok else "outdated", + "trades": wf.get("n") if ok else None, "% / trade": (wf["mean"] * 100) if ok and wf.get("mean") is not None else None, + "random %": (rb["mean"] * 100) if ok and rb.get("mean") is not None else None, "profit factor": wf.get("pf") if ok else None, + "put to work": (bool(m.get("deployable")) if sel else fly_selector.deployable(m)[0]) if ok else None, "in use": r["id"] == in_use}) + if rows: + st.dataframe(pd.DataFrame(rows), hide_index=True, column_config={"saved": st.column_config.DatetimeColumn(format="MMM D HH:mm"), + "% / trade": st.column_config.NumberColumn(format="%+.2f"), + "random %": st.column_config.NumberColumn(format="%+.2f"), + "profit factor": st.column_config.NumberColumn(format="%.2f")}) + st.caption("Outdated models were trained before the data fixes of 2026-09-14; their numbers are not valid, are hidden, and they are never loaded.") + else: + st.caption("No saved models.") + with st.expander("Trainer log", icon=":material/terminal:"): + st.code(get_supervisor().log_tail("train", 40) or "(no log yet)", language="json") + + +loaded() +retraining() +results() +plastic_fly() +history() diff --git a/fly_trader/ui/app_pages/overview.py b/fly_trader/ui/app_pages/overview.py new file mode 100644 index 0000000000000000000000000000000000000000..6cd24f7b171b38002f8417423f54df0b51ea8602 --- /dev/null +++ b/fly_trader/ui/app_pages/overview.py @@ -0,0 +1,138 @@ +"""Overview: what the system is doing in plain words, the paper book, this minute's scoring and open positions.""" +from datetime import datetime, timedelta, timezone + +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q, q1 +from fly_trader.ui.common import BOOK, ago, backtest_line, labels, pct, sol, system_state + + +def narrative(s: dict) -> str: + m = s["model"]; lines = [] + if m: + run = m.get("run") or {}; meta = m["meta"] + money = "No real money is at risk." if not s["live_money"] else "Trades spend real SOL from the bot wallet." + lines.append(f"**{s['trading_why']}** The selector is paper trading with model **#{m['id']}** (trained through {meta.get('trained_through', '—')}). " + f"Every minute it predicts the net {meta.get('horizon_min', '—')}-minute return of each PumpSwap token that traded, at least {(meta.get('model') or {}).get('min_age_h', '—')} hours " + f"past graduation, with a pool of at least {(meta.get('model') or {}).get('min_resq_sol', '—')} SOL and {(meta.get('model') or {}).get('min_vol_15m_sol', '—')} SOL traded in the last 15 minutes, and buys those predicted to make at least " + f"**{pct(float(run.get('threshold') or meta.get('threshold') or 0), 1)}** — sized by how certain the " + f"model is (up to {config.MAX_POSITION_FRACTION:.0%} of the bankroll, never the {config.GAS_RESERVE_SOL:g} SOL gas reserve) — " + f"and sells it **{run.get('horizon_min') or meta.get('horizon_min') or 30} minutes** later. {money}") + lines.append(backtest_line(meta)) + fly = s["fly"] + if fly.get("stage") and fly.get("stage") != "not trading": + lines.append("**The plastic fly** " + ("holds the seat" if s["handover"] else "races the selector on its own paper book") + + f": it buys at a predicted {pct(fly.get('line'), 1)} and learns every minute from each scored token's realized return, two hours later.") + else: + lines.append(f"**{s['trading_why']}** Start the trading engine on the Processes page to trade.") + if s["training"]: + tr = s["train_status"]; step, total = tr.get("step"), tr.get("total") + lines.append(f"Training is running: {tr.get('stage', '')}" + (f" ({step}/{total})" if step is not None and total else "") + ".") + return "\n\n".join(lines) + + +@st.fragment(run_every="5s") +def summary() -> None: + s = system_state() + with st.container(border=True): + st.markdown(narrative(s)) + w = q1("SELECT wealth, exposure, n_open, drawdown FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 1", (BOOK,)) or {} + c = q1("SELECT count(*) AS n, sum(realized_sol) AS pnl, count(*) FILTER (WHERE realized_sol > 0) AS wins FROM positions WHERE book = %s AND status = 'closed'", (BOOK,)) or {} + spark = [float(r["wealth"]) for r in q("SELECT wealth FROM wealth_marks WHERE book = %s ORDER BY ts DESC LIMIT 180", (BOOK,))][::-1] + n = int(c.get("n") or 0) + with st.container(horizontal=True): + st.metric("Paper wealth", sol(w.get("wealth")), delta=(f"{float(w['wealth']) - config.CAPITAL_SOL:+.4f} SOL since start" if w.get("wealth") is not None else None), + border=True, chart_data=spark or None, help=f"Started at {config.CAPITAL_SOL:g} SOL; open positions valued net of exit costs.") + st.metric("Realized P&L", sol(c.get("pnl"), signed=True), border=True, help="Closed trades, after fees and price impact.") + st.metric("Closed trades", n, delta=(f"{int(c.get('wins') or 0) / n:.0%} winners" if n else None), delta_color="off", border=True) + st.metric("Open positions", int(w.get("n_open") or 0), delta=(f"{float(w.get('exposure') or 0):.2f} SOL at cost" if w else None), delta_color="off", border=True) + st.metric("Drawdown", pct(w.get("drawdown"), signed=False), border=True, help="Below the book's peak wealth.") + sel = s["selector"] + with st.container(border=True): + st.markdown("**This minute**") + if not sel.get("minute"): + st.caption("No minute scored yet." if s["runner"] else "The trading engine is stopped.") + else: + thr = float(sel.get("threshold") or 0) + with st.container(horizontal=True): + st.metric("Minute (UTC)", str(sel["minute"])[11:16]) + st.metric("Tokens traded", int(sel.get("mints_traded") or 0)) + st.metric("Eligible", int(sel.get("eligible") or 0), help="Pool ≥ 20 SOL and ≥ 5 SOL traded in the last 15 minutes.") + st.metric("Best score", f"{float(sel.get('score_max') or 0):.3f}", delta=f"needs {thr:.3f}", delta_color="off") + st.metric("Picks", int(sel.get("picks") or 0)) + st.metric("Bought / sold", f"{int(sel.get('entered') or 0)} / {int(sel.get('exited') or 0)}") + top = sel.get("top_scores") or [] + if top: + names = labels([t[0] for t in top]) + st.dataframe(pd.DataFrame({"token": [names.get(t[0], t[0][:6]) for t in top], "score": [float(t[1]) for t in top]}), hide_index=True, + column_config={"score": st.column_config.ProgressColumn("score", min_value=0.0, max_value=1.0, format="%.3f")}) + lat = sel.get("latency_s") + st.caption(f"Scored {ago(s['selector_at'])}" + (f" · {float(lat):.0f} s after the minute closed" if lat is not None else "") + + (f" · {s['trading_why']}" if s["trading"] != "trading" else "")) + + +@st.fragment(run_every="15s") +def book() -> None: + left, right = st.columns([3, 2]) + with left: + with st.container(border=True): + st.markdown("**Paper wealth**") + hist = q("SELECT ts, wealth FROM wealth_marks WHERE book = %s ORDER BY ts", (BOOK,)) + if hist: + st.line_chart(pd.DataFrame(hist), x="ts", y="wealth", x_label="", y_label="SOL", height=240) + else: + st.caption("No marks yet.") + with right: + with st.container(border=True): + st.markdown("**Open positions**") + rows = q("SELECT mint, opened_at, cost_sol, entry_price, last_mark_price, hold_s, strategy FROM positions WHERE book = %s AND status = 'open' ORDER BY opened_at", (BOOK,)) + if not rows: + st.caption("None open.") + else: + hold = int((system_state().get("model") or {}).get("run", {}).get("horizon_min") or 30) + now = datetime.now(timezone.utc); names = labels([r["mint"] for r in rows]) + df = pd.DataFrame([{"token": names.get(r["mint"], r["mint"][:6]), "strategy": r["strategy"] or "—", "held (min)": (now - r["opened_at"]).total_seconds() / 60, + "exits in (min)": max(0.0, (float(r["hold_s"]) / 60 if r["hold_s"] else hold) - (now - r["opened_at"]).total_seconds() / 60), + "cost (SOL)": float(r["cost_sol"]), + "return": (float(r["last_mark_price"]) / float(r["entry_price"]) - 1) * 100 if r["entry_price"] and r["last_mark_price"] else None} + for r in rows]) + st.dataframe(df, hide_index=True, column_config={"held (min)": st.column_config.NumberColumn(format="%.0f"), + "exits in (min)": st.column_config.NumberColumn(format="%.0f"), + "cost (SOL)": st.column_config.NumberColumn(format="%.3f"), + "return": st.column_config.NumberColumn("return (gross)", format="%+.2f%%")}) + + +@st.fragment(run_every="30s") +def race() -> None: + s = system_state(); ho = s["handover"]; fly = s["fly"] + marks = q("SELECT ts, book, wealth FROM wealth_marks WHERE book IN ('paper_selector', 'paper_fly', 'live') AND ts > now() - interval '30 days' ORDER BY ts") + if not marks and not (fly.get("stage") and fly.get("stage") != "not trading"): + return + since = datetime.now(timezone.utc) - timedelta(days=14) + pnl = {r["book"]: r for r in q("SELECT book, COALESCE(sum(realized_sol), 0) AS pnl, count(*) AS n FROM positions WHERE book IN ('paper_selector', 'paper_fly', 'live') " + "AND status = 'closed' AND closed_at >= %s GROUP BY book", (since,))} + start = (q1("SELECT min(ts) AS t FROM wealth_marks WHERE book = 'paper_fly'") or {}).get("t") + with st.container(border=True): + st.markdown(":material/sports_score: **The race: selector vs plastic fly**") + if ho: + st.success(f"The fly took the selector's seat {ago(ho.get('at'))}: {float(ho.get('fly_pnl_sol') or 0):+.3f} SOL vs {float(ho.get('selector_pnl_sol') or 0):+.3f} SOL " + f"over {ho.get('days')} days. " + ("It trades the bot wallet; its paper book is the mirror." if s["live_money"] + else "Live trading waits for LIVE_ENABLED=1, the live prerequisites and a funded wallet."), icon=":material/swap_horiz:") + else: + days = (datetime.now(timezone.utc) - start).total_seconds() / 86400 if start else 0.0 + st.caption("The fly takes the seat when its realized P&L over the last 14 days is at least the selector's, with at least 30 trades" + + (f" · racing for {days:.1f} days." if start else " · the race starts when the fly first trades.")) + with st.container(horizontal=True): + for book, title in (("paper_selector", "Selector · 14 days"), ("paper_fly", "Fly · 14 days")) + ((("live", "Live · 14 days"),) if ho else ()): + r = pnl.get(book) or {} + st.metric(title, sol(r.get("pnl"), signed=True), delta=f"{int(r.get('n') or 0)} trades", delta_color="off", border=True, help="Realized P&L of closed trades.") + if marks: + df = pd.DataFrame(marks).pivot_table(index="ts", columns="book", values="wealth") + st.line_chart(df, x_label="", y_label="SOL", height=240) + + +summary() +race() +book() diff --git a/fly_trader/ui/app_pages/processes.py b/fly_trader/ui/app_pages/processes.py new file mode 100644 index 0000000000000000000000000000000000000000..29c52b457f8731ef52c805e99599527513895a51 --- /dev/null +++ b/fly_trader/ui/app_pages/processes.py @@ -0,0 +1,91 @@ +"""Processes: every worker thread with what it does, its state, start/stop, autostart and recent activity; the event log.""" +import json + +import pandas as pd +import streamlit as st + +from fly_trader.db.apilog import record_event +from fly_trader.db.connection import transaction +from fly_trader.db.queries import q, q1 +from fly_trader.ops import procs +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import ORDER, ROLE_LABEL, WORKER_INFO, ago, jv + + +def _set_autostart(name: str) -> None: + r = q1("SELECT value FROM ui_settings WHERE key = 'autostart'"); val = jv(r["value"]) if r else {} + val[name] = bool(st.session_state[f"auto_{name}"]) + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('autostart', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", (json.dumps(val),)) + record_event("info", "console", "autostart changed", val) + + +def _recent(name: str) -> pd.DataFrame: + rows = [] + for ln in [ln for ln in get_supervisor().log_tail(name, 200).splitlines() if ln.strip()][-12:]: + try: + d = json.loads(ln); rows.append({"time": d.get("ts", "")[11:19], "level": d.get("level", ""), "what": d.get("msg", "")[:300]}) + except ValueError: + rows.append({"time": "", "level": "", "what": ln[:300]}) + return pd.DataFrame(rows[::-1]) + + +@st.fragment(run_every="3s") +def workers() -> None: + sup = get_supervisor(); status = sup.status(); ext = sup.external() + auto = jv((q1("SELECT value FROM ui_settings WHERE key = 'autostart'") or {}).get("value")) + if ext: + st.error("Worker processes are running outside the console (started from the command line). Stop them to keep everything in one place.", icon=":material/warning:") + for name, a in ext.items(): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(f"External **{name}** · pid {a['pid']} · started {ago(a['started_at'])}") + if st.button("Stop external", key=f"ext_stop_{name}"): + procs.stop(name); st.rerun() + for name in ORDER: + title, icon, desc, role = WORKER_INFO[name]; stt = status[name] + state = "stopping" if stt["stopping"] else "running" if stt["alive"] else "crashed" if stt.get("error") else "stopped" + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(f"{icon} **{title}**") + st.badge(state, color={"running": "green", "stopping": "orange", "crashed": "red", "stopped": "gray"}[state]) + st.badge(ROLE_LABEL[role], color="gray") + st.space("stretch") + if st.button("Start", icon=":material/play_arrow:", key=f"start_{name}", disabled=stt["alive"] or name in ext): + try: + sup.start(name, started_by="console") + except RuntimeError as e: + st.error(str(e)) + st.rerun() + if st.button("Stop", icon=":material/stop:", key=f"stop_{name}", disabled=not stt["alive"] or stt["stopping"]): + sup.request_stop(name); st.rerun() + st.toggle("Autostart", value=bool(auto.get(name)), key=f"auto_{name}", on_change=_set_autostart, args=(name,), + help="Start this worker whenever the console starts.") + with st.popover("Activity", icon=":material/list:"): + df = _recent(name) + if len(df): + st.dataframe(df, hide_index=True, width=720) + else: + st.caption("Nothing logged yet.") + since = f"running since {ago(stt['started_at'])}" if stt["alive"] and stt.get("started_at") else (f"stopped {ago(stt['stopped_at'])}" if stt.get("stopped_at") else "not started") + st.caption(f"{desc} · {since} · thread `{name}`") + if stt.get("error") and not stt["alive"]: + st.error(stt["error"].splitlines()[0][:300]) + + +@st.fragment(run_every="10s") +def event_log() -> None: + with st.container(border=True): + with st.container(horizontal=True, vertical_alignment="center"): + st.markdown(":material/history_edu: **Event log**") + st.space("stretch") + levels = st.pills("Levels", ["info", "warning", "error"], selection_mode="multi", default=["warning", "error"], key="ev_levels", label_visibility="collapsed") + rows = q("SELECT ts, level, source, message FROM events WHERE level = ANY(%s) ORDER BY id DESC LIMIT 200", (levels or ["error"],)) + if rows: + st.dataframe(pd.DataFrame(rows), hide_index=True, height=320, column_config={"ts": st.column_config.DatetimeColumn("time", format="MMM D HH:mm:ss")}) + else: + st.caption("Nothing at these levels.") + + +st.caption(f"All workers are threads inside this console process (pid {get_supervisor().pid}); stopping one waits for it to finish its current step.") +workers() +event_log() diff --git a/fly_trader/ui/app_pages/safety.py b/fly_trader/ui/app_pages/safety.py new file mode 100644 index 0000000000000000000000000000000000000000..dc26b30a1c2f1af700cf5e0ad645f8f50d35143f --- /dev/null +++ b/fly_trader/ui/app_pages/safety.py @@ -0,0 +1,99 @@ +"""Safety & wallet: the trading rails and their controls, the configured mode and limits, the bot wallet, maintenance.""" +import pandas as pd +import streamlit as st + +from fly_trader import config +from fly_trader.agent import rails +from fly_trader.db.queries import q, q1 +from fly_trader.ops.supervisor import get_supervisor +from fly_trader.ui.common import ago + + +@st.fragment(run_every="5s") +def rails_panel() -> None: + c = q1("SELECT * FROM circuit_state WHERE id = 1") or {} + with st.container(border=True): + st.markdown(":material/shield: **Trading rails**") + with st.container(horizontal=True, vertical_alignment="center"): + st.badge("Kill switch tripped" if c.get("kill_switch") else "Kill switch armed", color="red" if c.get("kill_switch") else "green") + st.caption("Trips when the live book falls too far below its peak wealth; blocks new entries until cleared." + + (f" Reason: {c['kill_reason']}." if c.get("kill_reason") else "")) + st.space("stretch") + if st.button("Clear kill switch", key="kill_clear", disabled=not c.get("kill_switch"), help="Clears the switch and re-bases the peak at current wealth."): + rails.reset_circuit(kill=True); st.toast("Kill switch cleared"); st.rerun() + with st.container(horizontal=True, vertical_alignment="center"): + st.badge(f"Circuit tripped ({c.get('fail_count', 0)} failures)" if c.get("tripped") else f"Circuit ok ({c.get('fail_count', 0)} failures)", + color="red" if c.get("tripped") else "green") + st.caption("Trips after repeated failed or unverifiable live orders; blocks new live entries.") + st.space("stretch") + if st.button("Reset circuit", key="circuit_reset", disabled=not c.get("tripped") and not c.get("fail_count")): + rails.reset_circuit(kill=False); st.toast("Circuit reset"); st.rerun() + with st.container(horizontal=True, vertical_alignment="center"): + st.badge("New entries paused" if c.get("entries_paused") else "New entries allowed", color="orange" if c.get("entries_paused") else "green") + st.caption("Operator switch: no new positions while paused; open positions still exit on schedule.") + st.space("stretch") + if st.button("Resume entries" if c.get("entries_paused") else "Pause entries", key="entries_toggle"): + rails.set_entries_paused(not c.get("entries_paused")); st.rerun() + for b in q("SELECT book, halted, reason FROM book_state WHERE halted ORDER BY book"): + with st.container(horizontal=True, vertical_alignment="center"): + st.badge(f"{b['book']} halted", color="red") + st.caption(f"The paper book fell too far below its own peak ({b['reason']}); its entries are blocked, the other book trades on.") + st.space("stretch") + if st.button("Clear", key=f"halt_{b['book']}"): + rails.clear_book_halt(b["book"]); st.rerun() + + +def settings_panel() -> None: + with st.container(border=True): + st.markdown(":material/tune: **Mode and limits**") + rows = [("Money", "paper, then live once the fly holds the seat", "The race runs on paper books; after the handover the fly also trades the bot wallet, its paper book as the mirror."), + ("LIVE_ENABLED", "1" if config.LIVE_ENABLED else "0", "Allows signing real transactions: the fly trades the bot wallet once it holds the selector's seat."), + ("Cluster", config.SOLANA_CLUSTER, "Solana network for live orders."), + ("Starting capital", f"{config.CAPITAL_SOL:g} SOL", "Paper books start here."), + ("Position sizing", f"{config.KELLY_FRACTION:g} × Kelly", "Each buy is sized from how certain the model is: this share of the growth-optimal bet for its score band."), + ("Largest position", f"{config.MAX_POSITION_FRACTION:.0%} of the bankroll", "Bankroll = wealth at cost minus the gas reserve."), + ("Pool share cap", f"{config.MAX_POOL_SHARE:.0%} of the pool", "Bounds the price impact of larger buys."), + ("Smallest position", f"{config.MIN_POSITION_SOL:g} SOL", "Smaller sized buys are skipped."), + ("Fixed size (older models)", f"{config.MAX_POSITION_SOL:g} SOL", "Models trained before sizing trade this size."), + ("Gas reserve", f"{config.GAS_RESERVE_SOL:g} SOL", "Never spent on positions."), + ("Reset on start", "yes" if config.RESET_ON_START else "no", "Starting the trading engine archives and clears books other than the race books and the live book.")] + st.dataframe(pd.DataFrame(rows, columns=["setting", "value", "meaning"]), hide_index=True) + st.caption("Set in the project .env; changes apply after the console restarts.") + + +def wallet_panel() -> None: + ev = q("SELECT ts, kind, pubkey FROM wallet_events ORDER BY id DESC LIMIT 20") + with st.container(border=True): + st.markdown(":material/account_balance_wallet: **Bot wallet**") + st.code(ev[0]["pubkey"] if ev else "No wallet yet: run `fly-trader wallet new`.", language=None) + if ev: + with st.expander("Wallet events"): + st.dataframe(pd.DataFrame(ev), hide_index=True) + + +def maintenance_panel() -> None: + sup = get_supervisor() + with st.container(border=True): + st.markdown(":material/build: **Maintenance**") + with st.container(horizontal=True, vertical_alignment="center"): + st.caption("Archive, then clear old books, their decisions and wealth marks — never the race books (selector, fly), the live book, the fly's learning or the models. Happens automatically when the trading engine starts.") + st.space("stretch") + with st.popover("Reset paper history", disabled=sup.alive("runner"), help="Stop the trading engine first." if sup.alive("runner") else None): + st.markdown("Archive and clear the old books now? The race and live books are kept.") + if st.button("Reset", type="primary", key="reset_confirm"): + from fly_trader.ops.reset import reset_training_state + out = reset_training_state(reason="console button") + st.success(f"Archived to {out['archived_to']}") + errs = q("SELECT ts, service, endpoint, status, error FROM api_calls WHERE NOT ok ORDER BY id DESC LIMIT 30") + if errs: + with st.expander(f"Recent API errors ({len(errs)})"): + st.dataframe(pd.DataFrame(errs), hide_index=True) + + +rails_panel() +left, right = st.columns([3, 2]) +with left: + settings_panel() +with right: + wallet_panel() +maintenance_panel() diff --git a/fly_trader/ui/app_pages/trades.py b/fly_trader/ui/app_pages/trades.py new file mode 100644 index 0000000000000000000000000000000000000000..d794bd16b7f38e0f26ae8979604abd278c75214d --- /dev/null +++ b/fly_trader/ui/app_pages/trades.py @@ -0,0 +1,61 @@ +"""Trades: closed trades with their results, the decision log (every buy, sell and pick not taken) and fills.""" +import pandas as pd +import streamlit as st + +from fly_trader.db.queries import q, q1 +from fly_trader.ui.common import BOOK, labels, pct, sol + +books = [r["book"] for r in q("SELECT DISTINCT book FROM positions UNION SELECT DISTINCT book FROM fills ORDER BY 1")] or [BOOK] +with st.container(horizontal=True, vertical_alignment="bottom"): + view = st.segmented_control("View", ["Closed trades", "Decision log", "Fills"], default="Closed trades", key="trades_view") + book = st.selectbox("Book", books, index=books.index(BOOK) if BOOK in books else 0, key="trades_book", width=220, + help="paper_selector is the selector's paper book; live is the bot wallet.") + +if view == "Closed trades": + c = q1("SELECT count(*) AS n, sum(realized_sol) AS pnl, avg(realized_sol / NULLIF(cost_sol, 0)) AS avg_ret, " + "count(*) FILTER (WHERE realized_sol > 0) AS wins FROM positions WHERE book = %s AND status = 'closed'", (book,)) or {} + n = int(c.get("n") or 0) + with st.container(horizontal=True): + st.metric("Trades", n, border=True) + st.metric("Realized P&L", sol(c.get("pnl"), signed=True), border=True) + st.metric("Average return", pct(c.get("avg_ret")), border=True, help="Per trade, after fees and price impact.") + st.metric("Winners", f"{int(c.get('wins') or 0) / n:.0%}" if n else "—", border=True) + rows = q("SELECT mint, opened_at, closed_at, cost_sol, realized_sol, forced_exit_kind FROM positions WHERE book = %s AND status = 'closed' ORDER BY closed_at DESC LIMIT 300", (book,)) + if rows: + names = labels([r["mint"] for r in rows]) + df = pd.DataFrame([{"token": names.get(r["mint"], r["mint"][:6]), "opened": r["opened_at"], "closed": r["closed_at"], + "held (min)": (r["closed_at"] - r["opened_at"]).total_seconds() / 60 if r["closed_at"] else None, "cost (SOL)": float(r["cost_sol"] or 0), + "P&L (SOL)": float(r["realized_sol"] or 0), "return": float(r["realized_sol"] or 0) / float(r["cost_sol"]) * 100 if r["cost_sol"] else None, + "exit": r["forced_exit_kind"] or "time"} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"opened": st.column_config.DatetimeColumn(format="MMM D HH:mm"), "closed": st.column_config.DatetimeColumn(format="MMM D HH:mm"), + "held (min)": st.column_config.NumberColumn(format="%.0f"), "cost (SOL)": st.column_config.NumberColumn(format="%.3f"), + "P&L (SOL)": st.column_config.NumberColumn(format="%+.4f"), "return": st.column_config.NumberColumn(format="%+.2f%%")}) + else: + st.caption("No closed trades in this book yet.") + +elif view == "Decision log": + kinds = {"Bought": ("selector_enter", "enter"), "Sold": ("selector_exit", "exit"), "Not taken": ("blocked",)} + pick = st.pills("Show", list(kinds), selection_mode="multi", default=list(kinds), key="dec_kinds") + want = [k for p in (pick or []) for k in kinds[p]] + rows = q("SELECT ts, kind, mint, m_hat, size_sol, rail, reason FROM decisions WHERE kind = ANY(%s) ORDER BY id DESC LIMIT 300", (want,)) if want else [] + if rows: + names = labels([r["mint"] for r in rows]); action = {"selector_enter": "Bought", "enter": "Bought", "selector_exit": "Sold", "exit": "Sold", "blocked": "Not taken"} + df = pd.DataFrame([{"time": r["ts"], "action": action.get(r["kind"], r["kind"]), "token": names.get(r["mint"], (r["mint"] or "")[:6]), + "score": float(r["m_hat"]) if r["m_hat"] is not None else None, "size (SOL)": float(r["size_sol"] or 0), + "why": (f"{r['rail']}: " if r["rail"] else "") + (r["reason"] or "")} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "score": st.column_config.NumberColumn(format="%.3f"), + "size (SOL)": st.column_config.NumberColumn(format="%.3f")}) + else: + st.caption("No decisions recorded yet.") + +else: + rows = q("SELECT ts, side, mint, sol_delta_lamports, price_sol, fee_lamports, verified_by, signature FROM fills WHERE book = %s ORDER BY id DESC LIMIT 300", (book,)) + if rows: + names = labels([r["mint"] for r in rows]) + df = pd.DataFrame([{"time": r["ts"], "side": r["side"], "token": names.get(r["mint"], r["mint"][:6]), "SOL": (r["sol_delta_lamports"] or 0) / 1e9, + "price (SOL)": float(r["price_sol"]) if r["price_sol"] is not None else None, "fee (SOL)": (r["fee_lamports"] or 0) / 1e9, + "source": "simulated" if r["verified_by"] == "model" else r["verified_by"], "signature": r["signature"]} for r in rows]) + st.dataframe(df, hide_index=True, column_config={"time": st.column_config.DatetimeColumn(format="MMM D HH:mm:ss"), "SOL": st.column_config.NumberColumn(format="%+.4f"), + "price (SOL)": st.column_config.NumberColumn(format="%.3e"), "fee (SOL)": st.column_config.NumberColumn(format="%.5f")}) + else: + st.caption("No fills in this book yet.") diff --git a/fly_trader/ui/common.py b/fly_trader/ui/common.py new file mode 100644 index 0000000000000000000000000000000000000000..209ede024fad1a80d955217bc36eda80620e8a2e --- /dev/null +++ b/fly_trader/ui/common.py @@ -0,0 +1,215 @@ +"""Shared reads and renderers for the operator console: one place that answers what the system is doing — trading or +not, training or not, paper or live money, which model is loaded — so every page says it the same way.""" +from __future__ import annotations + +import json +import re +from datetime import datetime, timezone + +import streamlit as st + +from fly_trader import config +from fly_trader.db.queries import q, q1 +from fly_trader.ops.supervisor import WORKERS, get_supervisor +from fly_trader.train.selector import is_current, is_deployable + +BOOK = "paper_selector" +STALE_FEED_S = 180 # agent/minute_engine.py: no entries or exits when the newest complete minute is older + +WORKER_INFO = { # name → (title, icon, what it does, role) + "runner": ("Trading engine", ":material/candlestick_chart:", "Scores every traded token once a minute with the selector and the plastic fly, trades their books and lets the fly learn.", "trade"), + "pumpstream": ("Market feed", ":material/sensors:", "Streams every PumpSwap trade from pumpapi.io into 1-minute candles. The trading engine reads these.", "trade"), + "train": ("Trainer", ":material/model_training:", "On demand: builds decision points from the archive, backtests day by day and saves a new model.", "train"), + "replay": ("History archive", ":material/history:", "Downloads the hourly trade archive and builds the training feature set.", "train"), + "discover": ("Token stats", ":material/query_stats:", "Records Jupiter stats (holders, organic score, top-holder share) every 10 minutes for every token the selector can trade: history for future model inputs.", "optional"), +} +ROLE_LABEL = {"trade": "needed to trade", "train": "needed to train", "optional": "optional"} +ORDER = [n for n in ("runner", "pumpstream", "train", "replay", "discover") if n in WORKERS] + + +# ---------------------------------------------------------------- formatting +def jv(v) -> dict: + if isinstance(v, dict): + return v + try: + return json.loads(v or "{}") + except (TypeError, ValueError): + return {} + + +def setting(key: str) -> tuple[dict, datetime | None]: + r = q1("SELECT value, updated_at FROM ui_settings WHERE key = %s", (key,)) + return (jv(r["value"]), r["updated_at"]) if r else ({}, None) + + +def ago(ts) -> str: + if ts is None: + return "—" + if isinstance(ts, str): + ts = datetime.fromisoformat(ts) + s = max(0.0, (datetime.now(timezone.utc) - ts).total_seconds()) + return f"{s:.0f} s ago" if s < 120 else f"{s / 60:.0f} min ago" if s < 7200 else f"{s / 3600:.1f} h ago" if s < 172800 else f"{s / 86400:.0f} d ago" + + +def sol(x, nd: int = 4, signed: bool = False) -> str: + return "—" if x is None else f"{float(x):{'+' if signed else ''},.{nd}f} SOL" + + +def pct(x, nd: int = 2, signed: bool = True) -> str: + return "—" if x is None else f"{float(x) * 100:{'+' if signed else ''}.{nd}f}%" + + +# ---------------------------------------------------------------- models +_NOTE_OBJ = re.compile(r'"(walk_forward|random_baseline|fly|gbm|random)": (\{[^{}]*\})') +_NOTE_SCALAR = re.compile(r'"(threshold|trained_through|horizon_min|line|costs|rows|days|fly_beats_gbm)": ("[^"]*"|[-0-9.eE]+|true|false)') + + +def parse_note(note) -> dict: + """A snapshot's note (JSON; older notes were cut at 900 characters, so fall back to the fields that survive).""" + try: + return json.loads(note or "{}") + except (TypeError, ValueError): + out = {k: json.loads(v) for k, v in _NOTE_SCALAR.findall(note or "")} + for k, v in _NOTE_OBJ.findall(note or ""): + try: + out[k] = json.loads(v) + except ValueError: + pass + return out + + +def _snap(r) -> dict | None: + return {**r, "meta": parse_note(r["note"])} if r else None + + +def snapshot(sid: int) -> dict | None: + return _snap(q1("SELECT id, ts, kind, note FROM brain_snapshots WHERE id = %s", (sid,))) + + +def latest_snapshot(kind: str = "selector", deployable: bool = False) -> dict | None: + """The newest snapshot of this kind trained on the current data definitions (older ones are never loaded); + with ``deployable`` only one whose backtest qualified it to trade.""" + for r in q("SELECT id, ts, kind, note FROM brain_snapshots WHERE kind = %s ORDER BY id DESC LIMIT 50", (kind,)): + s = _snap(r) + if (is_deployable if deployable else is_current)(s["meta"]): + return s + return None + + +def loaded_model() -> dict | None: + """The snapshot the running selector session loaded (its runs row), with its backtest.""" + r = q1("SELECT brain_snapshot_id, config, started_at FROM runs WHERE kind = 'selector' AND status = 'running' ORDER BY started_at DESC LIMIT 1") + if not r or not r["brain_snapshot_id"]: + return None + s = snapshot(int(r["brain_snapshot_id"])) + return {**s, "run": jv(r["config"]), "since": r["started_at"]} if s else None + + +OUTDATED = "Trained on outdated data (before the data fixes of 2026-09-14), so its backtest is not valid and it will not be loaded." + + +def backtest_line(meta: dict) -> str: + if not is_current(meta): + return OUTDATED + wf, rb = meta.get("walk_forward") or {}, meta.get("random_baseline") or {} + if not wf: + return "No backtest recorded." + s = (f"Backtest on {wf.get('days')} held-out days: {pct(wf.get('mean'))} average per trade over {wf.get('n')} trades, " + f"profitable on {wf.get('days_positive')} of those {wf.get('days')} days") + s += (f"; random picks made {pct(rb['mean'])} per trade under the same costs." if rb.get("mean") is not None else ".") + if meta.get("deploy_reason"): + s += (" Put to work: " if meta.get("deployable") else " Not put to work: ") + meta["deploy_reason"] + "." + return s + + +def labels(mints) -> dict[str, str]: + """Readable token names: symbol from corpus_meta or the watch list, else the short mint.""" + mints = [m for m in set(mints) if m] + if not mints: + return {} + rows = q("""SELECT m.mint, COALESCE(NULLIF(cm.symbol, ''), NULLIF(t.symbol, '')) AS sym FROM unnest(%s::text[]) AS m(mint) + LEFT JOIN corpus_meta cm ON cm.mint = m.mint LEFT JOIN tokens t ON t.mint = m.mint""", (mints,)) + return {r["mint"]: (f"{r['sym']} · {r['mint'][:4]}…" if r["sym"] else f"{r['mint'][:6]}…") for r in rows} + + +# ---------------------------------------------------------------- system state +def system_state() -> dict: + sup = get_supervisor(); ws = sup.status(); now = datetime.now(timezone.utc) + sel, sel_at = setting("selector_status"); tr, tr_at = setting("training_status"); ps, _ = setting("pumpstream_status") + fly, fly_at = setting("fly_status"); ho, _ = setting("handover"); rp, _ = setting("fly_replay") + circuit = q1("SELECT kill_switch, kill_reason, tripped, fail_count, entries_paused FROM circuit_state WHERE id = 1") or {} + ft = ps.get("flushed_through") + feed_age = (now - datetime.fromisoformat(ft)).total_seconds() - 60 if ft else None # since the newest complete minute closed + feed_ok = ws["pumpstream"]["alive"] and feed_age is not None and feed_age < STALE_FEED_S + runner = ws["runner"]["alive"] + stage = str(tr.get("stage") or "") + training = ws["train"]["alive"] or (tr_at is not None and (now - tr_at).total_seconds() < 180 and not stage.endswith(("saved", "done"))) + if not runner: + err = (ws["runner"].get("error") or "").splitlines() + trading, why = "stopped", ("The trading engine is stopped: " + err[0].split(": ", 1)[-1]) if err else "The trading engine is stopped." + elif str(sel.get("stage", "")).startswith("waiting for a model") and fly.get("stage") != "trading": + trading, why = "waiting", "No model has qualified to trade yet: a model trades only if its backtest made money after costs and beat random picks. It starts as soon as one does." + elif circuit.get("kill_switch"): + trading, why = "blocked", "Kill switch tripped" + (f" ({circuit['kill_reason']})" if circuit.get("kill_reason") else "") + ": no new entries." + elif circuit.get("entries_paused"): + trading, why = "paused", "New entries are paused by the operator; open positions still exit on time." + elif str(sel.get("stage", "")).startswith("stream stale") or not feed_ok: + trading, why = "holding", "The market feed is stale: no entries or exits until it recovers." + elif not sel and not fly: + trading, why = "starting", "Warming up on the last 24 hours of market minutes." + else: + trading, why = "trading", "Trading every minute." + from fly_trader.chain.cluster_guard import signing_allowed + live_money = bool(ho) and runner and fly.get("stage") == "trading" and signing_allowed() # the fly holds the seat and may sign (agent/fly_live.py) + return {"workers": ws, "runner": runner, "trading": trading, "trading_why": why, "live_money": live_money, + "fly": fly, "fly_at": fly_at, "handover": ho or None, "replay": rp or None, + "model": loaded_model() if runner else None, "latest": latest_snapshot("selector"), "deployable": latest_snapshot("selector", deployable=True), + "training": training, "train_status": tr, "train_at": tr_at, + "train_external": training and not ws["train"]["alive"], "selector": sel, "selector_at": sel_at, "feed": ps, "feed_ok": feed_ok, "feed_age": feed_age, + "circuit": circuit} + + +TRADING_BADGE = {"trading": ("Trading", "green"), "holding": ("Holding: feed stale", "orange"), "paused": ("Entries paused", "orange"), + "blocked": ("Kill switch tripped", "red"), "starting": ("Starting", "blue"), "stopped": ("Not trading", "gray"), + "waiting": ("Waiting for a qualified model", "gray")} + + +@st.fragment(run_every="5s") +def status_strip() -> None: + s = system_state() + with st.container(horizontal=True, gap="small"): + if s["live_money"]: + st.badge("Live money", icon=":material/payments:", color="red", help="Trades spend real SOL from the bot wallet.") + else: + st.badge("Paper — no real SOL", icon=":material/receipt:", color="blue", help="Trades are simulated at market prices with real fees; no wallet funds are used.") + text, color = TRADING_BADGE[s["trading"]] + st.badge(text, icon=":material/candlestick_chart:", color=color, help=s["trading_why"]) + m, latest, dep = s["model"], s["latest"], s["deployable"] + st.badge(f"Model #{m['id']}" if m else "No model at work", icon=":material/psychology:", color="violet" if m else "gray", + help=backtest_line(m["meta"]) if m else ("No model has been trained on the current data yet." if not latest + else "No model has earned a place yet: " + (latest["meta"].get("deploy_reason") or "none qualified") if not dep + else "The trading engine is stopped, so no model is in use.")) + if latest and not is_deployable(latest["meta"]) and (not m or latest["id"] > m["id"]): + st.badge(f"Model #{latest['id']} not put to work", icon=":material/block:", color="orange", help=backtest_line(latest["meta"])) + if m and dep and dep["id"] > m["id"]: + st.badge(f"Switching to model #{dep['id']}", icon=":material/upgrade:", color="blue", help="The trading engine picks it up within 10 minutes.") + fly = s["fly"] + if fly.get("stage") and fly.get("stage") != "not trading": + st.badge(f"Fly learning · drift {float(fly.get('drift') or 0):.1%}" if not fly.get("learning_frozen") else "Fly: learning frozen", icon=":material/neurology:", + color="violet" if not fly.get("learning_frozen") else "orange", help=f"Plastic fly: buy line {pct(fly.get('line'))}, {fly.get('pending', 0)} labels pending.") + else: + st.badge("Fly not trading", icon=":material/neurology:", color="gray", help=fly.get("detail") or "The trading engine is stopped.") + if s["handover"]: + st.badge("Fly holds the seat", icon=":material/swap_horiz:", color="green", help="The fly won the race; the selector no longer trades while the fly can.") + tr = s["train_status"] + st.badge(f"Training · {tr.get('stage', '')}" if s["training"] else "Not training", icon=":material/model_training:", color="blue" if s["training"] else "gray") + st.badge("Market feed live" if s["feed_ok"] else "Market feed stale", icon=":material/sensors:", color="green" if s["feed_ok"] else "red", + help=f"newest complete minute closed {s['feed_age']:.0f} s ago" if s["feed_age"] is not None else "no minute written yet") + + +def restart_runner() -> None: + """Stop the trading engine and start it again (it loads the newest model; RESET_ON_START clears the paper book).""" + sup = get_supervisor() + if sup.alive("runner"): + sup.stop("runner", grace_s=60.0) + sup.start("runner", started_by="console") diff --git a/models/connectome/current.txt b/models/connectome/current.txt new file mode 100644 index 0000000000000000000000000000000000000000..587e43efcfadce0dcc085935a9fe7e8f39c42545 --- /dev/null +++ b/models/connectome/current.txt @@ -0,0 +1 @@ +fafb783_d1bad40e798b.npz diff --git a/models/connectome/fafb783_d1bad40e798b.npz b/models/connectome/fafb783_d1bad40e798b.npz new file mode 100644 index 0000000000000000000000000000000000000000..30a0cba9678c3c681f0f9cdab687ec3a085d915a --- /dev/null +++ b/models/connectome/fafb783_d1bad40e798b.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9435f08f175b1b569e6d6d2ea338f906d4a43349e6e3216213f511c5082436d0 +size 14403111 diff --git a/models/connectome/scale.json b/models/connectome/scale.json new file mode 100644 index 0000000000000000000000000000000000000000..f50d6c019fc5ca956f996461ee41fa0e943dd637 --- /dev/null +++ b/models/connectome/scale.json @@ -0,0 +1,19 @@ +{ + "connectome": "fafb783_d1bad40e798b.npz", + "content_sha256": "d1bad40e798b65881c05c38a896f67f51b5b8ff1d32b4c5166e4d2f9fd9a5c7c", + "s": 0.003683247034028392, + "s_km": 0.05893195254445427, + "ts": "2026-09-12T15:58:01.074657+00:00", + "s_note": "0.25 x calibrated s: sparse regime under the real encoder input (scale_sweep_encoder)", + "pathway_gains": { + "DAN_PAM->MBON_APP": 0.0, + "DAN_PAM->MBON_AV": 0.0, + "DAN_PAM->MBON_OTHER": 0.0, + "DAN_PAM->KC": 0.0, + "DAN_PPL1->MBON_APP": 0.0, + "DAN_PPL1->MBON_AV": 0.0, + "DAN_PPL1->MBON_OTHER": 0.0, + "DAN_PPL1->KC": 0.0 + }, + "pathway_gains_note": "dopamine acts on MBONs through KC->MBON plasticity (GPCR modulation), not fast excitation: Hige 2015, Handler 2019; direct DAN->MBON/KC edges silenced" +} \ No newline at end of file diff --git a/models/policies/fly_20260921T160246Z.pt b/models/policies/fly_20260921T160246Z.pt new file mode 100644 index 0000000000000000000000000000000000000000..e66a03e24bfe4671620967fa288cf1a26449181e --- /dev/null +++ b/models/policies/fly_20260921T160246Z.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d18ef071d4106cea6846b494340ce9518e5e5ac55cde31b79670fbead406c744 +size 28437189 diff --git a/models/selectors/selector_20260919T201310Z.joblib b/models/selectors/selector_20260919T201310Z.joblib new file mode 100644 index 0000000000000000000000000000000000000000..c9078c3151b9fdd4aca86cc577ea335dd4273010 --- /dev/null +++ b/models/selectors/selector_20260919T201310Z.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3997c13242e1ef3abcd9a4105cdb58c39a29a72f55c55d2585166d522d948ac1 +size 11814643 diff --git a/models/wallet_skill/TABLE b/models/wallet_skill/TABLE new file mode 100644 index 0000000000000000000000000000000000000000..6a489035ed35ad2ac85b2b85f33579d33611ee4d --- /dev/null +++ b/models/wallet_skill/TABLE @@ -0,0 +1 @@ +2026-09-23.parquet diff --git a/models/wallet_skill/config.json b/models/wallet_skill/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fb44b8140bbded3e2fc8a25160640db6df3a6888 --- /dev/null +++ b/models/wallet_skill/config.json @@ -0,0 +1 @@ +{"h": 10, "L": 30, "fit": {"h": 10, "L": 30, "selection": {"n": 2475, "win": 0.4488888888888889, "pf": 1.3692030597860738, "total": 109.31889405731636, "mean": 0.044169250124168224, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 777, "win": 0.4864864864864865, "pf": 1.0644973818938945, "total": 6.776452796999365, "mean": 0.008721303471041653, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}, "baseline": {"n": 2181, "win": 0.44154057771664373, "pf": 1.3029667552022617, "total": 79.60777615546431, "mean": 0.03650058512400931, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "beats_baseline": true, "candidates": 45, "trials": 1350, "days": "2026-04-19..2026-09-14|win0.65-pf1.3-trades-noweekly"}, "frozen_at": "2026-09-19T10:22:14Z"} \ No newline at end of file diff --git a/models/wallet_skill/fit.json b/models/wallet_skill/fit.json new file mode 100644 index 0000000000000000000000000000000000000000..4f84b0cb70115cddce97286c9edd71d1b896653b --- /dev/null +++ b/models/wallet_skill/fit.json @@ -0,0 +1 @@ +{"key": "2026-04-19..2026-09-14|win0.65-pf1.3-trades-noweekly", "candidates": {"h10-L1": {"h": 10, "L": 1, "selection": {"n": 411, "win": 0.45255474452554745, "pf": 1.6628983976635239, "total": 30.3908520268742, "mean": 0.07394367889750414, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12473, "win": 0.43173254229134933, "pf": 1.2139256661957805, "total": 276.25436249408085, "mean": 0.022148189087956453, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h10-L3": {"h": 10, "L": 3, "selection": {"n": 488, "win": 0.44672131147540983, "pf": 1.3335544788097422, "total": 20.96477964836231, "mean": 0.042960614033529326, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29003, "win": 0.4338516705168431, "pf": 1.112059463087188, "total": 269.1251749463594, "mean": 0.009279218527268191, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h10-L7": {"h": 10, "L": 7, "selection": {"n": 487, "win": 0.4414784394250513, "pf": 1.313626960196033, "total": 18.390065508419866, "mean": 0.03776194149572868, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29958, "win": 0.4313038253554977, "pf": 1.1201119007771136, "total": 292.96186221992593, "mean": 0.009779086127909939, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h10-L14": {"h": 10, "L": 14, "selection": {"n": 455, "win": 0.45714285714285713, "pf": 1.376585221201266, "total": 21.57372750050854, "mean": 0.04741478571540338, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12656, "win": 0.4252528445006321, "pf": 1.1026411770108024, "total": 135.32092964197545, "mean": 0.010692235275124484, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h10-L30": {"h": 10, "L": 30, "selection": {"n": 2475, "win": 0.4488888888888889, "pf": 1.3692030597860738, "total": 109.31889405731636, "mean": 0.044169250124168224, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29363, "win": 0.4320062663896741, "pf": 1.076067797512786, "total": 182.18092208351663, "mean": 0.00620443830955681, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h20-L1": {"h": 20, "L": 1, "selection": {}, "trials": 30, "best_seen": {"n": 12125, "win": 0.43892783505154637, "pf": 1.1204589564208616, "total": 150.68408716277372, "mean": 0.012427553580434946, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, "h20-L3": {"h": 20, "L": 3, "selection": {"n": 479, "win": 0.4509394572025052, "pf": 1.4785991236786813, "total": 26.876287563121878, "mean": 0.056109159839502876, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29689, "win": 0.43282023645121087, "pf": 1.0791862680858575, "total": 189.51746660856895, "mean": 0.00638342371277473, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h20-L7": {"h": 20, "L": 7, "selection": {"n": 465, "win": 0.5010752688172043, "pf": 1.736124856659587, "total": 36.33313309936784, "mean": 0.0781357701061674, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29449, "win": 0.43247648477028083, "pf": 1.1203159470470432, "total": 289.779373972714, "mean": 0.009840041222884104, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h20-L14": {"h": 20, "L": 14, "selection": {"n": 489, "win": 0.45807770961145194, "pf": 1.3611646281834915, "total": 23.128872835659422, "mean": 0.047298308457381234, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30188, "win": 0.43431164701205777, "pf": 1.0997330554575047, "total": 243.17469887243715, "mean": 0.008055343145370252, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h20-L30": {"h": 20, "L": 30, "selection": {"n": 464, "win": 0.4418103448275862, "pf": 1.349221031540961, "total": 18.606387535925023, "mean": 0.040099973137769446, "weeks": 10, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30067, "win": 0.43526124987527853, "pf": 1.139799725215274, "total": 338.82388231662014, "mean": 0.011268962061948985, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h30-L1": {"h": 30, "L": 1, "selection": {}, "trials": 30, "best_seen": {"n": 30132, "win": 0.4304725872826231, "pf": 1.116378976719453, "total": 287.0014120206097, "mean": 0.009524804593807569, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h30-L3": {"h": 30, "L": 3, "selection": {"n": 479, "win": 0.44676409185803756, "pf": 1.3609312706182177, "total": 20.623827689210884, "mean": 0.043056007701901634, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12480, "win": 0.4342147435897436, "pf": 1.2335853781991595, "total": 297.1919570079717, "mean": 0.023813458093587477, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": false}}, "h30-L7": {"h": 30, "L": 7, "selection": {}, "trials": 30, "best_seen": {"n": 30237, "win": 0.4323841650957436, "pf": 1.0626759230331928, "total": 153.69150480637597, "mean": 0.005082895287441743, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h30-L14": {"h": 30, "L": 14, "selection": {}, "trials": 30, "best_seen": {"n": 30150, "win": 0.4344610281923715, "pf": 1.065215627241937, "total": 158.42693394433388, "mean": 0.0052546246747706095, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h30-L30": {"h": 30, "L": 30, "selection": {"n": 2382, "win": 0.44920235096557515, "pf": 1.361994068601719, "total": 104.25902740225138, "mean": 0.04376953291446321, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30522, "win": 0.43050914094751325, "pf": 1.075935163809396, "total": 187.99778106898697, "mean": 0.006159418814919959, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h45-L1": {"h": 45, "L": 1, "selection": {}, "trials": 30, "best_seen": {"n": 12535, "win": 0.43765456721180696, "pf": 1.2066139322980447, "total": 265.3839662308401, "mean": 0.021171437274099727, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": false}}, "h45-L3": {"h": 45, "L": 3, "selection": {"n": 2261, "win": 0.4431667403803627, "pf": 1.313475537644341, "total": 86.19811352354009, "mean": 0.03812388921872627, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29698, "win": 0.43470940804094554, "pf": 1.0773590318653303, "total": 186.19317612089378, "mean": 0.006269552701222095, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h45-L7": {"h": 45, "L": 7, "selection": {"n": 457, "win": 0.424507658643326, "pf": 1.4130907575180491, "total": 20.335421765834326, "mean": 0.044497640625458044, "weeks": 9, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30422, "win": 0.4315955558477418, "pf": 1.1329248028591503, "total": 325.0529536391041, "mean": 0.010684798949415032, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h45-L14": {"h": 45, "L": 14, "selection": {"n": 515, "win": 0.458252427184466, "pf": 1.5812554065854254, "total": 34.48925028753001, "mean": 0.06696941803403886, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29922, "win": 0.4318227391217165, "pf": 1.0794062770115813, "total": 193.14306237683832, "mean": 0.006454884779655047, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h45-L30": {"h": 45, "L": 30, "selection": {"n": 496, "win": 0.4637096774193548, "pf": 1.71503694933712, "total": 41.372259248862974, "mean": 0.08341181300173987, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30601, "win": 0.42913630273520476, "pf": 1.104290003850196, "total": 258.7183465929123, "mean": 0.008454571634682276, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h60-L1": {"h": 60, "L": 1, "selection": {"n": 563, "win": 0.4404973357015986, "pf": 1.342417016551926, "total": 23.999613927444443, "mean": 0.04262808868107361, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12507, "win": 0.43239785719996804, "pf": 1.1250687631908762, "total": 161.26241673555523, "mean": 0.012893772826061824, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h60-L3": {"h": 60, "L": 3, "selection": {"n": 456, "win": 0.4100877192982456, "pf": 1.316940565240079, "total": 17.35277828667313, "mean": 0.038054338347967384, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29830, "win": 0.43171304056319143, "pf": 1.069563614501408, "total": 169.76903778421365, "mean": 0.0056912181623940215, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, "h60-L7": {"h": 60, "L": 7, "selection": {}, "trials": 30, "best_seen": {"n": 30066, "win": 0.4347768243198297, "pf": 1.1071783455017845, "total": 259.75526993170024, "mean": 0.00863950209311848, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h60-L14": {"h": 60, "L": 14, "selection": {"n": 520, "win": 0.4269230769230769, "pf": 1.3059529634090847, "total": 18.97284354182193, "mean": 0.03648623758042679, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29956, "win": 0.43053144612097743, "pf": 1.081086421470323, "total": 196.78025988301567, "mean": 0.006568976494959797, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h60-L30": {"h": 60, "L": 30, "selection": {"n": 503, "win": 0.44731610337972166, "pf": 1.3634802392215712, "total": 23.705870957870502, "mean": 0.047128968107098414, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 13262, "win": 0.4388478359221837, "pf": 1.1719698395027043, "total": 228.95601643798182, "mean": 0.017264063975115505, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": false}}, "h90-L1": {"h": 90, "L": 1, "selection": {}, "trials": 30, "best_seen": {"n": 12495, "win": 0.4335334133653461, "pf": 1.1922751974287047, "total": 245.38577194311983, "mean": 0.019638717242346525, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h90-L3": {"h": 90, "L": 3, "selection": {"n": 495, "win": 0.43434343434343436, "pf": 1.3513609226243015, "total": 20.431420824257657, "mean": 0.04127559762476294, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29352, "win": 0.4299195966203325, "pf": 1.1119894746735652, "total": 272.6796281124466, "mean": 0.009289984604539609, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": false}}, "h90-L7": {"h": 90, "L": 7, "selection": {}, "trials": 30, "best_seen": {"n": 29337, "win": 0.43068480076354093, "pf": 1.090302264567325, "total": 217.85578893175534, "mean": 0.007425973648694664, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h90-L14": {"h": 90, "L": 14, "selection": {}, "trials": 30, "best_seen": {"n": 12714, "win": 0.43511090136857006, "pf": 1.1521330992955825, "total": 199.65288521638468, "mean": 0.01570338880103702, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h90-L30": {"h": 90, "L": 30, "selection": {}, "trials": 30, "best_seen": {"n": 30150, "win": 0.43233830845771143, "pf": 1.0982278958033842, "total": 238.98482452163262, "mean": 0.007926528176505227, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h120-L1": {"h": 120, "L": 1, "selection": {}, "trials": 30, "best_seen": {"n": 12031, "win": 0.4378688388330147, "pf": 1.1576965337241967, "total": 196.4562442782062, "mean": 0.01632917000068209, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h120-L3": {"h": 120, "L": 3, "selection": {"n": 453, "win": 0.4326710816777042, "pf": 1.5406911168382045, "total": 25.259137703920715, "mean": 0.05575968588061968, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29406, "win": 0.4332449160035367, "pf": 1.124185107008106, "total": 297.7986328829138, "mean": 0.01012713843715275, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h120-L7": {"h": 120, "L": 7, "selection": {"n": 433, "win": 0.4226327944572748, "pf": 1.4703240157381445, "total": 21.843490829342045, "mean": 0.050446861037741444, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30323, "win": 0.4340269762226693, "pf": 1.1270415089972519, "total": 313.0891549248918, "mean": 0.010325137846680467, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h120-L14": {"h": 120, "L": 14, "selection": {}, "trials": 30, "best_seen": {"n": 12582, "win": 0.4426164361786679, "pf": 1.1538463502267997, "total": 193.05818321334846, "mean": 0.015343998029991134, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h120-L30": {"h": 120, "L": 30, "selection": {"n": 470, "win": 0.4702127659574468, "pf": 1.4852882276216066, "total": 26.342000090138754, "mean": 0.05604680870242288, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12602, "win": 0.4380257102047294, "pf": 1.1098776922808702, "total": 140.90195162965534, "mean": 0.011180919824603662, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h180-L1": {"h": 180, "L": 1, "selection": {"n": 425, "win": 0.4329411764705882, "pf": 1.4635358330008994, "total": 20.074030748175574, "mean": 0.047233013525118994, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12283, "win": 0.43849222502645935, "pf": 1.1533227356618816, "total": 194.5814632119309, "mean": 0.01584152594740136, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h180-L3": {"h": 180, "L": 3, "selection": {}, "trials": 30, "best_seen": {"n": 29876, "win": 0.4319185968670505, "pf": 1.1097458213310214, "total": 265.1423037923305, "mean": 0.008874759130818399, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, "h180-L7": {"h": 180, "L": 7, "selection": {}, "trials": 30, "best_seen": {"n": 12430, "win": 0.4349959774738536, "pf": 1.1327187784680073, "total": 166.38407358688164, "mean": 0.013385685727021854, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h180-L14": {"h": 180, "L": 14, "selection": {"n": 477, "win": 0.44234800838574423, "pf": 1.500764342123978, "total": 24.906862261799688, "mean": 0.05221564415471633, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12417, "win": 0.43223000724812755, "pf": 1.2272467478857823, "total": 288.4197344005497, "mean": 0.023227811419871926, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": false}}, "h180-L30": {"h": 180, "L": 30, "selection": {"n": 437, "win": 0.41647597254004576, "pf": 1.4119364269984391, "total": 20.467983357375488, "mean": 0.04683749052031004, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12704, "win": 0.4363979848866499, "pf": 1.217437151533649, "total": 279.48685075973304, "mean": 0.02199990953713264, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h240-L1": {"h": 240, "L": 1, "selection": {"n": 478, "win": 0.4393305439330544, "pf": 1.3731864541138712, "total": 21.948624900207506, "mean": 0.04591762531424164, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 29138, "win": 0.43263092868419245, "pf": 1.0771479252965896, "total": 184.9101392688293, "mean": 0.006346013428129223, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}}, "h240-L3": {"h": 240, "L": 3, "selection": {}, "trials": 30, "best_seen": {"n": 30361, "win": 0.43450479233226835, "pf": 1.1273830699983547, "total": 310.76762184234485, "mean": 0.010235750530033427, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h240-L7": {"h": 240, "L": 7, "selection": {"n": 476, "win": 0.4474789915966387, "pf": 1.3849643436131207, "total": 24.240971312276088, "mean": 0.050926410319907744, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30202, "win": 0.4316932653466658, "pf": 1.1088097931116205, "total": 266.3445177275762, "mean": 0.008818770867080862, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h240-L14": {"h": 240, "L": 14, "selection": {"n": 491, "win": 0.43584521384928715, "pf": 1.3431952894868167, "total": 21.959023855626583, "mean": 0.044723062842416665, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 30188, "win": 0.43543792235325296, "pf": 1.140457256989403, "total": 343.7236604825116, "mean": 0.011386102440788115, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, "h240-L30": {"h": 240, "L": 30, "selection": {"n": 436, "win": 0.4518348623853211, "pf": 1.3066658708134993, "total": 14.514542595250532, "mean": 0.033290235310207646, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}, "trials": 30, "best_seen": {"n": 12915, "win": 0.4410375532326752, "pf": 1.1150359553749367, "total": 152.45026084995698, "mean": 0.011804123952764768, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}}, "baseline": {"n": 2181, "win": 0.44154057771664373, "pf": 1.3029667552022617, "total": 79.60777615546431, "mean": 0.03650058512400931, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": true}} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..61dc191a2c1f049adfc69770a8a8b1ffd0be4476 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "fly-trader" +version = "0.1.0" +description = "FlyWire-connectome memecoin trader on Solana" +requires-python = ">=3.13,<3.14" +dependencies = [ + "torch>=2.12", + "numpy", + "pandas", + "pyarrow>=15", + "duckdb", + "psycopg[binary]>=3.2", + "scikit-learn>=1.5", + "scipy", + "joblib", + "solders>=0.21", + "base58", + "websockets>=12", + "httpx>=0.27", + "python-dotenv", + "orjson", + "streamlit>=1.40", + "altair", + "psutil", +] + +[project.optional-dependencies] +dev = ["pytest", "pytest-cov"] + +[project.scripts] +fly-trader = "fly_trader.__main__:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["fly_trader"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/seed/seed.sql b/seed/seed.sql new file mode 100644 index 0000000000000000000000000000000000000000..f48ce01ca8ce35bcc58029a2c6269889b88a7599 --- /dev/null +++ b/seed/seed.sql @@ -0,0 +1,7 @@ +-- generated by tools/make_seed.py: the rows a fresh install needs to trade. Safe to re-run: nothing here overwrites. +INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('59', '2026-09-19 20:13:10.568909+00:00', NULL, NULL, '/app/data/brain/selectors/selector_20260919T201310Z.joblib', '3997c13242e1ef3abcd9a4105cdb58c39a29a72f55c55d2585166d522d948ac1', 'selector', NULL, NULL, '{"threshold": 0.048561317734419324, "horizon_min": 120, "trained_through": "2026-09-18", "walk_forward": {"n": 3172, "win": 0.4735182849936948, "pf": 1.3978238516274475, "total": 128.41733735687012, "mean": 0.04048465868753787, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_mean": -0.04681328311562538, "days": null}, "selection": {"n": 6652, "win": 0.45625375826819003, "pf": 1.5568584699178285, "total": 359.86430143312464, "mean": 0.054098662271967024, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}, "random_baseline": {"mean": -0.04681328311562538}, "components": [{"name": "inputs: rug+flow+skill", "passed": false, "reason": "improves the ev strategy on the selection half, fails on the evaluation half", "selection": {"n": 5190, "win": 0.43641618497109824, "pf": 1.3862310369585873, "total": 231.10611372606309, "mean": 0.04452911632486765, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 1845, "win": 0.5116531165311653, "pf": 1.2823937362697073, "total": 62.2172390420601, "mean": 0.033722080781604394, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}, "params": {"groups": ["rug", "flow", "skill"]}, "trials": 30, "best_seen": {"n": 5190, "win": 0.43641618497109824, "pf": 1.3862310369585873, "total": 231.10611372606309, "mean": 0.04452911632486765, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}}, {"name": "inputs: rug", "passed": true, "reason": "improves the ev strategy on the selection half", "selection": {"n": 4856, "win": 0.4433690280065898, "pf": 1.4199964576000768, "total": 234.3598922135643, "mean": 0.048261921790272715, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 1687, "win": 0.4866627148784825, "pf": 1.3549911389698504, "total": 72.38500758860027, "mean": 0.04290753265477195, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"groups": ["rug"]}, "trials": 30, "best_seen": {"n": 12166, "win": 0.43021535426598717, "pf": 1.1943888942509129, "total": 249.60527102982905, "mean": 0.02051662592716004, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "inputs: flow", "passed": false, "reason": "does not improve the ev strategy on the selection half", "selection": {"n": 5089, "win": 0.4409510709373158, "pf": 1.3535782472945181, "total": 205.49713800986956, "mean": 0.040380651996437325, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": null, "params": {"groups": ["flow"]}, "trials": 30, "best_seen": {"n": 12908, "win": 0.42872637124264024, "pf": 1.183703489312015, "total": 244.49831808073122, "mean": 0.018941611255092285, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "inputs: skill", "passed": false, "reason": "does not improve the ev strategy on the selection half", "selection": {"n": 4944, "win": 0.436084142394822, "pf": 1.3860342884823982, "total": 212.80902668083218, "mean": 0.04304389698236897, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": null, "params": {"groups": ["skill"]}, "trials": 30, "best_seen": {"n": 30285, "win": 0.4312365857685323, "pf": 1.1148965895956877, "total": 282.3862591761157, "mean": 0.009324294508044104, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: ev", "passed": true, "reason": "profit fallback: below 65 % winners, PF \u22651.3 on both halves", "selection": {"n": 4856, "win": 0.4433690280065898, "pf": 1.4199964576000768, "total": 234.3598922135643, "mean": 0.048261921790272715, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 1687, "win": 0.4866627148784825, "pf": 1.3549911389698504, "total": 72.38500758860027, "mean": 0.04290753265477195, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "params": {"hold_min": 120, "line": 0.048561317734419324, "thr": {}, "high": null}, "trials": 254, "best_seen": {"n": 12166, "win": 0.43021535426598717, "pf": 1.1943888942509129, "total": 249.60527102982905, "mean": 0.02051662592716004, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: capitulation", "passed": true, "reason": "profit fallback: below 65 % winners, PF \u22651.3 on both halves", "selection": {"n": 7437, "win": 0.4318945811483125, "pf": 1.3572278876582393, "total": 260.93817915331056, "mean": 0.03508648368338182, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 3616, "win": 0.46432522123893805, "pf": 1.369075903134249, "total": 131.28512194135692, "mean": 0.036306726200596495, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "params": {"hold_min": 120, "line": 0.02170279622077948, "thr": {"neg_ret_1m": 0.000229347959975712, "top_sell_share_1m": 0.9872784614562988, "buyers_ret": -0.9555114507675171}, "high": null}, "trials": 326, "best_seen": {"n": 25141, "win": 0.43594129111809393, "pf": 1.1396617105231777, "total": 300.13287771363275, "mean": 0.011937984873856758, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "strategy: breakout", "passed": false, "reason": "fails on the evaluation half", "selection": {"n": 806, "win": 0.41811414392059554, "pf": 1.3704845024089776, "total": 37.82851162024781, "mean": 0.046933637245965026, "weeks": 10, "weeks_pos": 5, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 577, "win": 0.6949740034662045, "pf": 1.1277748365607487, "total": 13.312870207955712, "mean": 0.023072565351743003, "weeks": 10, "weeks_pos": 6, "sign_ok": false, "admissible": false, "base_ok": false}, "params": {"hold_min": 240, "line": 0.07295550033450127, "thr": {"org_imb_5m": null, "buyers_slope": null, "log_org_vol_15m": 2.082864999771118}, "high": "dd_1h"}, "trials": 334, "best_seen": {"n": 5418, "win": 0.3549280177187154, "pf": 1.1214306198017956, "total": 68.40240365069758, "mean": 0.012625028359301878, "weeks": 10, "weeks_pos": 7, "sign_ok": false, "admissible": false, "base_ok": false}}, {"name": "combination", "passed": true, "reason": "''score'' made more on the selection half", "selection": null, "evaluation": null, "params": {"rule": "score"}, "trials": 0, "best_seen": null}, {"name": "meta-label: ev", "passed": true, "reason": "improves the book on the selection half, passes on the evaluation half and beats random removal", "selection": {"n": 8538, "win": 0.44038416490981497, "pf": 1.37928681358971, "total": 328.3758206743696, "mean": 0.03846050839474931, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 4178, "win": 0.4691239827668741, "pf": 1.371375156148074, "total": 161.47078334595426, "mean": 0.03864786580803118, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 161.47079467773438, "random_total": 144.83329350066924, "removed": 135}}, "params": {"cut": 0.4034840017557144, "line": 0.048561317734419324}, "trials": 210, "best_seen": {"n": 13052, "win": 0.46276432730615996, "pf": 1.1850528442487627, "total": 220.3587098019707, "mean": 0.016883137435026868, "weeks": 9, "weeks_pos": 8, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "meta-label: capitulation", "passed": false, "reason": "rejected (selection, evaluation or random-removal test)", "selection": {"n": 8867, "win": 0.4710725160708244, "pf": 1.3000808722110484, "total": 238.6872466020177, "mean": 0.02691860230089294, "weeks": 9, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 4930, "win": 0.4776876267748479, "pf": 1.308051162803864, "total": 146.70473431929713, "mean": 0.02975755260026311, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 146.70474243164062, "random_total": 161.47078334595426, "removed": 0}}, "params": {"cut": 0.45691339671611786, "line": 0.011362157016992563}, "trials": 210, "best_seen": {"n": 11365, "win": 0.46889573251209854, "pf": 1.2577842532528587, "total": 220.86113635363654, "mean": 0.01943344798536177, "weeks": 9, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": false}}, {"name": "dump veto", "passed": true, "reason": "improves the book on the selection half, passes on the evaluation half and beats random removal", "selection": {"n": 8374, "win": 0.44124671602579413, "pf": 1.4118872282744042, "total": 345.0511686700811, "mean": 0.041205059549806675, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 4058, "win": 0.47042878265155247, "pf": 1.3931662708134354, "total": 161.12030342951766, "mean": 0.039704362599684, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 161.12030029296875, "random_total": 156.41600681219424, "removed": 120}}, "params": {"dump": -0.3325011432170868, "cut": 0.1505558244884014}, "trials": 56, "best_seen": {"n": 8374, "win": 0.44124671602579413, "pf": 1.4118872282744042, "total": 345.0511686700811, "mean": 0.041205059549806675, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}}, {"name": "hour and market gates", "passed": true, "reason": "improves the book, passes on the evaluation half and beats random removal", "selection": {"n": 6652, "win": 0.45625375826819003, "pf": 1.5568584699178285, "total": 359.86430143312464, "mean": 0.054098662271967024, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 3172, "win": 0.4735182849936948, "pf": 1.3978238516274475, "total": 128.41733735687012, "mean": 0.04048465868753787, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true, "random_removal": {"filtered_total": 128.41734313964844, "random_total": 123.31061138069417, "removed": 886}}, "params": {"ev": {"hours": [3], "regimes": [[11.075756072998047, 11.180865287780762]]}, "capitulation": {"hours": [2, 4, 10, 12, 13, 16, 17, 20, 21, 23], "regimes": [[11.4250337600708, 11.614822673797608]]}}, "trials": 0, "best_seen": null}], "strategies": {"ev": {"line": 0.048561317734419324, "hold_min": 120, "thr": {}, "high": null, "hours": [3], "regimes": [[11.075756072998047, 11.180865287780762]], "sizing": [{"lo": 0.0, "n": 1214, "mean": 0.043260690297942606, "win": 0.44233937397034595, "kelly": 0.25}, {"lo": 0.003545435145497322, "n": 1214, "mean": 0.030302551236103654, "win": 0.46622734761120266, "kelly": 0.16}, {"lo": 0.009100683033466339, "n": 1214, "mean": 0.030798823842828667, "win": 0.44069192751235586, "kelly": 0.17}, {"lo": 0.021355299279093742, "n": 1214, "mean": 0.08868562178421593, "win": 0.4242174629324547, "kelly": 0.06}], "selection": {"n": 4856, "win": 0.4433690280065898, "pf": 1.4199964576000768, "total": 234.3598922135643, "mean": 0.048261921790272715, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}, "evaluation": {"n": 1687, "win": 0.4866627148784825, "pf": 1.3549911389698504, "total": 72.38500758860027, "mean": 0.04290753265477195, "weeks": 10, "weeks_pos": 8, "sign_ok": false, "admissible": false, "base_ok": true}}, "capitulation": {"line": 0.02170279622077948, "hold_min": 120, "thr": {"neg_ret_1m": 0.000229347959975712, "top_sell_share_1m": 0.9872784614562988, "buyers_ret": -0.9555114507675171}, "high": null, "hours": [2, 4, 10, 12, 13, 16, 17, 20, 21, 23], "regimes": [[11.4250337600708, 11.614822673797608]], "sizing": [{"lo": 0.0, "n": 1859, "mean": 0.02187701561079286, "win": 0.45293168370091447, "kelly": 0.26}, {"lo": 0.00394807755947113, "n": 1859, "mean": 0.06614231310249848, "win": 0.4190424959655729, "kelly": 0.11}, {"lo": 0.010377474129199982, "n": 1859, "mean": 0.026808915657258844, "win": 0.4389456697149005, "kelly": 0.22}, {"lo": 0.02529596909880638, "n": 1860, "mean": 0.02552283487551493, "win": 0.4166666666666667, "kelly": 0.11}], "selection": {"n": 7437, "win": 0.4318945811483125, "pf": 1.3572278876582393, "total": 260.93817915331056, "mean": 0.03508648368338182, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}, "evaluation": {"n": 3616, "win": 0.46432522123893805, "pf": 1.369075903134249, "total": 131.28512194135692, "mean": 0.036306726200596495, "weeks": 10, "weeks_pos": 9, "sign_ok": true, "admissible": false, "base_ok": true}}}, "groups": ["rug"], "combine": "score", "veto": {"level_index": 1, "dump": -0.3325011432170868, "cut": 0.1505558244884014, "levels": [-0.43370068073272705, -0.3325011432170868, -0.21201005578041077, -0.13198548555374146], "selection": {"n": 8374, "win": 0.44124671602579413, "pf": 1.4118872282744042, "total": 345.0511686700811, "mean": 0.041205059549806675, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}, "trials": 56, "best_seen": {"n": 8374, "win": 0.44124671602579413, "pf": 1.4118872282744042, "total": 345.0511686700811, "mean": 0.041205059549806675, "weeks": 10, "weeks_pos": 10, "sign_ok": true, "admissible": false, "base_ok": true}}, "fallback": true, "line": 0.048561317734419324, "costs": "real fees (pool fee by market cap, Jupiter 10 bps, network fee) + impact", "data": {"agg": 4, "features": 3, "costs": "real-fees-1", "selector": "multi-1", "cols": "1ff02dc9", "skill": "h10-L30", "meta": "rug-1"}, "deployable": true, "deploy_reason": "its backtest made +4.05% per trade after costs (random -4.68%); profit fallback (below 65 % winners)", "sizing": [{"lo": 0.0, "n": 1214, "mean": 0.043260690297942606, "win": 0.44233937397034595, "kelly": 0.25}, {"lo": 0.003545435145497322, "n": 1214, "mean": 0.030302551236103654, "win": 0.46622734761120266, "kelly": 0.16}, {"lo": 0.009100683033466339, "n": 1214, "mean": 0.030798823842828667, "win": 0.44069192751235586, "kelly": 0.17}, {"lo": 0.021355299279093742, "n": 1214, "mean": 0.08868562178421593, "win": 0.4242174629324547, "kelly": 0.06}], "model": {"kind": "stack", "min_age_h": 6.0, "min_resq_sol": 10.0, "min_vol_15m_sol": 5.0}, "rows": 29361102, "days": 153, "first_day": "2026-04-19", "last_day": "2026-09-18"}') ON CONFLICT (id) DO NOTHING; +INSERT INTO brain_snapshots (id, ts, run_id, beat_id, path, sha256, kind, promoted_at, promoted_by, note) VALUES ('62', '2026-09-21 16:02:46.170785+00:00', NULL, NULL, '/app/data/brain/policies/fly_20260921T160246Z.pt', 'd18ef071d4106cea6846b494340ce9518e5e5ac55cde31b79670fbead406c744', 'fly_selector', NULL, NULL, '{"S": "2026-09-21", "train_through": "2026-09-12", "calibration_days": ["2026-09-14", "2026-09-20"], "line": 0.04060962647199624, "sizing": [{"lo": 0.0, "n": 81, "mean": 0.03765792501975357, "win": 0.49382716049382713, "kelly": 0.26}, {"lo": 0.0020544106140733473, "n": 81, "mean": 0.002718701303281166, "win": 0.4567901234567901, "kelly": 0.02}, {"lo": 0.006942897662520479, "n": 81, "mean": 0.0007167349764962255, "win": 0.48148148148148145, "kelly": 0.01}, {"lo": 0.015385258570313524, "n": 81, "mean": 0.07037295356829777, "win": 0.6296296296296297, "kelly": 0.5700000000000001}], "lines": {"ev": 0.04060962647199624, "capitulation": 0.04896915547549754}, "calibration": {"trades": 325, "mean": 0.027888946775346993, "total": 9.063907701987773, "per_strategy": {"ev": {"line": 0.04060962647199624, "trades": 324, "mean": 0.027866578716957184, "total": 9.028771504294127, "hold_min": 120}, "capitulation": {"line": 0.04896915547549754, "trades": 110, "mean": 0.04093937277370556, "total": 4.503331005107611, "hold_min": 120}}}, "strategies": ["ev", "capitulation"], "rules": {"ev": {"thr": {}, "high": null, "hold_min": 120}, "capitulation": {"thr": {"neg_ret_1m": 0.000229347959975712, "top_sell_share_1m": 0.9872784614562988, "buyers_ret": -0.9555114507675171}, "high": null, "hold_min": 120}}, "diagnostics": {"mbon_share": 0.3961058327857704, "mbon_saturated": 0.010416666977107525, "kc_active": 0.09959564357995987, "rows": 50000, "slope": 1.005056561531653, "rank_corr": 0.8535549287862947}, "gates_ok": true, "gate_failures": [], "fit": {"epochs": [{"epoch": 0, "mse": 0.3367231594663317, "secs": 3108.421549797058, "rows": 3912704, "val_mse": 0.24399760291454034, "val_se": 0.0038252531609198506}, {"epoch": 1, "mse": 0.20434178183626922, "secs": 6239.6601638793945, "rows": 3912704, "val_mse": 0.21627898232944742, "val_se": 0.0041017738607596445}, {"epoch": 2, "mse": 0.1676639474282961, "secs": 9364.127672672272, "rows": 3912704, "val_mse": 0.239599813865461, "val_se": 0.003708129521054708}], "stopped": false, "rows": 3913129, "training_rows": 22262569, "top_weight": 10.0, "strategies": ["ev", "capitulation"], "epochs_run": 3, "epoch_cap": 8, "val_rows": 200000, "val_mse": 0.21627898232944742, "early_stop": "epoch 2 moved the held-out error by -0.0233, inside its standard error 0.0037"}, "teacher": "deployed selector #59: ev (line 0.0486, 120 min), capitulation (line 0.0217, 120 min)", "teacher_line": 0.048561317734419324, "teacher_stack": null, "teacher_snapshot": 59, "data": {"agg": 4, "features": 3, "costs": "real-fees-sized-1", "selector": "multi-1", "cols": "1ff02dc9", "skill": "h10-L30", "meta": "rug-1", "fly": "flynet-3-ch", "plastic": "mb4-ch", "teacher": "deployed-1"}}') ON CONFLICT (id) DO NOTHING; +SELECT setval('brain_snapshots_id_seq', GREATEST((SELECT max(id) FROM brain_snapshots), 1)); +INSERT INTO ui_settings (key, value) VALUES ('fly_replay', '{"S": "2026-07-27", "data": {"agg": 4, "fly": "flynet-3-ch", "cols": "1ff02dc9", "meta": "rug-1", "costs": "real-fees-sized-1", "skill": "h10-L30", "plastic": "mb4-ch", "teacher": "deployed-1", "features": 3, "selector": "multi-1"}, "secs": 12592.698919296265, "alpha": 0.0003, "chosen": {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, "frozen": {"n": 2048, "pf": 1.0961381659648606, "win": 0.44140625, "days": 29, "mean": 0.010135887190699577, "median": -0.019196834415197372, "random": {"n": 39636, "pf": 0.46682219913822093, "win": 0.4265314360682208, "mean": -0.05251633748412132, "median": -0.0052637094631791115}, "days_positive": 18}, "passed": true, "random": {"n": 32155, "pf": 0.46781214203894617, "win": 0.42220494479863163, "mean": -0.052701301872730255, "median": -0.005381436087191105}, "reason": "its backtest made +1.52% per trade after costs (random -5.27%)", "configs": [{"mean": 0.016807934269309044, "alpha": 0.0, "total": 65.02989959716797, "config": 0, "trades": 3869, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.01756637543439865, "alpha": 0.0001, "total": 63.76594543457031, "config": 1, "trades": 3630, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020793743431568146, "alpha": 0.0001, "total": 68.8896713256836, "config": 2, "trades": 3313, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 3.0}, {"mean": 0.02111992798745632, "alpha": 0.0001, "total": 61.73354721069336, "config": 3, "trades": 2923, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 7.0}, {"mean": 0.020111948251724243, "alpha": 0.0001, "total": 50.62177276611328, "config": 4, "trades": 2517, "governance": {"ic": 0, "any": 68, "drift": 0, "checks": 1365, "shadow": 68}, "half_life_days": 30.0}, {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020104866474866867, "alpha": 0.0003, "total": 52.694854736328125, "config": 6, "trades": 2621, "governance": {"ic": 0, "any": 34, "drift": 0, "checks": 1365, "shadow": 34}, "half_life_days": 3.0}, {"mean": 0.015242534689605236, "alpha": 0.0003, "total": 31.61301612854004, "config": 7, "trades": 2074, "governance": {"ic": 0, "any": 93, "drift": 0, "checks": 1365, "shadow": 93}, "half_life_days": 7.0}, {"mean": 0.014115910045802593, "alpha": 0.0003, "total": 23.277135848999023, "config": 8, "trades": 1649, "governance": {"ic": 0, "any": 111, "drift": 0, "checks": 1365, "shadow": 111}, "half_life_days": 30.0}, {"mean": 0.014811468310654163, "alpha": 0.001, "total": 39.4577522277832, "config": 9, "trades": 2664, "governance": {"ic": 0, "any": 74, "drift": 0, "checks": 1365, "shadow": 74}, "half_life_days": 1.0}, {"mean": 0.011060998775064945, "alpha": 0.001, "total": 18.847942352294922, "config": 10, "trades": 1704, "governance": {"ic": 0, "any": 139, "drift": 0, "checks": 1365, "shadow": 139}, "half_life_days": 3.0}, {"mean": 0.002477346919476986, "alpha": 0.001, "total": 3.146230697631836, "config": 11, "trades": 1270, "governance": {"ic": 0, "any": 184, "drift": 0, "checks": 1365, "shadow": 184}, "half_life_days": 7.0}, {"mean": 0.018671352416276932, "alpha": 0.001, "total": 19.77296257019043, "config": 12, "trades": 1059, "governance": {"ic": 0, "any": 144, "drift": 0, "checks": 1365, "shadow": 144}, "half_life_days": 30.0}, {"mean": 0.01424670871347189, "alpha": 0.003, "total": 28.10875701904297, "config": 13, "trades": 1973, "governance": {"ic": 0, "any": 203, "drift": 0, "checks": 1365, "shadow": 203}, "half_life_days": 1.0}, {"mean": 0.027894049882888794, "alpha": 0.003, "total": 38.716941833496094, "config": 14, "trades": 1388, "governance": {"ic": 0, "any": 71, "drift": 0, "checks": 1365, "shadow": 71}, "half_life_days": 3.0}, {"mean": 0.01914161443710327, "alpha": 0.003, "total": 18.816207885742188, "config": 15, "trades": 983, "governance": {"ic": 0, "any": 112, "drift": 0, "checks": 1365, "shadow": 112}, "half_life_days": 7.0}, {"mean": 0.028120551258325577, "alpha": 0.003, "total": 22.524560928344727, "config": 16, "trades": 801, "governance": {"ic": 0, "any": 196, "drift": 0, "checks": 1365, "shadow": 196}, "half_life_days": 30.0}], "per_day": {"2026-08-24": {"n": 96, "pf": 1.0117903231205847, "win": 0.4479166666666667, "mean": 0.0016304353484883904, "median": -0.04346885532140732}, "2026-08-25": {"n": 92, "pf": 1.5657616101643343, "win": 0.5217391304347826, "mean": 0.06755560636520386, "median": 0.007279674988240004}, "2026-08-26": {"n": 263, "pf": 1.165655794982386, "win": 0.44866920152091255, "mean": 0.014974601566791534, "median": -0.012878135778009892}, "2026-08-27": {"n": 170, "pf": 0.6946020453820136, "win": 0.40588235294117647, "mean": -0.03204870969057083, "median": -0.03053605556488037}, "2026-08-28": {"n": 56, "pf": 1.4862921757026535, "win": 0.5178571428571429, "mean": 0.06174355000257492, "median": 0.010778131894767284}, "2026-08-29": {"n": 213, "pf": 1.011389332575889, "win": 0.43661971830985913, "mean": 0.0011059067910537124, "median": -0.026353871449828148}, "2026-08-30": {"n": 50, "pf": 1.3834646670134814, "win": 0.38, "mean": 0.0530250258743763, "median": -0.08351919054985046}, "2026-08-31": {"n": 41, "pf": 1.6108359675631871, "win": 0.43902439024390244, "mean": 0.07970548421144485, "median": -0.03631661832332611}, "2026-09-01": {"n": 43, "pf": 0.7714416116969681, "win": 0.4883720930232558, "mean": -0.031781017780303955, "median": -0.0033531002700328827}, "2026-09-02": {"n": 31, "pf": 1.8960080329396884, "win": 0.5161290322580645, "mean": 0.09989593923091888, "median": 0.056023046374320984}, "2026-09-03": {"n": 39, "pf": 0.47435046931837, "win": 0.38461538461538464, "mean": -0.0845755785703659, "median": -0.10096535086631775}, "2026-09-04": {"n": 44, "pf": 2.124818757306114, "win": 0.36363636363636365, "mean": 0.17666810750961304, "median": -0.04468941688537598}, "2026-09-05": {"n": 33, "pf": 3.159536891732615, "win": 0.45454545454545453, "mean": 0.11582708358764648, "median": -0.02216796576976776}, "2026-09-06": {"n": 34, "pf": 0.8038220998481828, "win": 0.47058823529411764, "mean": -0.028142571449279785, "median": -0.04691909998655319}, "2026-09-07": {"n": 25, "pf": 0.538075679014026, "win": 0.36, "mean": -0.07897567003965378, "median": -0.057357002049684525}, "2026-09-08": {"n": 24, "pf": 0.5229846306363845, "win": 0.4583333333333333, "mean": -0.0886785164475441, "median": -0.027842504903674126}, "2026-09-09": {"n": 6, "pf": 1.775177140941807, "win": 0.3333333333333333, "mean": 0.064696304500103, "median": -0.0772169902920723}, "2026-09-10": {"n": 10, "pf": 0.0, "win": 0.0, "mean": -0.4276605248451233, "median": -0.40527138113975525}, "2026-09-11": {"n": 9, "pf": 0.4853454520861287, "win": 0.3333333333333333, "mean": -0.07680502533912659, "median": -0.1473425328731537}, "2026-09-12": {"n": 104, "pf": 0.5093969547294807, "win": 0.4326923076923077, "mean": -0.043387047946453094, "median": -0.009646685793995857}, "2026-09-13": {"n": 13, "pf": 0.9683888718696267, "win": 0.46153846153846156, "mean": -0.003039261559024453, "median": -0.011067567393183708}, "2026-09-14": {"n": 19, "pf": 2.3648619908537536, "win": 0.631578947368421, "mean": 0.19480086863040924, "median": 0.07961719483137131}, "2026-09-15": {"n": 19, "pf": 1.9207007670652974, "win": 0.42105263157894735, "mean": 0.12944886088371277, "median": -0.0763954222202301}, "2026-09-16": {"n": 20, "pf": 1.989758339975232, "win": 0.6, "mean": 0.07185642421245575, "median": 0.06811605393886566}, "2026-09-17": {"n": 23, "pf": 2.308382394662165, "win": 0.5652173913043478, "mean": 0.07706345617771149, "median": 0.08088493347167969}, "2026-09-18": {"n": 63, "pf": 1.357841853369037, "win": 0.47619047619047616, "mean": 0.030619792640209198, "median": -0.004897014237940311}, "2026-09-19": {"n": 34, "pf": 1.4133537729578933, "win": 0.4117647058823529, "mean": 0.04197608679533005, "median": -0.05088033154606819}, "2026-09-20": {"n": 40, "pf": 1.2252380260800315, "win": 0.425, "mean": 0.03793734312057495, "median": -0.07361158728599548}, "2026-09-21": {"n": 37, "pf": 1.2097114398883206, "win": 0.5675675675675675, "mean": 0.016092097386717796, "median": 0.01574813388288021}}, "learning": [{"n": 931, "day": "2026-07-27", "step": [0.0, 0.06421024, 0.06420731, 0.06420637, 0.06420581, 0.18729984, 0.18720574, 0.18719876, 0.1871946, 0.58122008, 0.57589885, 0.57586091, 0.57583875, 1.56960882, 1.57073448, 1.57049312, 1.56501226], "drift": [0.0, 0.00123, 0.00152, 0.00163, 0.00169, 0.00348, 0.00437, 0.00467, 0.00485, 0.01091, 0.01395, 0.01494, 0.01554, 0.02869, 0.03589, 0.03833, 0.03947], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 45, 45, 45], "mean_delta": [-0.039099, -0.038733, -0.038703, -0.038693, -0.038688, -0.038012, -0.037925, -0.037898, -0.037881, -0.035483, -0.035206, -0.035118, -0.035067, -0.030323, -0.029782, -0.029599, -0.029538], "mean_abs_delta": [0.189807, 0.189711, 0.189705, 0.189703, 0.189702, 0.189527, 0.18951, 0.189504, 0.189501, 0.188888, 0.188834, 0.188818, 0.188808, 0.187597, 0.187515, 0.187486, 0.187462]}, {"n": 1044, "day": "2026-07-28", "step": [0.0, 0.07277379, 0.07218625, 0.07189107, 0.07176147, 0.20257254, 0.19562552, 0.19169733, 0.19072503, 0.53300112, 0.4800792, 0.46328981, 0.45527648, 1.08817285, 1.01850277, 0.99633343, 0.98662623], "drift": [0.0, 0.00112, 0.0018, 0.0021, 0.00231, 0.00342, 0.00533, 0.00625, 0.00678, 0.00927, 0.01447, 0.0171, 0.01855, 0.02377, 0.03621, 0.04178, 0.04546], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 36, 34, 32], "mean_delta": [-0.027195, -0.026147, -0.025859, -0.025755, -0.025687, -0.02413, -0.023407, -0.023065, -0.0229, -0.01855, -0.016465, -0.015533, -0.015078, -0.006329, -0.001372, 0.000907, 0.001877], "mean_abs_delta": [0.190608, 0.190481, 0.190461, 0.190455, 0.190451, 0.190261, 0.190232, 0.190217, 0.190211, 0.189725, 0.18969, 0.189675, 0.189664, 0.189226, 0.189369, 0.189431, 0.189449]}, {"n": 863, "day": "2026-07-29", "step": [0.0, 0.06034305, 0.05845988, 0.05776155, 0.05721584, 0.16231128, 0.14620346, 0.1383591, 0.13605227, 0.41583466, 0.35392232, 0.33466695, 0.32878579, 0.90389129, 0.76926308, 0.70084347, 0.65879158], "drift": [0.0, 0.00083, 0.00173, 0.00219, 0.00252, 0.00233, 0.0047, 0.00605, 0.00698, 0.00685, 0.01407, 0.01877, 0.02179, 0.02133, 0.03601, 0.04508, 0.05245], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 18, 15, 13], "mean_delta": [-0.012039, -0.011268, -0.010674, -0.010403, -0.010234, -0.009831, -0.008321, -0.007575, -0.007109, -0.006105, -0.002358, -7e-05, 0.001157, 0.001562, 0.00964, 0.014695, 0.017686], "mean_abs_delta": [0.188678, 0.188475, 0.188319, 0.188247, 0.188202, 0.188106, 0.187716, 0.187524, 0.187409, 0.187209, 0.186402, 0.18595, 0.18572, 0.185977, 0.184891, 0.184281, 0.184005]}, {"n": 1236, "day": "2026-07-30", "step": [0.0, 0.10988152, 0.10774172, 0.10559816, 0.10466212, 0.31706643, 0.29190488, 0.28316394, 0.27648895, 0.9586995, 0.81282441, 0.75331462, 0.72414899, 2.05303031, 1.7496128, 1.59959924, 1.50315613], "drift": [0.0, 0.00353, 0.00502, 0.00575, 0.00627, 0.01019, 0.01367, 0.01591, 0.01733, 0.03045, 0.0398, 0.04493, 0.04817, 0.06619, 0.08122, 0.08852, 0.09422], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 87, 80, 79], "mean_delta": [-0.091839, -0.090642, -0.089971, -0.089642, -0.089387, -0.088579, -0.087192, -0.086045, -0.085377, -0.082853, -0.07795, -0.074143, -0.072069, -0.066969, -0.05794, -0.052086, -0.04854], "mean_abs_delta": [0.237217, 0.236632, 0.236356, 0.236219, 0.23611, 0.235612, 0.235062, 0.234589, 0.234319, 0.232781, 0.231053, 0.229731, 0.229042, 0.226722, 0.22439, 0.222879, 0.222044]}, {"n": 371, "day": "2026-07-31", "step": [0.0, 0.04234476, 0.03978243, 0.03882724, 0.03834538, 0.10608156, 0.09335886, 0.09183339, 0.08744498, 0.23009973, 0.18223433, 0.16370089, 0.15161757, 0.3607691, 0.27135012, 0.22326726, 0.20295452], "drift": [0.0, 0.00223, 0.00455, 0.00569, 0.00657, 0.00623, 0.01209, 0.01551, 0.01774, 0.01846, 0.03446, 0.04292, 0.04901, 0.03862, 0.06792, 0.08091, 0.09142], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 1, 1], "mean_delta": [-0.052608, -0.050835, -0.049833, -0.049374, -0.049009, -0.047575, -0.045204, -0.043704, -0.04279, -0.037831, -0.031311, -0.027866, -0.025479, -0.019581, -0.008466, -0.00396, -0.00054], "mean_abs_delta": [0.276031, 0.275416, 0.275123, 0.274987, 0.274879, 0.27429, 0.273585, 0.273156, 0.272904, 0.271009, 0.269555, 0.268781, 0.268237, 0.265625, 0.263527, 0.26297, 0.262476]}, {"n": 54, "day": "2026-08-01", "step": [0.0, 0.00741917, 0.00741597, 0.00676187, 0.00605164, 0.0194067, 0.01646108, 0.01194924, 0.0110565, 0.03528636, 0.01351464, 0.0101124, 0.00974537, 0.03056685, 0.01951861, 0.0138812, 0.01356826], "drift": [0.0, 0.00138, 0.00387, 0.00522, 0.00637, 0.00379, 0.01028, 0.0145, 0.01744, 0.01216, 0.03019, 0.04078, 0.0486, 0.02671, 0.06085, 0.0774, 0.09115], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "mean_delta": [0.021166, 0.022723, 0.024943, 0.026034, 0.026946, 0.025496, 0.030874, 0.034262, 0.036473, 0.034028, 0.047387, 0.055297, 0.060891, 0.046447, 0.071219, 0.081994, 0.090024], "mean_abs_delta": [0.26533, 0.265417, 0.265613, 0.265685, 0.265759, 0.265548, 0.265895, 0.26615, 0.266267, 0.265943, 0.266645, 0.267432, 0.267988, 0.266451, 0.269451, 0.271657, 0.273752]}, {"n": 242, "day": "2026-08-02", "step": [0.0, 0.01872033, 0.01859977, 0.0181289, 0.01802677, 0.0558438, 0.05201761, 0.04925188, 0.04794973, 0.17378149, 0.14824457, 0.12593751, 0.1140936, 0.47729315, 0.33874456, 0.24572129, 0.17729059], "drift": [0.0, 0.00088, 0.00314, 0.00483, 0.00643, 0.00248, 0.00828, 0.01332, 0.01741, 0.00756, 0.02334, 0.03559, 0.04639, 0.01838, 0.04457, 0.06531, 0.0875], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 3, 2], "mean_delta": [-0.036087, -0.035588, -0.034009, -0.032926, -0.031938, -0.034705, -0.030626, -0.027369, -0.024855, -0.031729, -0.020271, -0.012292, -0.005555, -0.026377, -0.005737, 0.007242, 0.018867], "mean_abs_delta": [0.214218, 0.214074, 0.213654, 0.213363, 0.213101, 0.213812, 0.212741, 0.211945, 0.211312, 0.212985, 0.210212, 0.20864, 0.207606, 0.211498, 0.207818, 0.206297, 0.20526]}, {"n": 508, "day": "2026-08-03", "step": [0.0, 0.04134416, 0.0371876, 0.03266551, 0.0305573, 0.11778908, 0.08932215, 0.08048781, 0.07565604, 0.31825422, 0.24333699, 0.19889814, 0.18343096, 0.76600191, 0.54425102, 0.4655616, 0.33341111], "drift": [0.0, 0.00086, 0.00272, 0.0045, 0.00642, 0.00251, 0.0068, 0.01191, 0.0169, 0.00584, 0.01774, 0.03055, 0.04362, 0.01263, 0.03241, 0.05504, 0.08091], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 13, 5], "mean_delta": [-0.011377, -0.010753, -0.009203, -0.007918, -0.006615, -0.009584, -0.005852, -0.001992, 0.001368, -0.007048, 0.003835, 0.013027, 0.021712, -0.001486, 0.016475, 0.030461, 0.046891], "mean_abs_delta": [0.206632, 0.206521, 0.206251, 0.206031, 0.205803, 0.20633, 0.205706, 0.205081, 0.204565, 0.205893, 0.204323, 0.203515, 0.203302, 0.204999, 0.203303, 0.203833, 0.206042]}, {"n": 692, "day": "2026-08-04", "step": [0.0, 0.04460403, 0.03859602, 0.03441494, 0.03139943, 0.12362088, 0.09626486, 0.06885854, 0.05953138, 0.38367219, 0.20308522, 0.16834852, 0.131699, 1.01038427, 0.50971773, 0.37668987, 0.27438032], "drift": [0.0, 0.00046, 0.00194, 0.0039, 0.006, 0.00136, 0.00491, 0.0109, 0.01666, 0.00421, 0.01525, 0.02819, 0.0435, 0.01111, 0.02828, 0.05228, 0.08124], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 2, 1], "mean_delta": [0.013699, 0.013807, 0.015059, 0.016398, 0.017741, 0.014045, 0.017069, 0.021082, 0.02475, 0.013845, 0.023644, 0.032565, 0.042299, 0.013797, 0.02944, 0.045704, 0.063913], "mean_abs_delta": [0.183171, 0.183126, 0.183024, 0.182942, 0.18287, 0.183048, 0.182837, 0.182735, 0.182695, 0.182865, 0.182626, 0.182803, 0.1836, 0.182576, 0.182491, 0.184005, 0.187536]}, {"n": 1128, "day": "2026-08-05", "step": [0.0, 0.05903479, 0.05507833, 0.05134258, 0.04764679, 0.17682159, 0.15135335, 0.12787613, 0.11643432, 0.6171573, 0.40581984, 0.34735324, 0.30416585, 1.83372795, 1.00153648, 0.85559196, 0.64182627], "drift": [0.0, 0.00107, 0.00083, 0.00233, 0.00462, 0.00313, 0.00239, 0.00694, 0.01331, 0.01023, 0.0079, 0.01759, 0.03386, 0.01932, 0.01897, 0.03149, 0.05838], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 44, 39, 31], "mean_delta": [0.045258, 0.044848, 0.04591, 0.047412, 0.04904, 0.044154, 0.046767, 0.051566, 0.05613, 0.041694, 0.051311, 0.060843, 0.072213, 0.041475, 0.054393, 0.067997, 0.08784], "mean_abs_delta": [0.173116, 0.173179, 0.173065, 0.172906, 0.172747, 0.173298, 0.173014, 0.172589, 0.172375, 0.173776, 0.17276, 0.172651, 0.173458, 0.173557, 0.173238, 0.173589, 0.176942]}, {"n": 1058, "day": "2026-08-06", "step": [0.0, 0.03945935, 0.03698631, 0.03220626, 0.02746376, 0.12985755, 0.11435575, 0.07488091, 0.05460495, 0.63672671, 0.3500779, 0.1885276, 0.11419593, 1.80572728, 1.57435511, 0.62120528, 0.28846189], "drift": [0.0, 0.00064, 0.00098, 0.00238, 0.00474, 0.00196, 0.00278, 0.00677, 0.01397, 0.00734, 0.00897, 0.01889, 0.03563, 0.01789, 0.0232, 0.03536, 0.06194], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 40, 17, 6], "mean_delta": [0.003393, 0.002579, 0.003265, 0.004759, 0.006634, 0.00097, 0.002713, 0.007733, 0.01337, -0.00438, 0.003119, 0.014892, 0.02849, -0.00547, -0.003837, 0.016483, 0.041025], "mean_abs_delta": [0.168503, 0.168553, 0.168513, 0.168429, 0.168352, 0.168686, 0.168561, 0.168344, 0.16832, 0.169263, 0.168633, 0.168449, 0.169828, 0.169664, 0.169537, 0.168916, 0.172572]}, {"n": 892, "day": "2026-08-07", "step": [0.0, 0.04321418, 0.04232384, 0.03822837, 0.03495312, 0.13560243, 0.12616215, 0.09834065, 0.08321215, 0.49905728, 0.35899078, 0.26573406, 0.22310838, 0.96535605, 1.01828484, 0.73811684, 0.58092871], "drift": [0.0, 0.0006, 0.00106, 0.00252, 0.00501, 0.00186, 0.00299, 0.00707, 0.01449, 0.00655, 0.01045, 0.01962, 0.03622, 0.01756, 0.0272, 0.03745, 0.06359], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, 11], "mean_delta": [-0.026824, -0.026974, -0.026615, -0.025306, -0.023529, -0.027278, -0.026385, -0.022427, -0.017015, -0.028057, -0.023258, -0.014869, -0.0028, -0.020316, -0.019829, -0.007844, 0.009888], "mean_abs_delta": [0.165735, 0.165729, 0.165702, 0.16562, 0.165527, 0.165724, 0.165654, 0.165483, 0.165431, 0.165654, 0.165466, 0.165447, 0.166151, 0.165184, 0.165209, 0.165848, 0.168385]}, {"n": 1076, "day": "2026-08-08", "step": [0.0, 0.07095081, 0.07041009, 0.06571411, 0.05991169, 0.21319146, 0.20814377, 0.1705785, 0.13642696, 0.70387371, 0.57735057, 0.42904051, 0.32422108, 1.63103617, 1.54005343, 1.00802515, 0.76446274], "drift": [0.0, 0.00139, 0.00201, 0.00348, 0.00597, 0.00421, 0.00595, 0.00945, 0.01669, 0.01451, 0.01815, 0.02519, 0.04134, 0.03226, 0.04307, 0.04922, 0.07187], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 50, 29, 15], "mean_delta": [-0.031996, -0.031957, -0.031714, -0.030445, -0.028644, -0.031818, -0.031171, -0.027531, -0.022065, -0.03076, -0.027356, -0.019525, -0.007765, -0.026253, -0.024635, -0.010153, 0.006282], "mean_abs_delta": [0.199058, 0.199035, 0.198997, 0.198821, 0.198581, 0.19898, 0.198881, 0.198421, 0.197903, 0.198693, 0.198275, 0.19768, 0.197643, 0.197837, 0.197494, 0.197324, 0.198843]}, {"n": 868, "day": "2026-08-09", "step": [0.0, 0.06866672, 0.0674683, 0.06261659, 0.0572501, 0.19572992, 0.18850666, 0.16446996, 0.13726034, 0.57460789, 0.53087509, 0.43740982, 0.34744486, 1.49970195, 1.28868771, 1.09052244, 0.83394681], "drift": [0.0, 0.00092, 0.00141, 0.00246, 0.00496, 0.00263, 0.00422, 0.00675, 0.01374, 0.00836, 0.01363, 0.01823, 0.03352, 0.0207, 0.03373, 0.04033, 0.06014], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 48, 38, 26], "mean_delta": [0.045081, 0.045225, 0.045497, 0.046645, 0.048417, 0.045572, 0.046402, 0.049269, 0.054513, 0.047274, 0.049228, 0.05597, 0.068074, 0.049162, 0.053368, 0.062153, 0.07952], "mean_abs_delta": [0.223275, 0.223163, 0.223108, 0.222843, 0.222452, 0.222937, 0.222776, 0.222128, 0.221108, 0.22213, 0.221745, 0.220548, 0.219228, 0.220893, 0.220428, 0.219255, 0.218497]}, {"n": 750, "day": "2026-08-10", "step": [0.0, 0.05472073, 0.05364319, 0.04933666, 0.04520726, 0.16364656, 0.1575568, 0.13419577, 0.10970383, 0.5261399, 0.49561711, 0.3807543, 0.30355334, 1.53532154, 1.38939052, 1.08685966, 0.78345868], "drift": [0.0, 0.0007, 0.00148, 0.00293, 0.00562, 0.00219, 0.00443, 0.00785, 0.01565, 0.00741, 0.01389, 0.02234, 0.03848, 0.01995, 0.03779, 0.05147, 0.07327], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 44, 35, 20], "mean_delta": [-0.038261, -0.038247, -0.038023, -0.036867, -0.034865, -0.038081, -0.037483, -0.034662, -0.02846, -0.036775, -0.035351, -0.027956, -0.015405, -0.03419, -0.029574, -0.020264, -0.004209], "mean_abs_delta": [0.197346, 0.19733, 0.197299, 0.197197, 0.197041, 0.197289, 0.197205, 0.197002, 0.19669, 0.197076, 0.196911, 0.196633, 0.196898, 0.196582, 0.195933, 0.196452, 0.197811]}, {"n": 783, "day": "2026-08-11", "step": [0.0, 0.05639539, 0.05519085, 0.05250994, 0.0479547, 0.16113087, 0.153371, 0.13901212, 0.11707776, 0.46874315, 0.42715407, 0.35583433, 0.32026198, 1.04951349, 0.93101532, 0.88466587, 0.79205738], "drift": [0.0, 0.00075, 0.00164, 0.00333, 0.00616, 0.00249, 0.00511, 0.00897, 0.01773, 0.00797, 0.0169, 0.02615, 0.0421, 0.02123, 0.03919, 0.05574, 0.07721], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 18, 13, 10], "mean_delta": [-0.017449, -0.016968, -0.01666, -0.015667, -0.013979, -0.015814, -0.01491, -0.012715, -0.007314, -0.011976, -0.008872, -0.003129, 0.00642, -0.002249, 0.004524, 0.012359, 0.023258], "mean_abs_delta": [0.223001, 0.222889, 0.222832, 0.222668, 0.2224, 0.222629, 0.222472, 0.22216, 0.221525, 0.221802, 0.221368, 0.220954, 0.220967, 0.220241, 0.21992, 0.220531, 0.221761]}, {"n": 737, "day": "2026-08-12", "step": [0.0, 0.05897562, 0.05788302, 0.05464898, 0.0493692, 0.16978297, 0.15688009, 0.13920504, 0.112585, 0.47220465, 0.39037227, 0.34833647, 0.29217996, 1.05956363, 0.93436273, 0.77778303, 0.59210142], "drift": [0.0, 0.00193, 0.00289, 0.00468, 0.00769, 0.00563, 0.00849, 0.01268, 0.02191, 0.01657, 0.02624, 0.0356, 0.05084, 0.0402, 0.05596, 0.07187, 0.09195], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 14, 13], "mean_delta": [-0.070931, -0.070332, -0.069889, -0.068616, -0.066434, -0.069028, -0.067488, -0.064536, -0.057181, -0.064527, -0.058243, -0.05118, -0.039822, -0.0522, -0.04429, -0.032432, -0.017798], "mean_abs_delta": [0.21199, 0.211742, 0.211594, 0.211213, 0.210562, 0.211234, 0.210759, 0.209887, 0.207963, 0.209641, 0.207901, 0.206368, 0.204237, 0.206052, 0.204262, 0.202601, 0.201103]}, {"n": 911, "day": "2026-08-13", "step": [0.0, 0.07731382, 0.07438362, 0.06862943, 0.06052568, 0.20060044, 0.17363393, 0.15290134, 0.12778348, 0.47968175, 0.39746136, 0.34151492, 0.28810784, 0.95362768, 0.76048055, 0.63285901, 0.47991281], "drift": [0.0, 0.00205, 0.00341, 0.00526, 0.00838, 0.00603, 0.00985, 0.01463, 0.02425, 0.01872, 0.02971, 0.04066, 0.05536, 0.04117, 0.06156, 0.07875, 0.09625], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 24, 22, 19], "mean_delta": [-0.009271, -0.008132, -0.007288, -0.005889, -0.00358, -0.005773, -0.003224, 0.000174, 0.007648, 0.001879, 0.009824, 0.017645, 0.028744, 0.017, 0.03027, 0.041444, 0.053666], "mean_abs_delta": [0.213292, 0.21315, 0.213064, 0.212925, 0.21271, 0.212861, 0.212615, 0.212311, 0.211911, 0.211981, 0.211575, 0.211535, 0.212126, 0.21108, 0.211846, 0.213461, 0.21583]}, {"n": 1017, "day": "2026-08-14", "step": [0.0, 0.07309305, 0.0708886, 0.06746373, 0.06293516, 0.20178701, 0.18496513, 0.16956349, 0.14622314, 0.54395029, 0.47835179, 0.42497278, 0.3933873, 1.29079118, 1.11832579, 1.00047263, 0.93534584], "drift": [0.0, 0.00171, 0.00352, 0.00573, 0.00941, 0.00537, 0.01105, 0.01672, 0.0267, 0.01796, 0.03158, 0.04416, 0.06072, 0.03368, 0.06324, 0.08527, 0.10681], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 28, 27, 24], "mean_delta": [-0.033997, -0.033083, -0.032061, -0.030691, -0.028448, -0.030913, -0.027661, -0.024266, -0.017608, -0.022265, -0.014321, -0.006806, 0.002499, -0.008865, 0.006105, 0.01656, 0.025832], "mean_abs_delta": [0.237865, 0.237737, 0.237584, 0.237392, 0.237101, 0.23747, 0.237036, 0.236725, 0.236332, 0.23671, 0.236348, 0.236374, 0.237103, 0.236494, 0.23732, 0.238586, 0.239962]}, {"n": 782, "day": "2026-08-15", "step": [0.0, 0.05105143, 0.04709918, 0.04199555, 0.03510073, 0.13004292, 0.1017861, 0.08234778, 0.0686654, 0.28074686, 0.22478825, 0.18507749, 0.14782099, 0.65555293, 0.46043469, 0.34151427, 0.23708264], "drift": [0.0, 0.00118, 0.00309, 0.0054, 0.00937, 0.00334, 0.0091, 0.01512, 0.02533, 0.00953, 0.02327, 0.03684, 0.05429, 0.01554, 0.04102, 0.06802, 0.09635], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7, 4, 2], "mean_delta": [-0.021645, -0.02061, -0.019244, -0.017492, -0.014532, -0.01839, -0.014151, -0.009748, -0.001841, -0.011884, -0.001369, 0.008715, 0.021385, -0.005379, 0.015963, 0.033387, 0.049669], "mean_abs_delta": [0.181345, 0.181145, 0.18091, 0.180626, 0.18017, 0.180769, 0.180115, 0.179624, 0.179017, 0.179797, 0.178949, 0.178738, 0.17941, 0.179029, 0.178991, 0.180583, 0.18319]}, {"n": 756, "day": "2026-08-16", "step": [0.0, 0.04780883, 0.04495553, 0.03995133, 0.03440757, 0.13295923, 0.10577023, 0.08685807, 0.0707289, 0.34714333, 0.25140565, 0.19970523, 0.15146923, 0.91328464, 0.58876806, 0.40649874, 0.28196843], "drift": [0.0, 0.00237, 0.00446, 0.00687, 0.01082, 0.0068, 0.01227, 0.01787, 0.02777, 0.01864, 0.02973, 0.04149, 0.05804, 0.04125, 0.05632, 0.07413, 0.10157], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 3, 0], "mean_delta": [-0.078631, -0.077277, -0.075808, -0.073971, -0.070846, -0.074792, -0.070724, -0.066101, -0.057693, -0.068506, -0.059447, -0.048732, -0.035315, -0.060991, -0.046701, -0.028103, -0.007035], "mean_abs_delta": [0.185639, 0.185132, 0.184619, 0.183987, 0.182947, 0.184228, 0.182886, 0.181397, 0.179011, 0.182152, 0.17957, 0.17682, 0.17409, 0.180342, 0.176742, 0.173368, 0.171709]}, {"n": 848, "day": "2026-08-17", "step": [0.0, 0.0558308, 0.05080038, 0.04544581, 0.03801651, 0.14489821, 0.11096367, 0.09344246, 0.07635285, 0.33652354, 0.2530741, 0.21792884, 0.1736102, 0.68794068, 0.54871009, 0.45554741, 0.37407675], "drift": [0.0, 0.00201, 0.00461, 0.00733, 0.01173, 0.00597, 0.01257, 0.01913, 0.02969, 0.01726, 0.0309, 0.04521, 0.06438, 0.03726, 0.06387, 0.08498, 0.11341], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 3, 1], "mean_delta": [-0.03783, -0.03619, -0.034363, -0.032334, -0.028952, -0.033246, -0.028327, -0.023285, -0.014807, -0.024236, -0.013981, -0.003355, 0.010547, -0.006998, 0.010123, 0.023707, 0.04196], "mean_abs_delta": [0.190818, 0.190362, 0.189849, 0.189308, 0.188485, 0.189594, 0.188379, 0.187249, 0.185712, 0.187543, 0.185483, 0.184258, 0.18398, 0.184215, 0.183429, 0.184094, 0.186066]}, {"n": 454, "day": "2026-08-18", "step": [0.0, 0.04264899, 0.04017388, 0.03644714, 0.03388629, 0.11645878, 0.10072397, 0.09010919, 0.07385865, 0.31588276, 0.24178546, 0.1966936, 0.15891182, 0.66131289, 0.47835376, 0.35054658, 0.22946556], "drift": [0.0, 0.00171, 0.00439, 0.00743, 0.01223, 0.00493, 0.01222, 0.01965, 0.03117, 0.01535, 0.03179, 0.0463, 0.06717, 0.03504, 0.06446, 0.08716, 0.11527], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 1, 0], "mean_delta": [-0.065056, -0.063888, -0.062266, -0.060534, -0.057736, -0.061668, -0.057426, -0.053271, -0.046597, -0.055009, -0.046043, -0.037613, -0.026444, -0.04393, -0.027723, -0.015299, -0.000573], "mean_abs_delta": [0.263289, 0.262836, 0.262254, 0.261622, 0.260693, 0.262006, 0.260626, 0.259372, 0.25756, 0.25982, 0.257263, 0.25544, 0.253375, 0.256353, 0.253108, 0.251597, 0.250156]}, {"n": 517, "day": "2026-08-19", "step": [0.0, 0.0474315, 0.04435627, 0.04203624, 0.03866324, 0.13560452, 0.1174881, 0.11074909, 0.09387226, 0.3948683, 0.32294493, 0.26289569, 0.19850024, 0.97131711, 0.63472109, 0.49923458, 0.3656384], "drift": [0.0, 0.00078, 0.00324, 0.00651, 0.01163, 0.00211, 0.00898, 0.0168, 0.02923, 0.00666, 0.02246, 0.03943, 0.06472, 0.01641, 0.0475, 0.07743, 0.10934], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 9, 4, 3], "mean_delta": [0.008636, 0.009295, 0.010871, 0.012725, 0.015575, 0.010438, 0.014863, 0.019304, 0.026056, 0.014393, 0.024352, 0.033481, 0.045581, 0.021344, 0.040674, 0.054498, 0.068201], "mean_abs_delta": [0.254107, 0.253952, 0.253632, 0.253282, 0.252768, 0.253692, 0.252861, 0.252089, 0.251132, 0.252837, 0.251192, 0.250173, 0.249123, 0.251433, 0.24925, 0.248184, 0.247839]}, {"n": 501, "day": "2026-08-20", "step": [0.0, 0.0512835, 0.04814666, 0.04310128, 0.03719705, 0.14805485, 0.12118479, 0.09997414, 0.0662997, 0.4322925, 0.30496283, 0.18570551, 0.11847896, 1.07113278, 0.52280753, 0.32041808, 0.22553451], "drift": [0.0, 0.00121, 0.00347, 0.00674, 0.01217, 0.00332, 0.00938, 0.01713, 0.02928, 0.00984, 0.0231, 0.03693, 0.06333, 0.02102, 0.04091, 0.0694, 0.10466], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 9, 3, 3], "mean_delta": [-0.074604, -0.073799, -0.072062, -0.06974, -0.066028, -0.072373, -0.067635, -0.062199, -0.053684, -0.067778, -0.057609, -0.047488, -0.030146, -0.060298, -0.044217, -0.025152, -0.005762], "mean_abs_delta": [0.248258, 0.248032, 0.247679, 0.247237, 0.246558, 0.247613, 0.246757, 0.245898, 0.244698, 0.246351, 0.24496, 0.243951, 0.242646, 0.244344, 0.243456, 0.242858, 0.242606]}, {"n": 516, "day": "2026-08-21", "step": [0.0, 0.06285724, 0.06112776, 0.05891028, 0.0567097, 0.18427345, 0.17233692, 0.16541138, 0.14897121, 0.57162594, 0.50935302, 0.43821617, 0.34439457, 1.37686233, 1.17150963, 0.89756371, 0.65063914], "drift": [0.0, 0.00261, 0.00523, 0.00853, 0.0142, 0.00755, 0.01451, 0.02251, 0.03492, 0.02328, 0.04082, 0.05336, 0.07391, 0.05428, 0.07838, 0.0956, 0.12411], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 37, 19, 9], "mean_delta": [-0.147489, -0.146347, -0.145006, -0.143132, -0.140017, -0.144252, -0.14068, -0.136245, -0.129341, -0.137658, -0.1296, -0.122658, -0.109237, -0.125454, -0.115666, -0.103236, -0.087733], "mean_abs_delta": [0.287041, 0.286704, 0.286389, 0.285947, 0.285183, 0.286079, 0.285216, 0.284184, 0.282691, 0.284046, 0.282292, 0.281063, 0.278676, 0.280197, 0.278503, 0.276538, 0.275205]}, {"n": 984, "day": "2026-08-22", "step": [0.0, 0.07079557, 0.06621022, 0.06018513, 0.05257794, 0.19193424, 0.16152149, 0.13697887, 0.11283919, 0.48820673, 0.35875452, 0.31211081, 0.24346503, 1.00120737, 0.7723155, 0.54850571, 0.24029123], "drift": [0.0, 0.00287, 0.00597, 0.00952, 0.01553, 0.00788, 0.01613, 0.02413, 0.03714, 0.02219, 0.04046, 0.05341, 0.07458, 0.04108, 0.06905, 0.08551, 0.1208], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 24, 18, 5], "mean_delta": [-0.062656, -0.060646, -0.058451, -0.055591, -0.050791, -0.057183, -0.051396, -0.044593, -0.033894, -0.046839, -0.033256, -0.023211, -0.005956, -0.029865, -0.016195, -0.004611, 0.020509], "mean_abs_delta": [0.184449, 0.183733, 0.183, 0.182128, 0.180773, 0.182562, 0.180852, 0.179072, 0.17683, 0.179495, 0.17625, 0.174865, 0.174033, 0.17545, 0.173488, 0.172961, 0.174931]}, {"n": 738, "day": "2026-08-23", "step": [0.0, 0.04704602, 0.04141123, 0.03620002, 0.03101032, 0.12249323, 0.09463202, 0.07637467, 0.06118426, 0.28513528, 0.19911735, 0.15631127, 0.11387271, 0.59940241, 0.38841791, 0.33015157, 0.22280503], "drift": [0.0, 0.00231, 0.00581, 0.00983, 0.01631, 0.00643, 0.01581, 0.02458, 0.03834, 0.01784, 0.03734, 0.05325, 0.07589, 0.03373, 0.06374, 0.08371, 0.12176], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 7, 3, 1], "mean_delta": [-0.04995, -0.047962, -0.04565, -0.042778, -0.037946, -0.044612, -0.038553, -0.032108, -0.021452, -0.035592, -0.023722, -0.013256, 0.004438, -0.024243, -0.010924, 0.000147, 0.030133], "mean_abs_delta": [0.185658, 0.185118, 0.184506, 0.183816, 0.182837, 0.184212, 0.182902, 0.181814, 0.180355, 0.182238, 0.180339, 0.179151, 0.179191, 0.180085, 0.178329, 0.177767, 0.181409]}, {"n": 556, "day": "2026-08-24", "step": [0.0, 0.05199018, 0.04871911, 0.04527927, 0.040009, 0.14488231, 0.12315972, 0.10906305, 0.09013911, 0.40353722, 0.30657548, 0.24763644, 0.19324898, 0.99032098, 0.67078023, 0.56913058, 0.32789244], "drift": [0.0, 0.00217, 0.00579, 0.01006, 0.01708, 0.00624, 0.0155, 0.02514, 0.03965, 0.01756, 0.03639, 0.05457, 0.07878, 0.03532, 0.06671, 0.08866, 0.12579], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 10, 8], "mean_delta": [-0.072144, -0.070691, -0.068244, -0.065315, -0.06047, -0.068015, -0.061636, -0.055142, -0.045122, -0.06035, -0.048188, -0.036675, -0.020267, -0.049637, -0.031801, -0.020553, 0.004401], "mean_abs_delta": [0.236508, 0.236001, 0.235204, 0.234242, 0.232657, 0.235065, 0.233036, 0.231087, 0.228547, 0.232386, 0.229153, 0.226289, 0.223166, 0.228804, 0.224558, 0.222339, 0.220227]}, {"n": 644, "day": "2026-08-25", "step": [0.0, 0.04470174, 0.0406165, 0.03516736, 0.02863363, 0.12024254, 0.0903945, 0.07167589, 0.05667827, 0.29734376, 0.2064007, 0.15222247, 0.12518414, 0.65988753, 0.42234057, 0.33745672, 0.25093043], "drift": [0.0, 0.00154, 0.00507, 0.00953, 0.01706, 0.00428, 0.01341, 0.02367, 0.03932, 0.01205, 0.03114, 0.05043, 0.07709, 0.02609, 0.05574, 0.08201, 0.12521], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 9, 7, 3], "mean_delta": [-0.040169, -0.038763, -0.036122, -0.032954, -0.027635, -0.036294, -0.029576, -0.022216, -0.011144, -0.029791, -0.015745, -0.002789, 0.014981, -0.019097, 0.001565, 0.014984, 0.040777], "mean_abs_delta": [0.196488, 0.196209, 0.195725, 0.195222, 0.1946, 0.19574, 0.194742, 0.194119, 0.194085, 0.194747, 0.19374, 0.193818, 0.1968, 0.19402, 0.194103, 0.195771, 0.203331]}, {"n": 1525, "day": "2026-08-26", "step": [0.0, 0.09167986, 0.08027399, 0.07324188, 0.06339315, 0.2489593, 0.20470223, 0.17083082, 0.13164434, 0.70956551, 0.52298793, 0.3795703, 0.24847017, 1.64649651, 1.0648022, 0.72603168, 0.38225164], "drift": [0.0, 0.0021, 0.00533, 0.00984, 0.01784, 0.00635, 0.01422, 0.02464, 0.04067, 0.01809, 0.0348, 0.05169, 0.07962, 0.04288, 0.06292, 0.08695, 0.126], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 27, 17, 8], "mean_delta": [-0.0269, -0.026158, -0.023768, -0.02057, -0.014999, -0.024668, -0.018612, -0.011086, 0.000449, -0.020785, -0.00779, 0.005347, 0.025772, -0.011685, 0.00583, 0.024162, 0.051822], "mean_abs_delta": [0.191201, 0.190981, 0.190542, 0.190028, 0.189271, 0.190537, 0.189615, 0.188747, 0.187952, 0.189384, 0.188105, 0.187555, 0.188566, 0.187303, 0.187031, 0.187802, 0.193864]}, {"n": 1375, "day": "2026-08-27", "step": [0.0, 0.08118152, 0.06985468, 0.05589185, 0.03914406, 0.1985926, 0.13376636, 0.09940184, 0.07505503, 0.42244928, 0.28212476, 0.21233934, 0.16099011, 0.82615748, 0.60867617, 0.46572436, 0.33367715], "drift": [0.0, 0.00291, 0.00632, 0.01077, 0.01872, 0.00782, 0.01525, 0.02565, 0.04233, 0.01814, 0.03675, 0.05328, 0.08136, 0.04126, 0.06447, 0.08797, 0.12769], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 3, 1], "mean_delta": [-0.048981, -0.047106, -0.044455, -0.040856, -0.034472, -0.043466, -0.03739, -0.028902, -0.015966, -0.034957, -0.020119, -0.008064, 0.012922, -0.015645, 0.000754, 0.015879, 0.041156], "mean_abs_delta": [0.173538, 0.173107, 0.172563, 0.171906, 0.170879, 0.172345, 0.171306, 0.170143, 0.169173, 0.170896, 0.169355, 0.169006, 0.17, 0.169029, 0.169379, 0.170392, 0.174663]}, {"n": 467, "day": "2026-08-28", "step": [0.0, 0.04375223, 0.03980578, 0.03540602, 0.03030321, 0.11834353, 0.09898567, 0.08402507, 0.06656402, 0.33475573, 0.26345799, 0.20212572, 0.14414459, 0.72334079, 0.5348303, 0.38463634, 0.21109113], "drift": [0.0, 0.00135, 0.0048, 0.0096, 0.01825, 0.00371, 0.01181, 0.02316, 0.04156, 0.00935, 0.02977, 0.04957, 0.07977, 0.02628, 0.05585, 0.0805, 0.12531], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 20, 14, 9], "mean_delta": [0.001875, 0.003347, 0.005717, 0.008744, 0.014131, 0.005893, 0.010976, 0.018103, 0.029636, 0.010507, 0.023759, 0.035278, 0.053415, 0.022608, 0.040238, 0.053798, 0.07677], "mean_abs_delta": [0.231976, 0.23186, 0.231583, 0.231264, 0.230809, 0.231702, 0.231147, 0.230689, 0.23049, 0.231478, 0.230825, 0.230688, 0.231431, 0.231387, 0.231255, 0.231692, 0.235412]}, {"n": 1316, "day": "2026-08-29", "step": [0.0, 0.08629961, 0.074102, 0.0617587, 0.04749534, 0.2378084, 0.17612329, 0.13548512, 0.10750243, 0.68545477, 0.43172701, 0.33850964, 0.21798338, 1.37938537, 0.94471312, 0.66335817, 0.37055486], "drift": [0.0, 0.00133, 0.00434, 0.00902, 0.01829, 0.00381, 0.01053, 0.02242, 0.04222, 0.0109, 0.02924, 0.04937, 0.08057, 0.02976, 0.05662, 0.08318, 0.12906], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 23, 16, 5], "mean_delta": [-0.014835, -0.014124, -0.011384, -0.007726, -0.000931, -0.012717, -0.006712, 0.00231, 0.016959, -0.010527, 0.005986, 0.021256, 0.043403, -0.000581, 0.02187, 0.040534, 0.07154], "mean_abs_delta": [0.194105, 0.193981, 0.193731, 0.19348, 0.193273, 0.19378, 0.193352, 0.193213, 0.194318, 0.193394, 0.193076, 0.1947, 0.199462, 0.192366, 0.194274, 0.198236, 0.208934]}, {"n": 345, "day": "2026-08-30", "step": [0.0, 0.02939512, 0.02634118, 0.02106314, 0.01576835, 0.08304256, 0.06171494, 0.0449447, 0.031791, 0.23938493, 0.14502543, 0.09164533, 0.07063859, 0.49512937, 0.26300237, 0.20475419, 0.13706382], "drift": [0.0, 0.00112, 0.00389, 0.00859, 0.01832, 0.00312, 0.00955, 0.02165, 0.0425, 0.009, 0.02738, 0.04814, 0.08162, 0.02397, 0.05393, 0.08333, 0.12813], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 3, 2], "mean_delta": [-0.05648, -0.055857, -0.053708, -0.050188, -0.043223, -0.054794, -0.049888, -0.040677, -0.026103, -0.052482, -0.037555, -0.022475, -0.000264, -0.042398, -0.021649, -0.001424, 0.02677], "mean_abs_delta": [0.210476, 0.210248, 0.209698, 0.208814, 0.207283, 0.209851, 0.208584, 0.20673, 0.204709, 0.208838, 0.206018, 0.204099, 0.202998, 0.206186, 0.203376, 0.202384, 0.20401]}, {"n": 286, "day": "2026-08-31", "step": [0.0, 0.03179052, 0.02905972, 0.02426906, 0.01884788, 0.09156561, 0.07276787, 0.05196809, 0.03524788, 0.25354667, 0.15317311, 0.10622756, 0.06961744, 0.50010213, 0.29178164, 0.18970141, 0.07194889], "drift": [0.0, 0.00087, 0.00337, 0.00793, 0.01804, 0.00248, 0.00809, 0.02018, 0.04236, 0.00704, 0.02412, 0.04619, 0.08153, 0.01897, 0.05054, 0.08183, 0.12697], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 11, 4, 1], "mean_delta": [-0.019745, -0.019099, -0.017146, -0.013692, -0.006285, -0.018009, -0.013746, -0.004426, 0.011453, -0.015654, -0.001914, 0.014086, 0.03815, -0.007841, 0.015306, 0.037138, 0.063849], "mean_abs_delta": [0.230497, 0.230357, 0.230005, 0.229447, 0.228391, 0.230114, 0.229333, 0.228066, 0.22686, 0.229423, 0.227543, 0.226424, 0.2261, 0.227506, 0.22569, 0.224916, 0.228186]}, {"n": 262, "day": "2026-09-01", "step": [0.0, 0.02220945, 0.01957663, 0.01622015, 0.01220125, 0.06103343, 0.04962636, 0.03571233, 0.02316173, 0.17014137, 0.11125148, 0.06986754, 0.02751291, 0.38155811, 0.19625071, 0.08645905, 0.04951678], "drift": [0.0, 0.0009, 0.00329, 0.00772, 0.01794, 0.00279, 0.00799, 0.01925, 0.04183, 0.00778, 0.02225, 0.04328, 0.0806, 0.01858, 0.04371, 0.07682, 0.12559], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0], "mean_delta": [-0.050607, -0.049948, -0.048432, -0.045629, -0.039372, -0.048666, -0.045356, -0.038031, -0.024472, -0.045246, -0.035568, -0.022469, -0.001467, -0.037581, -0.021128, -0.002308, 0.020669], "mean_abs_delta": [0.209404, 0.209058, 0.208367, 0.207159, 0.204585, 0.208395, 0.206905, 0.203939, 0.1999, 0.206587, 0.202618, 0.199088, 0.194615, 0.20279, 0.197804, 0.194306, 0.193425]}, {"n": 215, "day": "2026-09-02", "step": [0.0, 0.01950953, 0.01855094, 0.01688687, 0.01535418, 0.05658671, 0.05216318, 0.04597486, 0.03700255, 0.17603072, 0.1492127, 0.11986204, 0.08855332, 0.43408566, 0.33790854, 0.25116014, 0.12548558], "drift": [0.0, 0.00085, 0.00301, 0.00738, 0.01788, 0.00263, 0.00749, 0.01871, 0.04187, 0.00813, 0.02176, 0.04291, 0.08015, 0.02065, 0.04383, 0.07437, 0.12557], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 11, 6, 0], "mean_delta": [-0.01788, -0.017634, -0.016312, -0.013712, -0.007847, -0.017064, -0.014105, -0.007431, 0.005483, -0.015539, -0.007046, 0.005721, 0.026103, -0.012387, 0.002943, 0.022069, 0.04647], "mean_abs_delta": [0.230125, 0.230057, 0.229883, 0.229648, 0.22928, 0.229913, 0.229551, 0.229102, 0.228939, 0.229479, 0.228688, 0.228447, 0.229273, 0.228839, 0.227682, 0.228334, 0.231888]}, {"n": 195, "day": "2026-09-03", "step": [0.0, 0.01902172, 0.01621971, 0.01246771, 0.00844851, 0.05370394, 0.03817254, 0.02533463, 0.01299596, 0.14095247, 0.08077155, 0.04582197, 0.02751261, 0.26999788, 0.14341474, 0.08500843, 0.0433968], "drift": [0.0, 0.00066, 0.00252, 0.00666, 0.01735, 0.00195, 0.00609, 0.01648, 0.04075, 0.00533, 0.01631, 0.03798, 0.07851, 0.01073, 0.03308, 0.06679, 0.12382], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 2, 2], "mean_delta": [-0.074095, -0.073693, -0.07247, -0.069856, -0.063262, -0.072899, -0.070276, -0.063475, -0.048718, -0.070584, -0.063795, -0.050275, -0.026565, -0.068727, -0.055076, -0.034139, -0.003212], "mean_abs_delta": [0.235281, 0.235175, 0.234946, 0.234486, 0.233372, 0.234967, 0.234529, 0.233359, 0.230828, 0.234402, 0.23328, 0.231155, 0.227299, 0.233989, 0.23181, 0.228659, 0.225279]}, {"n": 301, "day": "2026-09-04", "step": [0.0, 0.02933012, 0.02813112, 0.02632646, 0.02038493, 0.0870515, 0.07975674, 0.06365537, 0.03483518, 0.27816772, 0.23210254, 0.1298645, 0.05609047, 0.80538895, 0.49541149, 0.23193107, 0.08924744], "drift": [0.0, 0.0007, 0.00222, 0.00616, 0.01692, 0.00198, 0.00539, 0.01488, 0.03981, 0.006, 0.01373, 0.03476, 0.07505, 0.01537, 0.0292, 0.05698, 0.11641], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 9, 2, 3], "mean_delta": [-0.008926, -0.008796, -0.007617, -0.004779, 0.002813, -0.008594, -0.005971, 0.001216, 0.01878, -0.008111, -0.001724, 0.014758, 0.041606, -0.008391, 0.00581, 0.029073, 0.065003], "mean_abs_delta": [0.231664, 0.231597, 0.231179, 0.23021, 0.227716, 0.23149, 0.230585, 0.228231, 0.223005, 0.231199, 0.229116, 0.22411, 0.219352, 0.230975, 0.226723, 0.221188, 0.218476]}, {"n": 133, "day": "2026-09-05", "step": [0.0, 0.00975421, 0.00895728, 0.00811434, 0.0072122, 0.02755716, 0.02465262, 0.0218246, 0.01928112, 0.08333245, 0.07514489, 0.06559157, 0.05174995, 0.2404971, 0.21020005, 0.16684518, 0.12141792], "drift": [0.0, 0.00029, 0.00168, 0.00549, 0.01652, 0.00087, 0.00411, 0.01339, 0.0387, 0.00284, 0.01077, 0.03097, 0.07288, 0.00765, 0.02306, 0.05126, 0.11268], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "mean_delta": [-0.017264, -0.017125, -0.016208, -0.013737, -0.006923, -0.016843, -0.014788, -0.008722, 0.006836, -0.01581, -0.011096, 0.002354, 0.026787, -0.013528, -0.004682, 0.013665, 0.046423], "mean_abs_delta": [0.204211, 0.204155, 0.203775, 0.202752, 0.200191, 0.204046, 0.203193, 0.200857, 0.194883, 0.203664, 0.201818, 0.1965, 0.190329, 0.202774, 0.199263, 0.19261, 0.188486]}, {"n": 246, "day": "2026-09-06", "step": [0.0, 0.02326191, 0.02235886, 0.01986652, 0.01598883, 0.0693392, 0.06285499, 0.05081196, 0.03662, 0.22246512, 0.19159761, 0.14067066, 0.09014761, 0.66139445, 0.5137931, 0.35138259, 0.13267747], "drift": [0.0, 0.00035, 0.00113, 0.00457, 0.01564, 0.00104, 0.00284, 0.01097, 0.03593, 0.00366, 0.0077, 0.02399, 0.06572, 0.00903, 0.01867, 0.03506, 0.10298], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 17, 11, 6], "mean_delta": [0.040899, 0.040892, 0.041702, 0.044247, 0.051908, 0.040874, 0.042741, 0.04899, 0.066231, 0.040774, 0.045132, 0.058354, 0.085991, 0.040447, 0.048248, 0.064342, 0.106088], "mean_abs_delta": [0.235625, 0.235633, 0.235618, 0.23554, 0.235368, 0.23566, 0.235609, 0.235449, 0.236186, 0.23576, 0.235638, 0.235688, 0.239191, 0.235951, 0.235605, 0.236632, 0.243836]}, {"n": 137, "day": "2026-09-07", "step": [0.0, 0.01009121, 0.00953655, 0.00757871, 0.0049945, 0.03115872, 0.0281916, 0.01804591, 0.00879754, 0.11144284, 0.08926339, 0.04007388, 0.01630277, 0.33589161, 0.25725566, 0.12486059, 0.01063137], "drift": [0.0, 0.00026, 0.00095, 0.00418, 0.0153, 0.0008, 0.00245, 0.01007, 0.03514, 0.00269, 0.00692, 0.02212, 0.06434, 0.00599, 0.01722, 0.03329, 0.1007], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0], "mean_delta": [-0.016121, -0.016262, -0.015603, -0.012912, -0.003762, -0.016562, -0.015234, -0.008608, 0.011637, -0.017701, -0.01511, -0.001719, 0.031751, -0.016158, -0.014891, -0.003135, 0.05433], "mean_abs_delta": [0.193758, 0.193774, 0.193658, 0.193206, 0.191866, 0.193804, 0.193558, 0.192493, 0.190242, 0.193997, 0.193409, 0.191422, 0.189044, 0.193852, 0.193063, 0.191096, 0.190321]}, {"n": 167, "day": "2026-09-08", "step": [0.0, 0.01312416, 0.01311906, 0.01243212, 0.01101319, 0.04037402, 0.03838226, 0.03498283, 0.0288442, 0.13334417, 0.12410463, 0.10502601, 0.0544116, 0.36169737, 0.34682685, 0.29105325, 0.09765011], "drift": [0.0, 0.00039, 0.00108, 0.00412, 0.01519, 0.00114, 0.00285, 0.00995, 0.0349, 0.00361, 0.00798, 0.02222, 0.06319, 0.0101, 0.02004, 0.0353, 0.09691], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 8, 0], "mean_delta": [-0.068602, -0.068583, -0.068238, -0.066254, -0.059141, -0.068598, -0.067828, -0.063003, -0.046731, -0.068612, -0.067087, -0.056897, -0.029587, -0.066652, -0.064929, -0.054877, -0.011713], "mean_abs_delta": [0.195573, 0.195531, 0.195511, 0.195489, 0.195772, 0.195447, 0.195369, 0.195454, 0.196446, 0.195167, 0.194961, 0.195327, 0.197649, 0.194516, 0.19387, 0.194313, 0.20038]}, {"n": 65, "day": "2026-09-09", "step": [0.0, 0.00665872, 0.00665723, 0.00619565, 0.00542629, 0.01997365, 0.01974988, 0.01710868, 0.01312009, 0.06583992, 0.05978317, 0.04799426, 0.02451812, 0.18086942, 0.14875548, 0.13515674, 0.04138675], "drift": [0.0, 0.00026, 0.00086, 0.00384, 0.0149, 0.00074, 0.00221, 0.00914, 0.03426, 0.00235, 0.00598, 0.02019, 0.06202, 0.00729, 0.01507, 0.03144, 0.09584], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 2], "mean_delta": [-0.016557, -0.016481, -0.01612, -0.014263, -0.007485, -0.016365, -0.015528, -0.01113, 0.004215, -0.016017, -0.014099, -0.004853, 0.019949, -0.014279, -0.010224, -0.001068, 0.034953], "mean_abs_delta": [0.244374, 0.244347, 0.244295, 0.243738, 0.241291, 0.24431, 0.244242, 0.242924, 0.237317, 0.24418, 0.244277, 0.241599, 0.231702, 0.243548, 0.244356, 0.241885, 0.224518]}, {"n": 77, "day": "2026-09-10", "step": [0.0, 0.01405659, 0.01380972, 0.01371657, 0.01313055, 0.04142935, 0.04116493, 0.04044309, 0.03408544, 0.13658654, 0.13487526, 0.12856365, 0.06083363, 0.37658455, 0.37280872, 0.36325691, 0.05562118], "drift": [0.0, 0.00097, 0.00158, 0.00436, 0.01542, 0.00292, 0.00442, 0.01091, 0.03572, 0.0098, 0.01342, 0.02694, 0.06419, 0.0283, 0.03663, 0.05147, 0.09453], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 18, 18, 4], "mean_delta": [-0.325405, -0.325309, -0.325191, -0.324547, -0.321863, -0.325126, -0.324856, -0.323358, -0.3172, -0.324491, -0.323904, -0.320783, -0.311176, -0.322451, -0.321274, -0.318371, -0.305], "mean_abs_delta": [0.412495, 0.412479, 0.412434, 0.412177, 0.411142, 0.41245, 0.412349, 0.411746, 0.409522, 0.412352, 0.412126, 0.410878, 0.407852, 0.411813, 0.411371, 0.410266, 0.405797]}, {"n": 79, "day": "2026-09-11", "step": [0.0, 0.00534917, 0.00528184, 0.0050939, 0.00442459, 0.01583146, 0.0152748, 0.01371148, 0.01162795, 0.0457517, 0.04556668, 0.04149637, 0.02252047, 0.12201497, 0.11605729, 0.09059171, 0.04589043], "drift": [0.0, 0.00087, 0.00159, 0.00429, 0.01537, 0.00259, 0.00446, 0.01081, 0.03569, 0.00847, 0.01355, 0.0268, 0.06415, 0.02385, 0.0366, 0.05132, 0.09513], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "mean_delta": [-0.12375, -0.123222, -0.12281, -0.121171, -0.113884, -0.122179, -0.121137, -0.117325, -0.101147, -0.118601, -0.115969, -0.108226, -0.084206, -0.109043, -0.102222, -0.094814, -0.067413], "mean_abs_delta": [0.192215, 0.192039, 0.191933, 0.191517, 0.189605, 0.191704, 0.191437, 0.190475, 0.186239, 0.190558, 0.189884, 0.187978, 0.181836, 0.187596, 0.18583, 0.18412, 0.176645]}, {"n": 493, "day": "2026-09-12", "step": [0.0, 0.0370898, 0.03507038, 0.03225774, 0.01932832, 0.10301545, 0.09570568, 0.0697418, 0.03207083, 0.29446763, 0.21732125, 0.14732182, 0.06674761, 0.54893624, 0.33661904, 0.25043254, 0.12497538], "drift": [0.0, 0.00097, 0.00184, 0.00437, 0.01509, 0.00268, 0.00496, 0.01069, 0.03383, 0.00712, 0.01286, 0.02324, 0.0603, 0.01572, 0.02459, 0.04117, 0.08817], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 17, 12, 4], "mean_delta": [-0.052176, -0.05147, -0.050841, -0.048837, -0.040248, -0.050155, -0.048558, -0.043957, -0.025105, -0.046425, -0.042587, -0.033848, -0.006406, -0.040447, -0.03061, -0.019606, 0.010027], "mean_abs_delta": [0.172821, 0.172692, 0.17259, 0.172277, 0.171538, 0.172457, 0.172205, 0.171744, 0.171229, 0.171891, 0.171546, 0.171197, 0.172319, 0.171372, 0.171058, 0.171113, 0.175126]}, {"n": 84, "day": "2026-09-13", "step": [0.0, 0.00483925, 0.00460524, 0.00435357, 0.0029117, 0.01376317, 0.01295667, 0.00991481, 0.00450319, 0.04041816, 0.03158356, 0.02217941, 0.01329835, 0.09713418, 0.06785647, 0.04497612, 0.03795227], "drift": [0.0, 0.00056, 0.00154, 0.00403, 0.01479, 0.00151, 0.00413, 0.00978, 0.03299, 0.00389, 0.0104, 0.02135, 0.05848, 0.00808, 0.02009, 0.03642, 0.0844], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0], "mean_delta": [-0.003795, -0.003168, -0.002474, -0.000561, 0.007693, -0.00209, -0.000259, 0.004098, 0.021821, 0.000539, 0.005203, 0.012976, 0.039871, 0.005103, 0.012472, 0.023773, 0.053932], "mean_abs_delta": [0.129046, 0.129024, 0.128986, 0.128928, 0.129218, 0.128974, 0.128919, 0.128951, 0.131391, 0.128947, 0.129011, 0.129546, 0.136366, 0.129146, 0.129562, 0.131503, 0.141918]}, {"n": 143, "day": "2026-09-14", "step": [0.0, 0.01395576, 0.01309729, 0.01124796, 0.00897606, 0.04010188, 0.03377439, 0.02885052, 0.01906088, 0.12376363, 0.09607635, 0.07550813, 0.05758084, 0.31571325, 0.24308048, 0.18792083, 0.14813834], "drift": [0.0, 0.00038, 0.00114, 0.00339, 0.01425, 0.00111, 0.00276, 0.00818, 0.03209, 0.00348, 0.0069, 0.01847, 0.05609, 0.01049, 0.01591, 0.03238, 0.08018], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0], "mean_delta": [0.042167, 0.042324, 0.042822, 0.04409, 0.04979, 0.042599, 0.043851, 0.046939, 0.059201, 0.043063, 0.046646, 0.052594, 0.071165, 0.043394, 0.050826, 0.059147, 0.080106], "mean_abs_delta": [0.28548, 0.285509, 0.285652, 0.286063, 0.28775, 0.285551, 0.285925, 0.286922, 0.290437, 0.285563, 0.28668, 0.288459, 0.293552, 0.285445, 0.28746, 0.289817, 0.295308]}, {"n": 87, "day": "2026-09-15", "step": [0.0, 0.00589553, 0.00534711, 0.00437177, 0.0023167, 0.0176887, 0.01574199, 0.0092213, 0.00417802, 0.06091992, 0.03922848, 0.02030019, 0.00725556, 0.24855052, 0.07829639, 0.04196794, 0.01774509], "drift": [0.0, 0.00018, 0.00085, 0.00303, 0.01393, 0.00052, 0.00207, 0.00752, 0.03138, 0.00165, 0.00588, 0.01693, 0.05464, 0.00508, 0.0136, 0.03038, 0.07811], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0], "mean_delta": [-0.029959, -0.029943, -0.029289, -0.027475, -0.018695, -0.029969, -0.028517, -0.023939, -0.004823, -0.030586, -0.026434, -0.016761, 0.011357, -0.03272, -0.023351, -0.00884, 0.022281], "mean_abs_delta": [0.204738, 0.204741, 0.204547, 0.204045, 0.202169, 0.204765, 0.204331, 0.203264, 0.199482, 0.204999, 0.203755, 0.201677, 0.197665, 0.205918, 0.202919, 0.199844, 0.196499]}, {"n": 124, "day": "2026-09-16", "step": [0.0, 0.01832044, 0.01804405, 0.0175943, 0.01563341, 0.05491715, 0.05347187, 0.04937034, 0.04400562, 0.18374212, 0.17280494, 0.15474949, 0.11915406, 0.47576329, 0.42201889, 0.38254163, 0.2429094], "drift": [0.0, 0.00023, 0.00054, 0.00263, 0.01353, 0.00072, 0.00129, 0.00655, 0.03016, 0.0027, 0.00378, 0.01431, 0.05184, 0.0057, 0.00948, 0.02491, 0.075], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 29, 28, 18], "mean_delta": [0.022279, 0.022561, 0.023203, 0.025178, 0.035284, 0.023082, 0.024587, 0.029544, 0.051251, 0.024512, 0.029217, 0.038664, 0.069699, 0.028913, 0.036882, 0.049392, 0.085509], "mean_abs_delta": [0.230778, 0.230706, 0.230617, 0.230304, 0.229062, 0.230567, 0.23037, 0.229659, 0.227468, 0.230131, 0.229473, 0.228431, 0.225807, 0.229013, 0.228208, 0.226842, 0.225972]}, {"n": 191, "day": "2026-09-17", "step": [0.0, 0.01182056, 0.01127616, 0.01076521, 0.00703857, 0.0360324, 0.0341499, 0.02775102, 0.01566044, 0.15178881, 0.11024774, 0.07863011, 0.04397652, 0.56895505, 0.39013295, 0.18317343, 0.09496041], "drift": [0.0, 0.00035, 0.00027, 0.002, 0.01281, 0.00103, 0.0008, 0.00492, 0.02848, 0.003, 0.00262, 0.00954, 0.048, 0.01015, 0.00849, 0.01595, 0.06834], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 1, 0], "mean_delta": [0.06222, 0.061848, 0.062237, 0.063944, 0.073174, 0.061108, 0.061989, 0.066574, 0.08627, 0.058697, 0.061237, 0.070202, 0.100012, 0.054439, 0.060366, 0.072398, 0.110221], "mean_abs_delta": [0.211133, 0.211242, 0.211198, 0.210932, 0.209298, 0.211448, 0.211366, 0.210619, 0.208113, 0.211837, 0.211904, 0.21059, 0.208378, 0.211711, 0.211887, 0.211899, 0.208514]}, {"n": 319, "day": "2026-09-18", "step": [0.0, 0.02242248, 0.02217253, 0.02045203, 0.01633869, 0.07067607, 0.06859963, 0.05671508, 0.04174889, 0.25822656, 0.24802058, 0.18150416, 0.11894941, 0.78144932, 0.68174072, 0.54618854, 0.29896796], "drift": [0.0, 0.00047, 0.00052, 0.00155, 0.01229, 0.00136, 0.00153, 0.00396, 0.02727, 0.00401, 0.00485, 0.00799, 0.0462, 0.01092, 0.01303, 0.01686, 0.06539], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 24, 20, 8], "mean_delta": [0.009872, 0.009533, 0.009665, 0.010969, 0.018562, 0.008852, 0.009137, 0.012604, 0.028778, 0.006949, 0.007383, 0.014042, 0.039294, 0.004102, 0.006137, 0.013019, 0.045467], "mean_abs_delta": [0.207011, 0.207039, 0.207031, 0.206975, 0.206794, 0.207103, 0.207089, 0.20695, 0.207296, 0.207315, 0.207282, 0.206951, 0.208302, 0.207783, 0.207602, 0.207094, 0.208569]}, {"n": 192, "day": "2026-09-19", "step": [0.0, 0.02022867, 0.02022888, 0.01973887, 0.01385753, 0.06113315, 0.06201499, 0.05496726, 0.02287643, 0.20289687, 0.20613353, 0.17609878, 0.05229677, 0.56105344, 0.54359421, 0.49684691, 0.1352664], "drift": [0.0, 0.00044, 0.00063, 0.00197, 0.01252, 0.00137, 0.00194, 0.00538, 0.02754, 0.00466, 0.0069, 0.01263, 0.04721, 0.01402, 0.02082, 0.02768, 0.06799], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 18, 1], "mean_delta": [-0.105825, -0.105675, -0.105674, -0.10471, -0.098195, -0.105318, -0.105303, -0.102805, -0.089373, -0.103792, -0.103744, -0.099554, -0.079574, -0.098469, -0.097592, -0.094821, -0.072352], "mean_abs_delta": [0.235298, 0.235257, 0.235256, 0.234786, 0.231752, 0.235149, 0.235154, 0.233942, 0.227892, 0.234663, 0.234644, 0.232686, 0.2241, 0.232779, 0.232509, 0.230973, 0.221783]}, {"n": 285, "day": "2026-09-20", "step": [0.0, 0.03098814, 0.03096187, 0.02952989, 0.01886986, 0.09253663, 0.09229132, 0.07469206, 0.05010612, 0.29653753, 0.28435932, 0.19785869, 0.15575038, 0.63008355, 0.53885392, 0.50478423, 0.42051997], "drift": [0.0, 0.00061, 0.00083, 0.00211, 0.01247, 0.00185, 0.00245, 0.00576, 0.02776, 0.00626, 0.0084, 0.01378, 0.0487, 0.01765, 0.02334, 0.0337, 0.07197], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 19, 19, 17], "mean_delta": [-0.036569, -0.036506, -0.036457, -0.0355, -0.028211, -0.036347, -0.036182, -0.033628, -0.018538, -0.035528, -0.034718, -0.030588, -0.008546, -0.031474, -0.028088, -0.024574, -0.001976], "mean_abs_delta": [0.254601, 0.254568, 0.25457, 0.254423, 0.253539, 0.254498, 0.254512, 0.25415, 0.252636, 0.254219, 0.254275, 0.253819, 0.251753, 0.253477, 0.253899, 0.253214, 0.251228]}, {"n": 242, "day": "2026-09-21", "step": [0.0, 0.02376249, 0.02343056, 0.02027723, 0.00928573, 0.06476467, 0.06307917, 0.0473858, 0.01710585, 0.16063197, 0.14352258, 0.10422501, 0.03475859, 0.23264679, 0.18126442, 0.14457705, 0.06711467], "drift": [0.0, 0.00075, 0.00104, 0.00224, 0.01247, 0.00227, 0.00311, 0.00651, 0.02785, 0.0074, 0.01064, 0.01614, 0.04934, 0.01628, 0.02541, 0.03641, 0.07323], "capped": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 13, 12, 8], "mean_delta": [-0.048613, -0.04834, -0.04822, -0.047225, -0.038763, -0.047612, -0.047224, -0.044301, -0.027413, -0.044167, -0.042611, -0.03894, -0.015979, -0.035038, -0.031349, -0.027927, -0.007488], "mean_abs_delta": [0.189753, 0.18963, 0.189622, 0.18943, 0.188027, 0.189376, 0.189349, 0.188778, 0.187194, 0.188566, 0.188461, 0.188063, 0.187236, 0.187389, 0.187586, 0.18761, 0.188787]}], "bootstrap": {"lines": {"capitulation": 0.0025}, "gates_ok": true, "calibration": {"mean": 0.015530285738707813, "total": 19.59922060224926, "trades": 1262, "per_strategy": {"capitulation": {"line": 0.0025, "mean": 0.015530285738707813, "total": 19.59922060224926, "trades": 1262, "hold_min": 240}}}, "diagnostics": {"rows": 5648, "slope": 0.9636524869032652, "kc_active": 0.09993186593055725, "rank_corr": 0.8975306157777404, "mbon_share": 0.48021747339581045, "mbon_saturated": 0.010416666977107525}, "gate_failures": []}, "evaluation": {"n": 1651, "pf": 1.1344105930713058, "win": 0.4476075105996366, "days": 29, "mean": 0.015233186073601246, "median": -0.02210576832294464, "days_positive": 19}, "strategies": ["capitulation"], "finished_at": "2026-09-22T07:27:02.794988+00:00", "per_strategy": {"capitulation": {"alpha": 0.0003, "chosen": {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, "configs": [{"mean": 0.016807934269309044, "alpha": 0.0, "total": 65.02989959716797, "config": 0, "trades": 3869, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 0, "shadow": 0}, "half_life_days": null}, {"mean": 0.01756637543439865, "alpha": 0.0001, "total": 63.76594543457031, "config": 1, "trades": 3630, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020793743431568146, "alpha": 0.0001, "total": 68.8896713256836, "config": 2, "trades": 3313, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 3.0}, {"mean": 0.02111992798745632, "alpha": 0.0001, "total": 61.73354721069336, "config": 3, "trades": 2923, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 7.0}, {"mean": 0.020111948251724243, "alpha": 0.0001, "total": 50.62177276611328, "config": 4, "trades": 2517, "governance": {"ic": 0, "any": 68, "drift": 0, "checks": 1365, "shadow": 68}, "half_life_days": 30.0}, {"mean": 0.023196974769234657, "alpha": 0.0003, "total": 75.43656158447266, "config": 5, "trades": 3252, "governance": {"ic": 0, "any": 0, "drift": 0, "checks": 1365, "shadow": 0}, "half_life_days": 1.0}, {"mean": 0.020104866474866867, "alpha": 0.0003, "total": 52.694854736328125, "config": 6, "trades": 2621, "governance": {"ic": 0, "any": 34, "drift": 0, "checks": 1365, "shadow": 34}, "half_life_days": 3.0}, {"mean": 0.015242534689605236, "alpha": 0.0003, "total": 31.61301612854004, "config": 7, "trades": 2074, "governance": {"ic": 0, "any": 93, "drift": 0, "checks": 1365, "shadow": 93}, "half_life_days": 7.0}, {"mean": 0.014115910045802593, "alpha": 0.0003, "total": 23.277135848999023, "config": 8, "trades": 1649, "governance": {"ic": 0, "any": 111, "drift": 0, "checks": 1365, "shadow": 111}, "half_life_days": 30.0}, {"mean": 0.014811468310654163, "alpha": 0.001, "total": 39.4577522277832, "config": 9, "trades": 2664, "governance": {"ic": 0, "any": 74, "drift": 0, "checks": 1365, "shadow": 74}, "half_life_days": 1.0}, {"mean": 0.011060998775064945, "alpha": 0.001, "total": 18.847942352294922, "config": 10, "trades": 1704, "governance": {"ic": 0, "any": 139, "drift": 0, "checks": 1365, "shadow": 139}, "half_life_days": 3.0}, {"mean": 0.002477346919476986, "alpha": 0.001, "total": 3.146230697631836, "config": 11, "trades": 1270, "governance": {"ic": 0, "any": 184, "drift": 0, "checks": 1365, "shadow": 184}, "half_life_days": 7.0}, {"mean": 0.018671352416276932, "alpha": 0.001, "total": 19.77296257019043, "config": 12, "trades": 1059, "governance": {"ic": 0, "any": 144, "drift": 0, "checks": 1365, "shadow": 144}, "half_life_days": 30.0}, {"mean": 0.01424670871347189, "alpha": 0.003, "total": 28.10875701904297, "config": 13, "trades": 1973, "governance": {"ic": 0, "any": 203, "drift": 0, "checks": 1365, "shadow": 203}, "half_life_days": 1.0}, {"mean": 0.027894049882888794, "alpha": 0.003, "total": 38.716941833496094, "config": 14, "trades": 1388, "governance": {"ic": 0, "any": 71, "drift": 0, "checks": 1365, "shadow": 71}, "half_life_days": 3.0}, {"mean": 0.01914161443710327, "alpha": 0.003, "total": 18.816207885742188, "config": 15, "trades": 983, "governance": {"ic": 0, "any": 112, "drift": 0, "checks": 1365, "shadow": 112}, "half_life_days": 7.0}, {"mean": 0.028120551258325577, "alpha": 0.003, "total": 22.524560928344727, "config": 16, "trades": 801, "governance": {"ic": 0, "any": 196, "drift": 0, "checks": 1365, "shadow": 196}, "half_life_days": 30.0}], "plastic": true, "hold_min": 240, "half_life_days": 1.0}}, "half_life_days": 1.0, "selection_days": ["2026-07-27", "2026-08-23"], "evaluation_days": ["2026-08-24", "2026-09-21"], "plastic_beats_frozen": true}'::jsonb) ON CONFLICT (key) DO NOTHING; +INSERT INTO ui_settings (key, value) VALUES ('autostart', '{"pumpstream": true, "runner": true, "train": false, "replay": false, "discover": false}'::jsonb) ON CONFLICT (key) DO NOTHING; +INSERT INTO ui_settings (key, value) VALUES ('pinned_selector_snapshot', '{"id": 59, "why": "shipped with the distribution: the selector this fly was taught by", "pinned_at": "2026-09-21"}'::jsonb) ON CONFLICT (key) DO NOTHING; diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..04fab42d6a9623dd293248a7dee53ba69544e148 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +"""Every test runs against fly_trader_test. DATABASE_URL is forced before any fly_trader import so +the refuse-production guard in fly_trader.db.connection can never see the real database.""" +import os + +os.environ["DATABASE_URL"] = "postgresql:///fly_trader_test" + +import pytest # noqa: E402 + +TEST_URL = "postgresql:///fly_trader_test" + + +@pytest.fixture(scope="session", autouse=True) +def _test_database(): + from fly_trader.db import schema + schema.ensure_database(TEST_URL) + schema.apply_schema(TEST_URL) + yield + + +@pytest.fixture +def db_conn(): + from fly_trader.db.connection import connect + conn = connect(TEST_URL) + try: + yield conn + finally: + conn.rollback() + conn.close() diff --git a/tests/test_balances.py b/tests/test_balances.py new file mode 100644 index 0000000000000000000000000000000000000000..a668f9f491c2012275bd7a68c2d63cbfab0b4910 --- /dev/null +++ b/tests/test_balances.py @@ -0,0 +1,29 @@ +from datetime import datetime, timezone + +from fly_trader.chain.balances import Snapshot, compute_delta, snapshot_balances + +TS = datetime(2026, 9, 12, tzinfo=timezone.utc) + + +def test_compute_delta_over_union_of_mints(): + pre = Snapshot(100, {"A": 5, "B": 7}, TS) + post = Snapshot(90, {"A": 9, "C": 1}, TS) + assert compute_delta(pre, post) == {"lamports_delta": -10, "token_deltas": {"A": 4, "B": -7, "C": 1}} + assert compute_delta(pre, pre) == {"lamports_delta": 0, "token_deltas": {"A": 0, "B": 0}} + + +def test_snapshot_sums_accounts_per_mint_and_keeps_decimals(): + class Rpc: + def get_balance(self, pk): + return 42 + + def get_token_accounts_by_owner(self, pk): + return [{"address": "x", "mint": "M", "amount": 5, "decimals": 6, "program": "p1"}, + {"address": "y", "mint": "M", "amount": 7, "decimals": 6, "program": "p2"}, + {"address": "z", "mint": "N", "amount": 0, "decimals": 9, "program": "p1"}] + + snap = snapshot_balances(Rpc(), "owner") + assert snap.lamports == 42 and snap.tokens == {"M": 12, "N": 0} and snap.decimals == {"M": 6, "N": 9} + assert snap.ts.tzinfo is not None and snap.token("missing") == 0 + js = snap.to_json() + assert js["lamports"] == 42 and js["tokens"]["M"] == 12 and js["decimals"]["N"] == 9 and js["ts"].endswith("+00:00") diff --git a/tests/test_book_halt.py b/tests/test_book_halt.py new file mode 100644 index 0000000000000000000000000000000000000000..45dad28cde4334d1acf68f5f085307eb4b537f7c --- /dev/null +++ b/tests/test_book_halt.py @@ -0,0 +1,20 @@ +"""A paper race book's own kill switch: a drawdown halts that book only, never the other book or the live circuit.""" +from fly_trader import config +from fly_trader.agent import rails +from fly_trader.db.connection import transaction + + +def test_book_drawdown_halts_only_that_book(db_conn, monkeypatch): + monkeypatch.setattr(config, "KILL_SWITCH_DRAWDOWN", 0.30) + with transaction() as conn: + conn.execute("DELETE FROM book_state WHERE book IN ('paper_fly','paper_selector')") + kill_before = rails.load_circuit(conn).kill_switch + assert not rails.check_book_drawdown(conn, "paper_fly", 4.0, 5.0) # -20 %: fine + assert rails.check_book_drawdown(conn, "paper_fly", 3.4, 5.0) # -32 %: halted + assert rails.check_book_drawdown(conn, "paper_fly", 5.0, 5.0) # stays halted until cleared + assert not rails.book_halted(conn, "paper_selector") + assert rails.load_circuit(conn).kill_switch == kill_before # the live circuit is untouched + rails.clear_book_halt("paper_fly") + with transaction() as conn: + assert not rails.book_halted(conn, "paper_fly") + conn.execute("DELETE FROM book_state WHERE book = 'paper_fly'") diff --git a/tests/test_broker_live.py b/tests/test_broker_live.py new file mode 100644 index 0000000000000000000000000000000000000000..56b9bfacdaaf8c55fb27842bb8d7f2ec82da8cdc --- /dev/null +++ b/tests/test_broker_live.py @@ -0,0 +1,277 @@ +"""LiveBroker against fakes: the fake /order builds a real unsigned v0 transaction with the bot +keypair as fee payer, the fake /execute checks that the broker submitted a fully signed transaction +and then applies a scripted on-chain outcome to the fake RPC's balances.""" +import base64 +import random +import uuid + +import pytest +from solders.hash import Hash +from solders.instruction import AccountMeta, Instruction +from solders.keypair import Keypair +from solders.message import MessageV0 +from solders.pubkey import Pubkey +from solders.signature import Signature +from solders.system_program import TransferParams, transfer +from solders.transaction import VersionedTransaction + +from fly_trader import config +from fly_trader.chain.cluster_guard import SigningRefused +from fly_trader.chain.rpc import TOKEN_PROGRAM_ID +from fly_trader.execution import broker_live +from fly_trader.execution.broker_live import LiveBroker + +MINT = str(Pubkey.new_unique()) +FAKE_DEX = str(Pubkey.new_unique()) +SYSTEM = "11111111111111111111111111111111" +BUY_LAMPORTS = 10_000_000 +BUY_TOKENS = 1_000_000 +BUY_FEES = 7_000 + + +class FakeRpc: + def __init__(self, lamports: int, tokens: dict | None = None, decimals: dict | None = None): + self.lamports = lamports + self.tokens = dict(tokens or {}) + self.decimals = dict(decimals or {}) + self.height = 1_000 + self.landed: set[str] = set() + self.status_polls = 0 + + def get_balance(self, pk): + return self.lamports + + def get_token_accounts_by_owner(self, pk): + return [{"address": f"ata-{m[:6]}", "mint": m, "amount": a, "decimals": self.decimals.get(m, 6), + "program": TOKEN_PROGRAM_ID} for m, a in self.tokens.items()] + + def get_signature_statuses(self, sigs, search_history=False): + self.status_polls += 1 + return [({"slot": 5, "confirmations": 12, "err": None, "confirmationStatus": "confirmed"} if s in self.landed else None) + for s in sigs] + + def get_block_height(self): + self.height += 40 + return self.height + + def get_transaction(self, sig, encoding="jsonParsed"): + return {"slot": 5, "meta": {"err": None, "fee": 5000}} if sig in self.landed else None + + def get_mint_decimals(self, mint): + return None + + +class FakeJup: + def __init__(self, keypair: Keypair, rpc: FakeRpc, outcomes: list[str]): + self.kp = keypair + self.rpc = rpc + self.outcomes = list(outcomes) + self.orders: list[dict] = [] + self.executes: list[tuple] = [] + + def order(self, input_mint, output_mint, amount, taker=None, slippage_bps=None, exclude_routers="jupiterz"): + self.orders.append(dict(input_mint=input_mint, output_mint=output_mint, amount=amount, taker=taker, + slippage_bps=slippage_bps, exclude_routers=exclude_routers)) + assert taker == str(self.kp.pubkey()) + ixs = [transfer(TransferParams(from_pubkey=self.kp.pubkey(), to_pubkey=Pubkey.new_unique(), lamports=1)), + Instruction(Pubkey.from_string(FAKE_DEX), b"\x09", [AccountMeta(self.kp.pubkey(), True, True)])] + msg = MessageV0.try_compile(self.kp.pubkey(), ixs, [], Hash.new_unique()) + tx = VersionedTransaction.populate(msg, [Signature.default()]) + return {"transaction": base64.b64encode(bytes(tx)).decode(), "requestId": str(uuid.uuid4()), + "lastValidBlockHeight": str(self.rpc.height + 150), "signatureFeePayer": taker, "taker": taker, + "router": "metis", "feeBps": 10, "inAmount": str(amount), "outAmount": str(BUY_TOKENS), + "slippageBps": slippage_bps, "signatureFeeLamports": 5000, "prioritizationFeeLamports": 1000, + "platformFee": {"feeBps": 10, "feeMint": input_mint}, "_latency_ms": 7} + + def execute(self, signed_transaction_b64, request_id, last_valid_block_height=None): + tx = VersionedTransaction.from_bytes(base64.b64decode(signed_transaction_b64)) + assert tx.verify_with_results() == [True], "broker must submit a fully signed transaction" + assert tx.message.account_keys[0] == self.kp.pubkey() + sig = str(tx.signatures[0]) + outcome = self.outcomes.pop(0) + self.executes.append((request_id, last_valid_block_height, outcome)) + if outcome == "success_buy": + self.rpc.lamports -= BUY_LAMPORTS + BUY_FEES + self.rpc.tokens[MINT] = self.rpc.tokens.get(MINT, 0) + BUY_TOKENS + self.rpc.landed.add(sig) + return {"status": "Success", "signature": sig, "slot": "5", "code": 0, "totalInputAmount": str(BUY_LAMPORTS), + "totalOutputAmount": str(BUY_TOKENS), "inputAmountResult": "9990000", "outputAmountResult": str(BUY_TOKENS), + "swapEvents": [], "_latency_ms": 900} + if outcome == "success_sell": + self.rpc.tokens[MINT] = 0 + self.rpc.lamports += 9_990_000 - 6_000 + self.rpc.landed.add(sig) + return {"status": "Success", "signature": sig, "slot": "6", "code": 0, "totalInputAmount": str(BUY_TOKENS), + "totalOutputAmount": "9990000", "_latency_ms": 800} + if outcome == "success_no_move": + self.rpc.landed.add(sig) + return {"status": "Success", "signature": sig, "slot": "5", "code": 0, "totalInputAmount": str(BUY_LAMPORTS), + "totalOutputAmount": str(BUY_TOKENS), "_latency_ms": 500} + if outcome == "failed_but_moved": + self.rpc.lamports -= BUY_LAMPORTS + BUY_FEES + self.rpc.tokens[MINT] = self.rpc.tokens.get(MINT, 0) + BUY_TOKENS + self.rpc.landed.add(sig) + return {"status": "Failed", "signature": sig, "code": -1001, "error": "unknown", "_latency_ms": 500} + if outcome == "fail_to_land": + return {"status": "Failed", "signature": sig, "code": -1000, "error": "Transaction failed to land", "_latency_ms": 30_000} + raise AssertionError(outcome) + + def program_id_to_label(self): + return {FAKE_DEX: "FakeDex"} + + +@pytest.fixture +def gate_open(monkeypatch): + monkeypatch.setattr(broker_live, "assert_signing_allowed", lambda: None) + monkeypatch.setattr(broker_live.time, "sleep", lambda s: None) + + +def _broker(outcomes, lamports=1_000_000_000, tokens=None): + kp = Keypair() + rpc = FakeRpc(lamports, tokens, {MINT: 6}) + jup = FakeJup(kp, rpc, outcomes) + return LiveBroker(rpc=rpc, jup=jup, keypair=kp), rpc, jup + + +def _swap(broker, conn, decision_id, side="buy", amount=BUY_LAMPORTS, max_slippage=300): + return broker.swap(conn, decision_id=decision_id, book="live", side=side, mint=MINT, amount_in=amount, + slippage_bps=150, max_slippage_bps=max_slippage, step_bps=100) + + +def _orders(conn, decision_id): + return conn.execute("SELECT * FROM orders WHERE decision_id = %s ORDER BY attempt", (decision_id,)).fetchall() + + +def _fills(conn, order_ids): + return conn.execute("SELECT * FROM fills WHERE order_id = ANY(%s) ORDER BY id", (list(order_ids),)).fetchall() + + +def test_verified_buy_lands_orders_and_fills_rows(db_conn, gate_open): + broker, rpc, jup = _broker(["success_buy"]) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert res.ok and res.attempts == 1 and res.status == "confirmed" and res.code == 0 and res.error is None + assert res.token_delta == BUY_TOKENS and res.lamports_delta == -(BUY_LAMPORTS + BUY_FEES) + assert res.price_sol == pytest.approx(((BUY_LAMPORTS + BUY_FEES) / 1e9) / (BUY_TOKENS / 1e6)) + assert jup.orders[0]["exclude_routers"] == "jupiterz" and jup.orders[0]["slippage_bps"] == 150 + assert jup.orders[0]["input_mint"] == config.WSOL_MINT and jup.orders[0]["output_mint"] == MINT + + orders = _orders(db_conn, did) + assert len(orders) == 1 + o = orders[0] + assert o["id"] == res.order_id and o["status"] == "confirmed" and o["side"] == "buy" and o["book"] == "live" + assert o["input_mint"] == config.WSOL_MINT and o["output_mint"] == MINT and int(o["amount_in"]) == BUY_LAMPORTS + assert o["slippage_bps"] == 150 and o["router"] == "metis" and o["fee_bps"] == 10 and o["error_code"] == 0 + assert set(o["program_ids"]) == {SYSTEM, FAKE_DEX} + assert o["request_id"] == o["order_response"]["requestId"] == jup.executes[0][0] + assert o["last_valid_block_height"] == int(o["order_response"]["lastValidBlockHeight"]) == jup.executes[0][1] + assert o["signature"] == res.signature and o["execute_response"]["status"] == "Success" + assert o["execute_request"]["requestId"] == o["request_id"] and o["latency_ms"] == 900 + assert o["request"]["taker"] == broker.pubkey and "excludeRouters" in o["request"] + + fills = _fills(db_conn, [o["id"]]) + assert len(fills) == 1 + f = fills[0] + assert f["id"] == res.fill_id and f["verified_by"] == "balance_delta" and f["book"] == "live" + assert int(f["token_delta"]) == BUY_TOKENS and f["sol_delta_lamports"] == -(BUY_LAMPORTS + BUY_FEES) + assert f["fee_lamports"] == BUY_FEES and f["mint"] == MINT and f["side"] == "buy" and f["slot"] == 5 + assert f["signature"] == res.signature and f["platform_fee"]["feeBps"] == 10 + assert f["pre_snapshot"]["lamports"] == 1_000_000_000 and MINT not in f["pre_snapshot"]["tokens"] + assert f["post_snapshot"]["tokens"][MINT] == BUY_TOKENS and f["post_snapshot"]["decimals"][MINT] == 6 + + +def test_verified_sell(db_conn, gate_open): + broker, rpc, jup = _broker(["success_sell"], tokens={MINT: BUY_TOKENS}) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did, side="sell", amount=BUY_TOKENS) + assert res.ok and res.token_delta == -BUY_TOKENS and res.lamports_delta == 9_984_000 + assert res.price_sol == pytest.approx(9_984_000 / 1e9 / 1.0) + assert jup.orders[0]["input_mint"] == MINT and jup.orders[0]["output_mint"] == config.WSOL_MINT + f = _fills(db_conn, [res.order_id])[0] + assert f["verified_by"] == "balance_delta" and f["side"] == "sell" and f["fee_lamports"] == 6_000 + + +def test_success_without_movement_is_unverified(db_conn, gate_open): + broker, rpc, jup = _broker(["success_no_move"]) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert not res.ok and res.attempts == 1 and res.status == "confirmed" and res.token_delta == 0 and res.lamports_delta == 0 + orders = _orders(db_conn, did) + assert len(orders) == 1 and orders[0]["status"] == "confirmed" + fills = _fills(db_conn, [orders[0]["id"]]) + assert len(fills) == 1 and fills[0]["verified_by"] == "unverified" and fills[0]["id"] == res.fill_id + ev = db_conn.execute("SELECT * FROM events WHERE source = 'broker_live' AND (detail->>'order_id')::bigint = %s", + (orders[0]["id"],)).fetchall() + assert len(ev) == 1 and ev[0]["level"] == "warning" and ev[0]["detail"]["tx_found"] is True + + +def test_failed_status_with_moved_balances_is_a_fill(db_conn, gate_open): + broker, rpc, jup = _broker(["failed_but_moved"]) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert res.ok and res.attempts == 1 and res.code == -1001 + f = _fills(db_conn, [res.order_id])[0] + assert f["verified_by"] == "balance_delta_despite_failed" and int(f["token_delta"]) == BUY_TOKENS + + +def test_failed_to_land_retries_with_higher_slippage(db_conn, gate_open): + broker, rpc, jup = _broker(["fail_to_land", "success_buy"]) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert res.ok and res.attempts == 2 and res.code == 0 + assert [o["slippage_bps"] for o in jup.orders] == [150, 250] + orders = _orders(db_conn, did) + assert [o["attempt"] for o in orders] == [1, 2] + assert [o["slippage_bps"] for o in orders] == [150, 250] + assert [o["status"] for o in orders] == ["expired", "confirmed"] + assert orders[0]["error_code"] == -1000 and "land" in orders[0]["error"] + assert orders[0]["request_id"] != orders[1]["request_id"] + assert rpc.status_polls >= 2 # the first attempt was polled until lastValidBlockHeight passed + fills = _fills(db_conn, [o["id"] for o in orders]) + assert len(fills) == 1 and fills[0]["order_id"] == orders[1]["id"] == res.order_id + + +def test_retry_ladder_stops_at_max_slippage(db_conn, gate_open): + broker, rpc, jup = _broker(["fail_to_land", "fail_to_land", "fail_to_land"]) + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did, max_slippage=250) + assert not res.ok and res.attempts == 2 and res.code == -1000 and res.status == "expired" + assert [o["slippage_bps"] for o in jup.orders] == [150, 250] + assert len(jup.outcomes) == 1 # third outcome never consumed + orders = _orders(db_conn, did) + assert len(orders) == 2 and not _fills(db_conn, [o["id"] for o in orders]) + + +def test_order_error_code_is_recorded_and_not_retried(db_conn, gate_open): + broker, rpc, jup = _broker([]) + jup.order = lambda *a, **k: {"errorCode": 1, "errorMessage": "Insufficient funds", "transaction": None} + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert not res.ok and res.status == "order_error" and res.code == 1 and "Insufficient" in res.error + orders = _orders(db_conn, did) + assert len(orders) == 1 and orders[0]["status"] == "order_error" and orders[0]["error_code"] == 1 + assert orders[0]["order_response"]["errorCode"] == 1 + + +def test_fee_payer_mismatch_refused(db_conn, gate_open): + broker, rpc, jup = _broker(["success_buy"]) + real_order = jup.order + + def bad_order(*a, **k): + o = real_order(*a, **k) + o["signatureFeePayer"] = str(Pubkey.new_unique()) + return o + + jup.order = bad_order + did = random.randrange(1 << 40) + res = _swap(broker, db_conn, did) + assert not res.ok and res.status == "order_error" and "signatureFeePayer" in res.error + assert jup.executes == [] + + +def test_signing_gate_is_checked_first(db_conn, monkeypatch): + monkeypatch.setattr(config, "LIVE_ENABLED", False) + broker, rpc, jup = _broker(["success_buy"]) + with pytest.raises(SigningRefused): + _swap(broker, db_conn, random.randrange(1 << 40)) + assert jup.orders == [] diff --git a/tests/test_cluster_guard.py b/tests/test_cluster_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..0bd4a97c4c5ab2bef4d4a477b5d9b0e2e995c1e1 --- /dev/null +++ b/tests/test_cluster_guard.py @@ -0,0 +1,37 @@ +import pytest + +from fly_trader import config +from fly_trader.chain.cluster_guard import SigningRefused, assert_signing_allowed, signing_allowed + + +def _open_gate(monkeypatch): + monkeypatch.setattr(config, "SOLANA_CLUSTER", "mainnet-beta") + monkeypatch.setattr(config, "LIVE_ENABLED", True) + monkeypatch.setattr(config, "live_prerequisites_missing", lambda: []) + + +def test_default_deny(monkeypatch): + monkeypatch.setattr(config, "LIVE_ENABLED", False) + with pytest.raises(SigningRefused, match="LIVE_ENABLED"): + assert_signing_allowed() + assert signing_allowed() is False + + +def test_wrong_cluster_refused_even_when_live(monkeypatch): + _open_gate(monkeypatch) + monkeypatch.setattr(config, "SOLANA_CLUSTER", "devnet") + with pytest.raises(SigningRefused, match="devnet"): + assert_signing_allowed() + + +def test_missing_prerequisite_refused(monkeypatch): + _open_gate(monkeypatch) + monkeypatch.setattr(config, "live_prerequisites_missing", lambda: ["BOT_PRIVATE_KEY"]) + with pytest.raises(SigningRefused, match="BOT_PRIVATE_KEY"): + assert_signing_allowed() + + +def test_allowed_only_under_the_full_conjunction(monkeypatch): + _open_gate(monkeypatch) + assert assert_signing_allowed() is None + assert signing_allowed() is True diff --git a/tests/test_connectome.py b/tests/test_connectome.py new file mode 100644 index 0000000000000000000000000000000000000000..ba79380888b6fb62d950b8aec70b3099cb2bfba8 --- /dev/null +++ b/tests/test_connectome.py @@ -0,0 +1,54 @@ +"""The connectome as the fly sees it: scale.json's pathway gains drop the silenced edges, the mushroom-body block +(KC→MBON, kept apart from the sparse graph by the build) survives a sub-connectome, and the real artifact has no +KC→MBON edge in its sparse graph.""" +import json + +import numpy as np +import pytest + +from fly_trader.brain import connectome as cn + +POPS = {"KC": [0, 4], "MBON_APP": [4, 5], "MBON_AV": [5, 6], "MBON_OTHER": [6, 7], "DAN_PAM": [7, 8], "VISUAL": [8, 10], "DESCENDING": [10, 12]} + + +def _artifact(tmp_path, monkeypatch, gains): + post, pre = [4, 0, 10, 11, 9, 10], [7, 7, 4, 5, 8, 9] # DAN→MBON, DAN→KC, MBON→DN ×2, VISUAL→VISUAL, VISUAL→DN + W = np.zeros((4, 3), np.float32); W[0, 0] = 7; W[1, 1] = 2; W[3, 2] = 1 + path = tmp_path / "t.npz" + np.savez(path, N=12, indices=np.array([post, pre], np.int64), values_raw=np.array([3, 2, 5, -4, 1, 6], np.float32), + pop_ranges=json.dumps(POPS), W_KM0=W, M_KM=W != 0, spectral_radius_raw=10.0, content_sha256="x", s0=0.1) + monkeypatch.setattr(cn, "SCALE_FILE", tmp_path / "scale.json") + (tmp_path / "scale.json").write_text(json.dumps({"connectome": "t.npz", "s": 0.05, "pathway_gains": gains})) + return path + + +def test_pathway_gains_drop_silenced_edges_and_the_mb_block_loads(tmp_path, monkeypatch): + c = cn.Connectome(_artifact(tmp_path, monkeypatch, {"DAN_PAM->MBON_APP": 0.0, "DAN_PAM->KC": 0.0, "MBON_AV->DESCENDING": 0.5}), device="cpu") + pre = c.indices[1].tolist() + assert 7 not in pre and c.values_raw.numel() == 4 # both DAN edges gone + assert float(c.values_raw[c.indices[1] == 5]) == pytest.approx(-2.0) # scaled, sign kept + assert c.s == pytest.approx(0.05) and tuple(c.W_KM0.shape) == (4, 3) and int(c.M_KM.sum()) == 3 + + +def test_sub_connectome_keeps_the_mushroom_body_and_refuses_to_drop_it(tmp_path, monkeypatch): + c = cn.Connectome(_artifact(tmp_path, monkeypatch, {}), device="cpu") + sub = cn.SubConnectome(c, exclude=("VISUAL",)) + assert sub.N == 10 and sub.pop_ranges["KC"] == (0, 4) and sub.pop_ranges["DESCENDING"] == (8, 10) + assert sub.W_KM0 is c.W_KM0 and sub.M_KM is c.M_KM + assert sub.indices.shape[1] == 4 and int(sub.indices[1].max()) <= 7 # both VISUAL edges dropped + with pytest.raises(ValueError, match="mushroom body"): + cn.SubConnectome(c, exclude=("KC",)) + + +def test_real_artifact_has_no_kc_to_mbon_edge_in_the_sparse_graph(): + try: + c = cn.Connectome.load(device="cpu") + except FileNotFoundError: + pytest.skip("no connectome artifact") + (k0, k1), m0, m1 = c.pop_ranges["KC"], c.pop_ranges["MBON_APP"][0], c.pop_ranges["MBON_OTHER"][1] + post, pre = c.indices[0], c.indices[1] + assert not bool(((pre >= k0) & (pre < k1) & (post >= m0) & (post < m1)).any()) + assert tuple(c.W_KM0.shape) == (k1 - k0, m1 - m0) and bool(c.M_KM.any()) + d0, d1 = c.pop_ranges["DAN_PAM"] + if c.pathway_gains.get("DAN_PAM->MBON_APP") == 0.0: + assert not bool(((pre >= d0) & (pre < d1) & (post >= m0) & (post < c.pop_ranges["MBON_APP"][1])).any()) diff --git a/tests/test_curve_parity.py b/tests/test_curve_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..623b032a912dfb00f67bdc31d76301413042dbde --- /dev/null +++ b/tests/test_curve_parity.py @@ -0,0 +1,65 @@ +"""Graduation-time rug facts: the archive scan (corpus_meta.update_curve_facts) and the live stream (pumpstream curve +tracking + _graduation) compute the same bundle share, dev holding and selling, holding concentration and insider +wallets from the same curve trading; the creator's first buy rides on the create event.""" +from datetime import datetime, timedelta, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from fly_trader import config +from fly_trader.db.connection import transaction +from fly_trader.ingest import pumpstream +from fly_trader.train import corpus_meta + +T0 = datetime(2026, 9, 4, 10, 0, tzinfo=timezone.utc) +SUPPLY, IB = 1e9, 1e6 +LEGS = [ # (seconds, slot, trader, side, tokens) + (1, 100, "B1", 1, 2e6), (2, 100, "B9", 1, 1e6), (10, 105, "B2", 1, 3e6), (20, 110, "C", -1, 5e5), (30, 120, "B2", -1, 1e6), (35, 121, "B9", -1, 1e6)] + + +def _facts(conn, mint): + r = conn.execute("SELECT " + ", ".join(corpus_meta.CURVE_COLS) + ", curve_known FROM corpus_meta WHERE mint = %s", (mint,)).fetchone() + ins = sorted((x["wallet"], x["kind"]) for x in conn.execute("SELECT wallet, kind FROM token_insiders WHERE mint = %s", (mint,)).fetchall()) + return dict(r), ins + + +def test_expected_values(): + facts, ins = corpus_meta.curve_facts(100, "C", SUPPLY, IB, {"B1": [100, 2e6, 0], "B9": [100, 1e6, 1e6], "B2": [105, 3e6, 1e6], "C": [None, 0, 5e5]}) + assert facts["bundle_share"] == pytest.approx(2e6 / SUPPLY) and facts["dev_hold_share"] == pytest.approx(5e5 / SUPPLY) + assert facts["dev_sold_frac"] == pytest.approx(0.5) and facts["grad_hhi"] == pytest.approx((2 / 4.5) ** 2 + (2 / 4.5) ** 2 + (0.5 / 4.5) ** 2) + assert ins == [("C", "creator"), ("B1", "bundle"), ("B9", "bundle")] + + +def test_archive_and_live_curve_facts_match(tmp_path, monkeypatch, db_conn): + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay") + day = tmp_path / "replay" / T0.date().isoformat(); day.mkdir(parents=True) + arch, live = "CurveArchpump", "CurveLivepump" + g = T0 + timedelta(seconds=40) + pq.write_table(pa.table({"ts": pa.array([T0], pa.timestamp("ms", tz="UTC")), "slot": [100], "action": ["create"], "pool": ["pump"], "mint": [arch]}), + day / "10_events.parquet") + pq.write_table(pa.table({"ts": pa.array([T0 + timedelta(seconds=s) for s, *_ in LEGS], pa.timestamp("ms", tz="UTC")), "slot": [sl for _, sl, *_ in LEGS], + "pool": ["pump"] * len(LEGS), "mint": [arch] * len(LEGS), "trader": [t for *_, t, _, _ in LEGS], + "side": pa.array([sd for *_, sd, _ in LEGS], pa.int8()), "tokens": [v for *_, v in LEGS]}), day / "10_trades.parquet") + with transaction() as conn: + conn.execute("INSERT INTO replay_hours (hour, status) VALUES (%s, 'done') ON CONFLICT (hour) DO UPDATE SET status = 'done'", (T0,)) + conn.execute("DELETE FROM corpus_meta WHERE mint IN (%s, %s)", (arch, live)); conn.execute("DELETE FROM token_insiders WHERE mint IN (%s, %s)", (arch, live)) + conn.execute("INSERT INTO corpus_meta (mint, create_ts, graduated_at, creator, supply, dev_tokens) VALUES (%s,%s,%s,'C',%s,%s)", (arch, T0, g, SUPPLY, IB)) + assert corpus_meta.update_curve_facts() >= 1 + agg = pumpstream.Aggregator(); ms = lambda s: int((T0.timestamp() + s) * 1000) + agg.ingest({"action": "create", "pool": "pump", "mint": live, "timestamp": ms(0), "block": 100, "txSigner": "C", "supply": SUPPLY, "initialBuy": IB, + "quoteAmount": 0.03, "signature": "sigCurveLive"}) + for s, sl, t, sd, v in LEGS: + agg.ingest({"action": "buy" if sd == 1 else "sell", "pool": "pump", "mint": live, "timestamp": ms(s), "block": sl, "txSigner": t, "tokenAmount": v}) + agg.ingest({"action": "migrate", "pool": "pump-amm", "mint": live, "timestamp": ms(40), "block": 130, "poolId": "PL", "txSigner": "M", "quoteInPool": 80.0, + "signature": "sigCurveMig"}) + with transaction() as conn: + fa, ia = _facts(conn, arch); fl, il = _facts(conn, live) + conn.execute("DELETE FROM corpus_meta WHERE mint IN (%s, %s)", (arch, live)); conn.execute("DELETE FROM token_insiders WHERE mint IN (%s, %s)", (arch, live)) + conn.execute("DELETE FROM pump_events WHERE sig IN ('sigCurveLive', 'sigCurveMig')"); conn.execute("DELETE FROM corpus_tokens WHERE mint = %s", (live,)) + conn.execute("DELETE FROM replay_hours WHERE hour = %s", (T0,)) + assert fa["curve_known"] is True and fl["curve_known"] is True + for c in corpus_meta.CURVE_COLS: + assert fa[c] == pytest.approx(fl[c]), c + assert ia == il == [("B1", "bundle"), ("B9", "bundle"), ("C", "creator")] + assert agg.insiders[live] == frozenset({"C", "B1", "B9"}) diff --git a/tests/test_custom_pools.py b/tests/test_custom_pools.py new file mode 100644 index 0000000000000000000000000000000000000000..cd2da4d5bfccd95f7cc5deb574f7f6e0dc279a91 --- /dev/null +++ b/tests/test_custom_pools.py @@ -0,0 +1,97 @@ +"""Directly created (custom) PumpSwap pools are excluded from the universe: live stream, training aggregates, archive backfill.""" +from datetime import date, datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq + +from fly_trader import config +from fly_trader.ingest import pumpstream +from fly_trader.train import corpus_meta, mature + +WSOL = pumpstream.WSOL + + +def _leg(mint, ts_ms, price, pool_id, action="buy"): + return {"action": action, "pool": "pump-amm", "mint": mint, "timestamp": ts_ms, "price": price, "quoteMint": WSOL, "quoteInPool": 80.0, + "txSigner": "t", "breakdown": [{"action": action, "trader": "t", "quoteAmount": 1.0, "tokenAmount": 1.0}], "poolId": pool_id} + + +def _create_pool(mint, ts_ms, pool_id, by="custom"): + e = {"action": "createPool", "pool": "pump-amm", "mint": mint, "timestamp": ts_ms, "poolId": pool_id, "quoteMint": WSOL} + if by is not None: + e["poolCreatedBy"] = by + return e + + +def test_stream_drops_custom_pool_legs(db_conn): + agg = pumpstream.Aggregator(); m = 1_800_000_000 + agg.ingest(_leg("Apump", (m + 1) * 1000, 1.0, "MIGRATED")) # a migrated pool (no createPool here): kept + agg.ingest(_create_pool("Bpump", (m + 2) * 1000, "CUSTOM_T1")) + agg.ingest(_create_pool("Cpump", (m + 2) * 1000, "CUSTOM_T2", by=None)) # missing poolCreatedBy counts as custom + agg.ingest(_create_pool("Dpump", (m + 2) * 1000, "MIGR_T3", by="pump")) # created by a pump.fun migration: not blocked + for pid, mint in (("CUSTOM_T1", "Bpump"), ("CUSTOM_T2", "Cpump"), ("MIGR_T3", "Dpump"), ("MIGRATED", "Apump")): + agg.ingest(_leg(mint, (m + 5) * 1000, 1.0, pid)); agg.ingest(_leg(mint, (m + 6) * 1000, 1.0, pid, "sell")) + assert agg.blocked == {"CUSTOM_T1", "CUSTOM_T2"} and agg.stats["dropped_custom_pool"] == 4 + assert set(agg.minutes[m]) == {"Apump", "Dpump"} and agg.row(m, "Apump").nb == 2 + try: + assert agg.write_pools() == 2 and agg.pending_pools == [] + rows = db_conn.execute("SELECT pool_id, mint, created_by, source, ts FROM pump_pools WHERE pool_id LIKE 'CUSTOM_T%' ORDER BY 1").fetchall() + assert [(r["pool_id"], r["mint"], r["created_by"], r["source"]) for r in rows] == [("CUSTOM_T1", "Bpump", "custom", "stream"), ("CUSTOM_T2", "Cpump", None, "stream")] + assert rows[0]["ts"] == datetime.fromtimestamp(m + 2, timezone.utc) + finally: + db_conn.execute("DELETE FROM pump_pools WHERE pool_id LIKE 'CUSTOM_T%'"); db_conn.commit() + + +def test_stream_loads_blocked_set(db_conn): + db_conn.execute("INSERT INTO pump_pools (pool_id, mint, created_by, source) VALUES ('LOAD_T1', 'Lpump', 'custom', 'archive'), ('LOAD_T2', 'Lother', 'custom', 'archive') " + "ON CONFLICT DO NOTHING"); db_conn.commit() + try: + agg = pumpstream.Aggregator(); pumpstream._load_blocked(agg) + assert "LOAD_T1" in agg.blocked and "LOAD_T2" not in agg.blocked # a non-pump mint never reaches the pool check + agg.ingest(_leg("Lpump", 1_800_000_001_000, 1.0, "LOAD_T1")) + assert agg.stats["dropped_custom_pool"] == 1 and not agg.minutes + finally: + db_conn.execute("DELETE FROM pump_pools WHERE pool_id LIKE 'LOAD_T%'"); db_conn.commit() + + +def test_aggregate_day_excludes_custom_pools(db_conn, tmp_path, monkeypatch): + d = date(2031, 1, 1); t0 = datetime(2031, 1, 1, tzinfo=timezone.utc).timestamp() + rows = [] + for k in range(3): # GOOD: minutes 0..2; AGG_BAD (custom) minutes 5..7 for the same mint, and a mint only in AGG_BAD + rows.append(("Xpump", t0 + 60 * k + 1, "GOOD", 1.0)); rows.append(("Xpump", t0 + 60 * (5 + k) + 1, "AGG_BAD", 1.5)); rows.append(("Ypump", t0 + 60 * k + 1, "AGG_BAD", 2.0)) + tab = pa.table({"ts": pa.array([datetime.fromtimestamp(r[1], timezone.utc) for r in rows], pa.timestamp("ms", tz="UTC")), "slot": pa.array(range(len(rows)), pa.int64()), + "pool": ["pump-amm"] * len(rows), "mint": [r[0] for r in rows], "trader": ["t"] * len(rows), "side": pa.array([1] * len(rows), pa.int8()), + "sol": [1.0] * len(rows), "price": [r[3] for r in rows], "quote_in_pool": [80.0] * len(rows), "quote_mint": [WSOL] * len(rows), + "pool_id": [r[2] for r in rows]}) + (tmp_path / "replay" / d.isoformat()).mkdir(parents=True) + pq.write_table(tab, tmp_path / "replay" / d.isoformat() / "00_trades.parquet") + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay"); monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature") + db_conn.execute("INSERT INTO pump_pools (pool_id, mint, created_by, source) VALUES ('AGG_BAD', 'Ypump', 'custom', 'archive') ON CONFLICT DO NOTHING"); db_conn.commit() + try: + assert mature.aggregate_day(d) == 3 + out = pq.read_table(tmp_path / "mature" / f"{d.isoformat()}.parquet").to_pandas() + assert set(out["mint"]) == {"Xpump"} and (out["close"] == 1.0).all() and len(out) == 3 + assert mature.part_version(tmp_path / "mature" / f"{d.isoformat()}.parquet") == mature.AGG_VERSION == 4 + finally: + db_conn.execute("DELETE FROM pump_pools WHERE pool_id = 'AGG_BAD'"); db_conn.execute("DELETE FROM mature_days WHERE day = %s", (d,)); db_conn.commit() + + +def test_archive_backfill_idempotent(db_conn, tmp_path, monkeypatch): + day = tmp_path / "replay" / "2031-01-02"; day.mkdir(parents=True) + t = lambda s: datetime(2031, 1, 2, 0, 0, s, tzinfo=timezone.utc) + ev = pa.table({"ts": pa.array([t(5), t(1), t(2), t(3), t(4)], pa.timestamp("ms", tz="UTC")), "sig": ["s1", "s2", "s3", "s4", "s5"], + "action": ["createPool", "createPool", "createPool", "migrate", "createPool"], "pool": ["pump-amm", "pump-amm", "pump-amm", "pump-amm", "raydium-cpmm"], + "mint": ["Apump", "Apump", "Bpump", "Cpump", "Dpump"], "pool_id": ["BF_C1", "BF_C1", "BF_C2", "BF_M1", "BF_R1"], + "pool_created_by": ["custom", "custom", None, "pump", "custom"]}) + pq.write_table(ev, day / "00_events.parquet") + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay") + db_conn.execute("DELETE FROM ui_settings WHERE key = 'pump_pools_backfill'"); db_conn.execute("DELETE FROM pump_pools WHERE pool_id LIKE 'BF_%'"); db_conn.commit() + try: + assert corpus_meta.backfill_pump_pools() == 2 + assert corpus_meta.backfill_pump_pools() == 0 # re-run: nothing new + rows = db_conn.execute("SELECT pool_id, mint, created_by, ts, source FROM pump_pools WHERE pool_id LIKE 'BF_%' ORDER BY 1").fetchall() + assert [(r["pool_id"], r["mint"], r["created_by"], r["source"]) for r in rows] == [("BF_C1", "Apump", "custom", "archive"), ("BF_C2", "Bpump", None, "archive")] + assert rows[0]["ts"] == t(1) # the pool's first createPool + assert "BF_C1" in corpus_meta.blocked_pool_ids(db_conn) + finally: + db_conn.execute("DELETE FROM ui_settings WHERE key = 'pump_pools_backfill'"); db_conn.execute("DELETE FROM pump_pools WHERE pool_id LIKE 'BF_%'"); db_conn.commit() diff --git a/tests/test_decisions.py b/tests/test_decisions.py new file mode 100644 index 0000000000000000000000000000000000000000..cb97b9d77e87a177d86c0acdca3ec7a87fa4f887 --- /dev/null +++ b/tests/test_decisions.py @@ -0,0 +1,96 @@ +"""Decision points: labels, pessimistic fills, the past-only scale-break rule, holds and per-day accounting.""" +from datetime import datetime, timedelta, timezone + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from fly_trader.market.features import FEATURES +from fly_trader.train import decisions +from fly_trader.train.mature import SCHEMA + + +def _part(tmp_path, rows, version=None): + from fly_trader.market.features import FEATURE_VERSION + from fly_trader.train.mature import AGG_VERSION, write_part + write_part(pa.Table.from_pylist(rows, schema=SCHEMA), tmp_path / "2026-09-01" / "part.parquet", FEATURE_VERSION if version is None else version, {"fly_agg": AGG_VERSION}) + return tmp_path + + +def _row(mint, ts, close, resq=100.0, open_=None, logvol_15m=3.0): + r = {"mint": mint, "ts": ts, "phase": "amm", "t_rel_min": 10, "open": open_ if open_ is not None else close, "high": close, "low": close, "close": close, + "volume_sol": 10.0, "resq": resq, "age_h": 2.0, "has_trades": False, "mask": 0, + "pre_minutes": 0.0, "pre_trades": 0.0, "pre_buyers": 0.0, "pre_vol_sol": 0.0, "pre_top10_pct": 0.0, + "traders_15m": 5.0, "traders_1h": 10.0, "n_trades_1m": 2.0} + for f in FEATURES: + r[f] = 0.0 + r["logvol_15m"] = logvol_15m + return r + + +def test_labels_fills_and_past_only_rule(tmp_path): + t0 = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) + rows = [] + # token A: flat 1.0 for 40 minutes, then +10 % at minute 20 relative to a decision at minute 0 + for k in range(80): + px = 1.0 if k < 20 else 1.10 + rows.append(_row("A", t0 + timedelta(minutes=k), px, open_=px * 1.01 if k == 1 else px)) + # token B: normal for 30 minutes, then a 100x print at minute 30 (scale break), then normal again + for k in range(80): + px = 1.0 if k != 30 else 100.0 + rows.append(_row("B", t0 + timedelta(minutes=k), px)) + ds = decisions.build(days=None, feature_dir=_part(tmp_path, rows), horizon_min=30, fee=0.0) + a = ds.mint == "A"; b = ds.mint == "B" + # A at minute 0 sees the close at minute 30 (1.10): +10 %; pessimistic fill uses minute 1's open (1.01) + i0 = np.flatnonzero(a & (ds.ts == t0.timestamp()))[0] + assert ds.fwd[i0] == pytest.approx(0.10, abs=1e-6) + assert ds.fwd_pess[i0] == pytest.approx(1.10 / 1.01 - 1, abs=1e-6) + assert ds.y[i0] == 1 + # the last 35 minutes of the file have no full horizon: ineligible + assert ds.ts.max() <= (t0 + timedelta(minutes=80 - 36)).timestamp() + # B: rows before the break are eligible and their labels see the 100x print (no leak in either direction); + # rows from the break onward are gone + tb = ds.ts[b] + assert tb.max() < (t0 + timedelta(minutes=30)).timestamp() + assert not (b & (ds.ts == t0.timestamp())).any() # its 30-minute exit lands on the reverting 100x print: not a label + ib5 = np.flatnonzero(b & (ds.ts == (t0 + timedelta(minutes=5)).timestamp()))[0] + assert ds.fwd[ib5] == pytest.approx(0.0, abs=1e-6) # its mark (minute 35) is back at 1.0 + + +def test_model_costs_on_both_sides_and_random_baseline(tmp_path): + t0 = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) + rows = [] + for k in range(80): # flat price; entry minutes cost 1 %, the rest 2 % one-way + r = _row("C", t0 + timedelta(minutes=k), 1.0); r["exit_cost_0p1"] = 0.01 if k < 10 else 0.02; rows.append(r) + ds = decisions.build(days=None, feature_dir=_part(tmp_path, rows), horizon_min=30) # default: the paper broker's cost model + i0 = np.flatnonzero(ds.ts == t0.timestamp())[0] + from fly_trader.market.exit_cost import cost_at_size # costs are charged at the size the book trades + e_in, e_out = cost_at_size(0.01, 100.0), cost_at_size(0.02, 100.0) + assert e_in > 0.01 and e_out > 0.02 # dearer than the stored 0.1 SOL figure + assert ds.fwd[i0] == pytest.approx((1 - e_in) * (1 - e_out) - 1, abs=1e-6) # entry cost at the signal minute, exit cost at the exit minute + rr = decisions.random_trades(ds, np.ones(len(ds.y), bool), 3, seeds=2) + assert len(rr) >= 2 and (rr < 0).all() # a flat market loses exactly the costs + assert len(decisions.random_trades(ds, np.ones(len(ds.y), bool), 0)) == 0 + + +def test_trades_respect_hold_and_summaries(): + ts = np.array([0.0, 60.0, 120.0, 1800.0, 1900.0, 0.0], dtype=float) + ds = decisions.DecisionSet(X=np.zeros((6, 1), np.float32), y=np.zeros(6, np.int8), fwd=np.array([0.1, 0.2, 0.3, 0.4, 0.5, -0.1], np.float32), + fwd_pess=np.array([0.1, 0.2, 0.3, 0.4, 0.5, -0.1], np.float32), + day=np.array([datetime(2026, 9, 1).date()] * 5 + [datetime(2026, 9, 2).date()], dtype=object), + ts=ts, mint=np.array(["A", "A", "A", "A", "A", "B"]), cols=["x"], horizon_s=1800.0) + r = decisions.trades_from_picks(ds, np.ones(6, bool)) + assert list(r) == pytest.approx([0.1, 0.4, -0.1]) # A re-enters only after the 30-minute hold; B once + s = decisions.summarize(np.array([0.1, 0.2], np.float32)) + assert s["pf"] is None and s["win"] == 1.0 # no losing trade → profit factor undefined, never inf + ev = decisions.evaluate(ds, np.ones(6), np.ones(6, bool), 0.5) + assert ev["pooled"]["n"] == 3 and ev["per_day"]["2026-09-01"]["n"] == 2 and ev["per_day"]["2026-09-02"]["n"] == 1 + + +def test_stale_feature_parts_are_refused(tmp_path): + t0 = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) + rows = [_row("A", t0 + timedelta(minutes=k), 1.0) for k in range(80)] + with pytest.raises(RuntimeError, match="feature or aggregation version"): + decisions.build(days=None, feature_dir=_part(tmp_path, rows, version=1), horizon_min=30, fee=0.0) diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000000000000000000000000000000000000..b9ff82b0f184338d5800489ac1a6eadcfd5f1481 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,37 @@ +import math, random, time +from fly_trader.market.features import TokenMeta, TokenState, FIDX, D +import numpy as np +import pytest +from fly_trader.market.exit_cost import impact_fraction, exit_cost_fraction, fee_fraction, pool_fee_rate + + +def _fill(mint, now, n=600, up=True): + st = TokenState(mint) + for i in range(n): + p = 1e-6 * math.exp((0.001 if up else -0.001) * i) + st.append(now - 3600 + i * 6, p, 0.2, random.random() < 0.7, f"s{i % 20}", 50.0) + return st + + +def test_features_shape_and_signs(): + random.seed(0) + now = time.time() + up, dn = _fill("UP", now, up=True), _fill("DN", now, up=False) + fu, mu = up.features(now, TokenMeta("UP", graduated_at=now - 7200)) + fd, md = dn.features(now, TokenMeta("DN", graduated_at=now - 7200)) + assert len(fu) == D and fu[FIDX["ret_1h"]] > 0 > fd[FIDX["ret_1h"]] + assert fu[FIDX["imb_1h"]] > 0 and mu & (1 << FIDX["ret_1h"]) + assert abs(fu[FIDX["log_liquidity_sol"]] - math.log1p(100.0)) < 1e-6 + assert up.volume_since(now - 10800) > 0 and up.last_price > 0 + + +def test_exit_cost(): + assert abs(impact_fraction(0.1, 100.0) - 0.1 / 100.1) < 1e-9 + assert impact_fraction(0.1, None) == 1.0 + assert 0 < exit_cost_fraction(0.1, 100.0, 1.0) < 0.02 + # measured pump.fun schedule: 1.25 % below 420 SOL market cap, stepping down to 0.30 % above 98,178 SOL + assert pool_fee_rate(100.0) == 0.0125 and pool_fee_rate(5000.0) == 0.01 and pool_fee_rate(2e5) == 0.003 + assert pool_fee_rate(None) == 0.0125 and list(pool_fee_rate(np.array([100.0, np.nan, 2e5]))) == [0.0125, 0.0125, 0.003] + # fees = pool fee + Jupiter 10 bps + 5,000-lamport network fee; a stream-reported pool fee overrides the schedule + assert fee_fraction(0.1, 2e5) == pytest.approx(0.003 + 0.001 + 5e-6 / 0.1) + assert fee_fraction(0.1, 2e5, pool_fee=0.0095) == pytest.approx(0.0095 + 0.001 + 5e-6 / 0.1) diff --git a/tests/test_fix_ingest.py b/tests/test_fix_ingest.py new file mode 100644 index 0000000000000000000000000000000000000000..7e12249fb9e09320dcf77295ad56b0c0bef52512 --- /dev/null +++ b/tests/test_fix_ingest.py @@ -0,0 +1,116 @@ +"""Ingest/db review fixes (2026-09-14): DSN parsing, corpus/replay DB resilience, token stats for the selector's universe.""" +from __future__ import annotations + +import threading +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone + +import httpx +import psycopg +import pytest + +from fly_trader import config +from fly_trader.db import schema +from fly_trader.db.connection import database_name +from fly_trader.ingest import corpus_pull, discovery + + +def _tx_on(conn): + @contextmanager + def tx(): + yield conn + return tx + + +# ---- 4: DSN parsing ---- +@pytest.mark.parametrize("url,name", [("postgresql://fly_trader:pw@localhost/fly_trader", "fly_trader"), + ("postgresql:///fly_trader_test", "fly_trader_test"), + ("dbname=fly_trader_test user=fly_trader host=/tmp", "fly_trader_test"), + ("host=localhost user=bob", "fly_trader")]) +def test_database_name_reads_urls_and_key_value_dsns(url, name): + assert database_name(url) == name + + +def test_ensure_database_admin_dsn_keeps_user_and_host(monkeypatch): + seen = [] + + class Conn: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def execute(self, sql, params=None): + class R: + def fetchone(self): + return (1,) + return R() + + monkeypatch.setattr(schema.psycopg, "connect", lambda conninfo, **kw: seen.append(conninfo) or Conn()) + assert schema.ensure_database("postgresql://fly_trader:pw@localhost/fly_trader") is False + assert psycopg.conninfo.conninfo_to_dict(seen[0]) == {"user": "fly_trader", "password": "pw", "host": "localhost", "dbname": "postgres"} + schema.ensure_database("dbname=fly_trader user=fly_trader host=/tmp") + assert psycopg.conninfo.conninfo_to_dict(seen[1]) == {"user": "fly_trader", "host": "/tmp", "dbname": "postgres"} + + +# ---- 2 / 3: corpus + replay resilience ---- +def test_db_retry_rides_out_postgres_errors(monkeypatch): + monkeypatch.setattr(corpus_pull, "_sleep", lambda s, stop: None) + calls = [] + + def flaky(x): + calls.append(x) + if len(calls) < 3: + raise psycopg.OperationalError("server closed the connection unexpectedly") + return x * 2 + + def bug(): + raise ValueError("not a database error") + + assert corpus_pull._db_retry(None, "t", flaky, 21) == 42 and len(calls) == 3 + stopped = threading.Event() + stopped.set() + assert corpus_pull._db_retry(stopped, "t", flaky, 1) is None + with pytest.raises(ValueError): + corpus_pull._db_retry(None, "t", bug) + + +def test_swap_api_4xx_is_permanent(): + api = corpus_pull.SwapApi(corpus_pull.Pacer(0.0, 0.0), corpus_pull.Status(), None) + api.c = httpx.Client(base_url="http://swap.test", transport=httpx.MockTransport(lambda req: httpx.Response(404, text="not found"))) + with pytest.raises(corpus_pull.Permanent): + api.get("/v2/coins/M/candles", {}) + + +def test_corpus_requeues_cooled_transient_errors_only(db_conn, monkeypatch): + monkeypatch.setattr(corpus_pull, "transaction", _tx_on(db_conn)) + grad = datetime.fromisoformat(config.CORPUS_PULL_BEFORE).replace(tzinfo=timezone.utc) - timedelta(days=1) + rows = {"TestFix3Pending": ("pending", None, 0.0), + "TestFix3TransientOld": ("error", "RuntimeError: swap-api /v2/coins/M/candles: gave up after retries", corpus_pull.RETRY_ERROR_H + 1), + "TestFix3TransientNew": ("error", "RuntimeError: swap-api /v2/coins/M/candles: gave up after retries", 0.5), + "TestFix3Permanent": ("error", "permanent: Permanent: swap-api /v2/coins/M/candles HTTP 404: nf", corpus_pull.RETRY_ERROR_H + 1)} + for m, (st, err, age_h) in rows.items(): + db_conn.execute("INSERT INTO corpus_tokens (mint, graduated_at, status, last_error, updated_at) VALUES (%s, %s, %s, %s, now() - %s * interval '1 hour')", + (m, grad, st, err, age_h)) + picked = {r["mint"] for r in corpus_pull._next_pending(limit=1_000_000)} + assert {"TestFix3Pending", "TestFix3TransientOld"} <= picked + assert not {"TestFix3TransientNew", "TestFix3Permanent"} & picked + + +# ---- stats for the selector's universe: token_stats only, no tokens/watch_pools side effects ---- +def test_refresh_stats_covers_selector_universe_without_watching(db_conn): + mint = "TestUniverseMint111111111111111111111111111" + db_conn.execute("INSERT INTO pump_minutes (mint, ts, close, resq_sol) VALUES (%s, date_trunc('minute', now()) - interval '5 minutes', 1e-7, 30.0)", (mint,)) + asked = [] + + class Client: + def search_many(self, mints): + asked.append(list(mints)) + return [{"id": mint, "organicScore": 42.0}] if mint in mints else [] + + c = discovery.refresh_stats(db_conn, Client()) + assert c["universe"] >= 1 and c["refreshed"] == 1 and any(mint in a for a in asked) + assert db_conn.execute("SELECT organic_score FROM token_stats WHERE mint = %s", (mint,)).fetchone()["organic_score"] == 42.0 + assert db_conn.execute("SELECT 1 FROM tokens WHERE mint = %s", (mint,)).fetchone() is None + assert db_conn.execute("SELECT 1 FROM watch_pools WHERE mint = %s", (mint,)).fetchone() is None diff --git a/tests/test_fix_live.py b/tests/test_fix_live.py new file mode 100644 index 0000000000000000000000000000000000000000..9cbbff23a00aea0389cd0892089cb5011d474319 --- /dev/null +++ b/tests/test_fix_live.py @@ -0,0 +1,145 @@ +"""Review fixes: partial-sell realized P&L, training reset leaves the circuit armed, close guard + runner lock, +confirmation polling past lastValidBlockHeight.""" +import uuid + +import pytest + +from fly_trader import config +from fly_trader.agent import runner +from fly_trader.db.connection import transaction +from fly_trader.execution import broker_live, ledger +from fly_trader.execution.broker_live import FillResult +from fly_trader.execution.broker_paper import PaperBroker +from fly_trader.execution.worker import ExecRequest, ExecutionWorker + + +def _pos(conn, pid): + return conn.execute("SELECT * FROM positions WHERE id=%s", (pid,)).fetchone() + + +# ---- 1. partial sells accumulate realized P&L; paper cash stays consistent ---- +def test_paper_partial_then_close_accumulates_realized(db_conn): + book, mint = f"replay_t{uuid.uuid4().hex[:8]}", f"M{uuid.uuid4().hex[:10]}" + pid = ledger.open_position(db_conn, book=book, mint=mint, pool=None, qty_raw=1_000_000, cost_sol=1.0, entry_price=1.0, + decision_id=None, fees_sol=0.0) + assert ledger.paper_cash(db_conn, book) == pytest.approx(config.CAPITAL_SOL - 1.0) + pb = PaperBroker(book) + kw = dict(decision_id=None, res_quote_sol=100.0, mcap_sol=100.0, program_label=None, forced_kind=None) + f1 = pb.sell(db_conn, position=ledger.open_positions(db_conn, book)[0], price=1.2, fraction=0.5, **kw) + assert f1.ok and 0 < f1.sol_delta < 0.6 + p = _pos(db_conn, pid) + assert p["status"] == "open" and int(p["qty"]) == 500_000 and p["cost_sol"] == pytest.approx(0.5) + assert p["realized_sol"] == pytest.approx(f1.sol_delta - 0.5) + assert ledger.paper_cash(db_conn, book) == pytest.approx(config.CAPITAL_SOL - 1.0 + f1.sol_delta) + stale = ledger.open_positions(db_conn, book)[0] + f2 = pb.sell(db_conn, position=stale, price=0.8, **kw) + assert f2.ok and "realized" in f2.reason + p = _pos(db_conn, pid) + assert p["status"] == "closed" and p["realized_sol"] == pytest.approx(f1.sol_delta + f2.sol_delta - 1.0) + assert ledger.paper_cash(db_conn, book) == pytest.approx(config.CAPITAL_SOL - 1.0 + f1.sol_delta + f2.sol_delta) + # 4. the close guard: a second close of the same position books nothing and records no fill + f3 = pb.sell(db_conn, position=stale, price=0.8, **kw) + assert not f3.ok and f3.fill_id is None + assert ledger.close_position(db_conn, position_id=pid, exit_price=1.0, proceeds_sol=5.0, fees_sol=0.0, + decision_id=None, forced_kind=None) is None + assert _pos(db_conn, pid)["realized_sol"] == pytest.approx(f1.sol_delta + f2.sol_delta - 1.0) + n = db_conn.execute("SELECT count(*) AS n FROM fills WHERE book=%s AND mint=%s AND side='sell'", (book, mint)).fetchone()["n"] + assert n == 2 + + +class _FakeBroker: + def __init__(self): + self.next: FillResult | None = None + + def swap(self, conn, **kw): + return self.next + + +def test_worker_partial_then_close_books_each_sale_once(): + mint = f"M{uuid.uuid4().hex[:10]}" + with transaction() as conn: + circ = dict(conn.execute("SELECT fail_count, tripped FROM circuit_state WHERE id=1").fetchone()) + pid = ledger.open_position(conn, book="live", mint=mint, pool=None, qty_raw=1_000_000, cost_sol=1.0, entry_price=1.0, + decision_id=None, fees_sol=0.0) + fb = _FakeBroker() + w = ExecutionWorker(fb) + try: + req = ExecRequest(decision_id=None, mint=mint, pool=None, side="sell", amount_in=400_000, slippage_bps=100, + max_slippage_bps=100, decimals=6, position_id=pid) + fb.next = FillResult(True, None, None, "s1", "confirmed", 0, None, -400_000, 500_000_000, 1.25, 1) + r1 = w._execute(req) + assert r1.ok and r1.realized_sol == pytest.approx(0.1) + fb.next = FillResult(True, None, None, "s2", "confirmed", 0, None, -600_000, 900_000_000, 1.5, 1) + r2 = w._execute(req) + assert r2.ok and r2.realized_sol == pytest.approx(0.3) + fb.next = FillResult(True, None, None, "s3", "confirmed", 0, None, -1, 1, 1.0, 1) + r3 = w._execute(req) + assert r3.ok and r3.realized_sol is None + with transaction() as conn: + p = _pos(conn, pid) + assert p["status"] == "closed" and p["realized_sol"] == pytest.approx(0.4) and p["cost_sol"] == pytest.approx(0.6) + finally: + w.stop() + with transaction() as conn: + conn.execute("DELETE FROM positions WHERE id=%s", (pid,)) + conn.execute("UPDATE circuit_state SET fail_count=%s, tripped=%s WHERE id=1", (circ["fail_count"], circ["tripped"])) + + +# ---- 2. a training reset never re-arms the circuit ---- +def test_training_reset_leaves_circuit_alone(tmp_path, monkeypatch): + from fly_trader.ops.reset import reset_training_state + monkeypatch.setattr(config, "PG_ARCHIVE_DIR", tmp_path / "arch") + monkeypatch.setattr(config, "BRAIN_DIR", tmp_path / "brain") + cols = "fail_count, tripped, kill_switch, kill_reason, peak_wealth" + with transaction() as conn: + before = dict(conn.execute(f"SELECT {cols} FROM circuit_state WHERE id=1").fetchone()) + conn.execute("UPDATE circuit_state SET fail_count=3, tripped=true, kill_switch=true, kill_reason='dd', peak_wealth=5.0 WHERE id=1") + try: + reset_training_state(reason="test", archive=False) + with transaction() as conn: + after = dict(conn.execute(f"SELECT {cols} FROM circuit_state WHERE id=1").fetchone()) + assert after == {"fail_count": 3, "tripped": True, "kill_switch": True, "kill_reason": "dd", "peak_wealth": 5.0} + finally: + with transaction() as conn: + conn.execute("UPDATE circuit_state SET fail_count=%(fail_count)s, tripped=%(tripped)s, kill_switch=%(kill_switch)s, " + "kill_reason=%(kill_reason)s, peak_wealth=%(peak_wealth)s WHERE id=1", before) + + +# ---- 4. one runner per database ---- +def test_runner_lock_refuses_a_second_runner(monkeypatch): + monkeypatch.setattr(runner, "setup", lambda name: None) + ran = [] + monkeypatch.setattr(runner, "_main", lambda stop_event=None: ran.append(1)) + held = runner.acquire_runner_lock() + try: + with pytest.raises(RuntimeError, match="runner lock"): + runner.acquire_runner_lock() + with pytest.raises(RuntimeError, match="runner lock"): + runner.main() + assert ran == [] + finally: + held.close() + runner.main() # lock free again; main releases it on return + assert ran == [1] + runner.acquire_runner_lock().close() + + +# ---- 5. a processed signature is polled past lastValidBlockHeight ---- +class _StatusRpc: + def __init__(self, statuses): + self.statuses = list(statuses) + + def get_signature_statuses(self, sigs): + return [self.statuses.pop(0)] + + def get_block_height(self): + return 10_000 + + +def test_await_confirmation_keeps_polling_a_processed_signature(monkeypatch): + monkeypatch.setattr(broker_live.time, "sleep", lambda s: None) + proc = {"err": None, "confirmationStatus": "processed"} + rpc = _StatusRpc([proc, proc, {"err": None, "confirmationStatus": "confirmed"}]) + assert broker_live.await_confirmation(rpc, "sig", 100)[0] == "confirmed" + rpc = _StatusRpc([proc, None]) + assert broker_live.await_confirmation(rpc, "sig", 100)[0] == "expired" diff --git a/tests/test_fix_train.py b/tests/test_fix_train.py new file mode 100644 index 0000000000000000000000000000000000000000..2d6425db647c277cbac367a4c55d1867e1a7f9cb --- /dev/null +++ b/tests/test_fix_train.py @@ -0,0 +1,236 @@ +"""Review fixes in the feature engine and the train pipeline (compaction, window highs, exit-cost size, mature +graduation refresh, causal replay band, backtest exits and baseline, corpus part commits, selector folds).""" +import math +import random +from datetime import date, datetime, timedelta, timezone + +import duckdb +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from fly_trader import config +from fly_trader.market.exit_cost import exit_cost_fraction +from fly_trader.market.features import FEATURE_VERSION, FEATURES, FIDX, TokenMeta, TokenState +from fly_trader.train import corpus_backtest as cb +from fly_trader.train import corpus_features as cf +from fly_trader.train import mature, selector +from fly_trader.train.decisions import DecisionSet +from fly_trader.train.replay_assemble import band_amm + + +# ---------------------------------------------------------------- market/features.py +def test_compaction_keeps_window_start_price(): + st = TokenState("M"); t = 0.0 + for i in range(20500): + st.append(t, 1.0, 0.1, True, f"s{i % 50}", 100.0); t += 1.0 + t += 3 * 3600 + 300 + st.append(t, 2.0, 0.1, True, "new", 100.0) # triggers compaction + assert len(st.ts) == 2 # the last pre-window trade survives + f, mask = st.features(t, TokenMeta("M")) + assert f[FIDX["ret_3h"]] == pytest.approx(math.log(2.0)) and mask & (1 << FIDX["ret_3h"]) + assert f[FIDX["logsigners_1h"]] == pytest.approx(math.log1p(1)) + assert st.cum_vol[-1] == pytest.approx(0.2) + + +def test_window_highs_exact_on_long_windows_and_after_compaction(): + random.seed(1); st = TokenState("M"); t = 0.0; lp = 0.0; compacted = False + for i in range(50000): + lp += random.gauss(0, 0.01); st.append(t, math.exp(lp), 0.1, i % 2 == 0, None, 100.0); t += 0.5 + compacted |= st.n_since_compact == 0 + if i % 4999 == 0 and i: + f, _ = st.features(t, TokenMeta("M")) + for k, w in (("1h", 3600.0), ("3h", 10800.0)): + j = next(n for n, x in enumerate(st.ts) if x >= t - w) + assert f[FIDX[f"dd_{k}"]] == pytest.approx(st.logp[-1] - max(st.logp[j:]), abs=1e-12) + assert compacted + + +def test_exit_cost_uses_fixed_size(monkeypatch): + monkeypatch.setattr(config, "MAX_POSITION_SOL", 7.0) + st = TokenState("M"); st.append(0.0, 1.0, 1.0, True, None, 30.0) + f, _ = st.features(60.0, TokenMeta("M", graduated_at=-3600.0, program_label="Pump.fun Amm")) + assert f[FIDX["exit_cost_0p1"]] == pytest.approx(exit_cost_fraction(0.1, 30.0, 1.0 * 1e9, "Pump.fun Amm")) # market cap = price × supply + + +# ---------------------------------------------------------------- train/mature.py +def _candles(path, day: date, mints): + t0 = datetime(day.year, day.month, day.day, 12, tzinfo=timezone.utc); rows = [] + for m in mints: + for k in range(30): + rows.append({"mint": m, "ts": t0 + timedelta(minutes=k), "open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0 + k / 100, "buy_sol": 1.0, "sell_sol": 0.5, + "n_buys": 2, "n_sells": 1, "n_traders": 2, "resq_sol": 50.0}) + pq.write_table(pa.Table.from_pylist(rows), path) + + +def test_mature_part_records_known_graduations_and_refreshes(tmp_path, monkeypatch): + monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature"); monkeypatch.setattr(mature, "MATURE_FEAT_DIR", tmp_path / "feat") + (tmp_path / "mature").mkdir(); d = date(2026, 9, 2) + _candles(tmp_path / "mature" / "2026-09-01.parquet", date(2026, 9, 1), ["A", "B"]); _candles(tmp_path / "mature" / "2026-09-02.parquet", d, ["A", "B"]) + grads = {"A": datetime(2026, 8, 30, tzinfo=timezone.utc)} + assert mature.build_day(d, grads=grads) > 0 + part = tmp_path / "feat" / "2026-09-02" / "part.parquet" + assert mature.part_version(part) == FEATURE_VERSION and mature.part_known(part) == 1 + t = pq.read_table(part, columns=["mint", "age_h"]).to_pandas() + assert t.loc[t["mint"] == "A", "age_h"].notna().all() and t.loc[t["mint"] == "B", "age_h"].isna().all() + more = {**grads, "B": datetime(2026, 8, 31, tzinfo=timezone.utc)} + assert not mature._knows_more(part, grads) + assert not mature._knows_more(part, more) # one more dated mint is below the rebuild bar (no rebuild storm) + monkeypatch.setattr(mature, "KNOWN_REBUILD_MIN", 1) + assert mature._knows_more(part, more) + # a replaced part stays readable until its successor lands; the old one is kept aside + monkeypatch.setattr(mature, "STALE_DIR", tmp_path / "stale") + mature._archive(part, "feat_2026-09-02") + assert part.exists() and len(list((tmp_path / "stale").glob("feat_2026-09-02_v*.parquet"))) == 1 + # a part from before the metadata key: the known count comes from its age_h column + legacy = tmp_path / "legacy.parquet"; tab = pq.read_table(part) + pq.write_table(tab.replace_schema_metadata({b"fly_version": str(FEATURE_VERSION).encode()}), legacy) + assert mature.part_known(legacy) == 1 + + +# ---------------------------------------------------------------- train/replay_assemble.py +def test_replay_band_is_causal(): + con = duckdb.connect() + con.execute("""CREATE TEMP TABLE t0 AS SELECT 'M' AS mint, TIMESTAMP '2026-09-01' + to_seconds(i) AS ts, i AS slot, 'pump-amm' AS pool, + CASE WHEN i < 10 THEN 1.0 ELSE exp(ln(10000.0) * (i - 9) / 291.0) END AS price FROM range(300) r(i)""") + con.execute("INSERT INTO t0 VALUES ('M', TIMESTAMP '2026-09-01' + to_seconds(150), 150, 'pump-amm', 1e9), " + "('M', TIMESTAMP '2026-08-31 23:50', 0, 'pump', 1e-9)") + band_amm(con, "t0", "t") + assert con.execute("SELECT count(*) FROM t WHERE price = 1.0").fetchone()[0] == 10 # a later 50x+ run no longer deletes graduation-time legs + assert con.execute("SELECT count(*) FROM t WHERE price >= 1e9").fetchone()[0] == 0 # the off-scale leg is dropped + assert con.execute("SELECT count(*) FROM t WHERE pool = 'pump'").fetchone()[0] == 1 # curve legs are not banded + assert con.execute("SELECT count(*) FROM t").fetchone()[0] == 301 + + +# ---------------------------------------------------------------- train/corpus_backtest.py +def _bt_frame(minutes, closes, t0, resq=1e12, broken=None): + ts = [t0 + timedelta(minutes=m) for m in minutes] + df = pd.DataFrame({"mint": "A", "ts": pd.to_datetime(ts, utc=True), "close": closes, "resq": resq, "has_trades": False}) + df["grad_day"] = df["ts"].dt.date.astype(str); df["broken"] = False if broken is None else broken + return df + + +def test_time_stop_between_traded_minutes_and_entry_day_fold(): + t0 = datetime(2026, 9, 1, 23, 58, tzinfo=timezone.utc) + df = _bt_frame([0, 1, 2, 200], [1.0, 1.1, 1.2, 5.0], t0) + cand = np.array([True, False, False, False]) + tr = cb.simulate(df, cand, 0.0, 1.0, 1.0, 30) + assert tr.shape == (1, 2) + assert tr[0, 0] == date(2026, 9, 1).toordinal() # entry day, although the exit is on the 2nd + assert tr[0, 1] == pytest.approx(0.2, abs=1e-6) # last close before the 30-minute stop, not minute 200's 5.0 + df = _bt_frame([0, 10, 30, 31], [1.0, 1.1, 1.3, 2.0], t0) + assert cb.simulate(df, cand, 0.0, 1.0, 1.0, 30)[0, 1] == pytest.approx(0.3, abs=1e-6) # a traded minute right at the stop exits there + + +def test_random_baseline_respects_gates(monkeypatch): + t0 = datetime(2026, 9, 1, 0, 0, tzinfo=timezone.utc); frames = [] + for j in range(12): + f = _bt_frame(list(range(200)), list(1.0 + 0.001 * np.arange(200)), t0 + timedelta(days=j), resq=np.where(np.arange(200) % 2, 50.0, 5.0), + broken=np.arange(200) >= 150) + f["mint"] = f"M{j}"; frames.append(f) + df = pd.concat(frames, ignore_index=True) + gate = (df["resq"].to_numpy() >= cb.MIN_RESQ_SOL) & ~df["broken"].to_numpy(); seen = [] + real = cb.simulate + monkeypatch.setattr(cb, "simulate", lambda d, cand, *a, **k: (seen.append(cand.copy()), real(d, cand, *a, **k))[1]) + monkeypatch.setattr(cb, "STRATEGIES", {"S9 all": lambda d: [("all", np.ones(len(d), bool))]}) + cb.run(df, 0.003, min_trades=1) + assert len(seen) == len(cb.EXITS) + 5 + assert all(not (c & ~gate).any() for c in seen) # strategy and random entries only where the gate allows + assert seen[-1].any() + + +# ---------------------------------------------------------------- train/corpus_features.py +class _Tx: + def __init__(self, fail=False, pending=()): + self.fail, self.pending, self.rows = fail, list(pending), None + + def __call__(self): + return self + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def cursor(self): + return self + + def execute(self, *a): + return self + + def fetchall(self): + return self.pending + + def executemany(self, sql, rows): + if self.fail: + raise RuntimeError("db down") + self.rows = rows + + +def _fake_rows(mint): + g = datetime(2026, 9, 1, tzinfo=timezone.utc).timestamp() + return [cf._row(mint, g + 60 * k, g, 1.0, 1.0, 1.0, 1.0, 1.0, 50.0, False, [0.0] * len(FEATURES), 0, {}) for k in range(3)], 0 + + +def test_corpus_features_failures_and_atomic_commit(tmp_path, monkeypatch): + g = datetime(2026, 9, 1, tzinfo=timezone.utc) + todo = [{"mint": m, "graduated_at": g, "candle_path": None, "trade_path": None} for m in ("GOOD", "BAD")] + monkeypatch.setattr(config, "CORPUS_FEATURES_DIR", tmp_path); monkeypatch.setattr(cf, "_failed", set()) + monkeypatch.setattr(cf, "build_token", lambda m, *a: (_ for _ in ()).throw(ValueError("corrupt")) if m == "BAD" else _fake_rows(m)) + monkeypatch.setattr(cf, "_pending", lambda limit: todo) + monkeypatch.setattr(cf, "transaction", _Tx(fail=True)) + with pytest.raises(RuntimeError): + cf.build_batch() + assert not [p for p in tmp_path.rglob("*") if p.is_file()] # failed insert: no part left behind, so no duplicate later + tx = _Tx(); monkeypatch.setattr(cf, "transaction", tx) + assert cf.build_batch() == 2 + assert [r[0] for r in tx.rows] == ["GOOD"] and "BAD" in cf._failed # the failure is not registered + files = [p for p in tmp_path.rglob("*") if p.is_file()] + assert len(files) == 1 and files[0].name.startswith("part-") and files[0].suffix == ".parquet" + assert pq.read_table(files[0]).num_rows == 3 and tx.rows[0][4] == str(files[0]) + monkeypatch.undo(); monkeypatch.setattr(cf, "_failed", {"BAD"}) + monkeypatch.setattr(cf, "transaction", _Tx(pending=[{"mint": "BAD"}, {"mint": "X"}, {"mint": "Y"}])) + assert [r["mint"] for r in cf._pending(1)] == ["X"] # skipped in this process + + +# ---------------------------------------------------------------- train/selector.py +def _ds(n_days=5, per_day=400, seed=0): + rng = np.random.default_rng(seed); n = n_days * per_day + X = np.c_[rng.normal(size=(n, 3)), np.zeros((n, 2))].astype(np.float32); y = (X[:, 0] + 0.3 * rng.normal(size=n) > 0).astype(np.int8) + day = np.array([date(2026, 9, 1) + timedelta(days=k) for k in range(n_days) for _ in range(per_day)], dtype=object) + ts = np.array([datetime(d.year, d.month, d.day, tzinfo=timezone.utc).timestamp() + 60 * (i % per_day) for i, d in enumerate(day)]) + fwd = np.where(y == 1, 0.05, -0.02).astype(np.float32) + return DecisionSet(X=X, y=y, fwd=fwd, fwd_pess=fwd, day=day, ts=ts, mint=np.array([f"M{i % 7}" for i in range(n)]), + cols=["a", "b", "c", "age_known", "log_age_h"], horizon_s=1800.0) # age unknown: graduated before the archive, tradable + + +def test_selector_fold_guards(): + ds = _ds(); days = ds.days + fo = selector.fold(ds, days[-1], min_train=100, min_test=10) + pick = fo.test & (fo.scores >= selector.MIN_EV) + assert fo is not None and fo.ic is not None and fo.ic > 0.5 and pick.any() and np.isnan(fo.scores[~fo.test]).all() + assert selector.fold(ds, days[0], min_train=100, min_test=10) is None # empty train + assert selector.fold(ds, days[-1]) is None # below the default sizes + from fly_trader.train import selector_eval # importable: runs only as a script + assert selector_eval.fold is selector.fold + + +def test_selector_trades_only_aged_tokens(): + ds = _ds(); days = ds.days; young = (ds.mint == "M0") + ds.X[young, 3] = 1.0; ds.X[young, 4] = np.log1p(2.0) # M0: known graduation, 2 h old + fo = selector.fold(ds, days[-1], min_train=100, min_test=10); pick = fo.test & (fo.scores >= selector.MIN_EV) + assert pick.any() and not (pick & young).any() and not (fo.test & young).any() # never a test row or a pick + assert list(selector.in_universe(np.array([[0, 0, 0, 1, np.log1p(30.0)], [0, 0, 0, 1, np.log1p(5.0)], [0, 0, 0, 0, 0]]), ds.cols)) == [True, False, True] + assert fo.model.threshold == selector.MIN_EV and fo.model.scaler is not None + + +def test_buy_line_is_chosen_by_total_profit(): + ds = _ds(); rows = np.ones(len(ds.y), bool) + scores = ds.fwd_pess.astype(np.float64) - 0.01 # good rows +4 %, bad rows -3 % + line, table = selector.choose_line(ds, scores, rows) + assert line in selector.LINE_CANDIDATES and [t["line"] for t in table] == list(selector.LINE_CANDIDATES) + best = max((t for t in table if t["trades"] >= selector.MIN_LINE_TRADES), key=lambda t: t["total"]) + assert line == best["line"] and best["mean"] > 0 diff --git a/tests/test_flow_parity.py b/tests/test_flow_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..02edd5bdc3ee5c1bb0b0d10b8bdd9a69af73ed0f --- /dev/null +++ b/tests/test_flow_parity.py @@ -0,0 +1,142 @@ +"""Wallet flow parity: the same trades through the archive (mature.aggregate_day, DuckDB) and the live stream +(pumpstream.Aggregator → pump_minutes) give identical per-minute wallet columns — buyers, wash round trips, the top +seller, insider selling and skill-bucket buying — and FlowWindow derives the model inputs from them.""" +from datetime import date, datetime, timezone + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from fly_trader import config +from fly_trader.db.connection import transaction +from fly_trader.ingest import pumpstream +from fly_trader.train import flow, mature + +WSOL = pumpstream.WSOL +MINT = "FlowTestpump" +T0 = datetime(2026, 9, 3, 12, 0, tzinfo=timezone.utc) +LEGS = [ # (seconds after T0, trader, side, sol) + (5, "W1", 1, 1.0), (7, "W1", -1, 0.4), (9, "W2", 1, 2.0), (12, "I1", -1, 3.0), (20, "W3", -1, 0.5), (25, "W4", 1, 0.7), + (65, "W2", -1, 1.0), (70, "W1", 1, 0.3)] + + +@pytest.fixture() +def world(tmp_path, monkeypatch): + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay"); monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature") + monkeypatch.setattr(mature, "SKILL_DIR", tmp_path / "skill") + (tmp_path / "skill").mkdir() + pq.write_table(pa.table({"wallet": ["W1", "W2"], "bucket": pa.array([9, 3], pa.int8())}), tmp_path / "skill" / "2026-09-03.parquet") + n = len(LEGS) + trades = pa.table({"ts": pa.array([datetime.fromtimestamp(T0.timestamp() + s, timezone.utc) for s, *_ in LEGS], pa.timestamp("ms", tz="UTC")), + "slot": list(range(n)), "pool": ["pump-amm"] * n, "mint": [MINT] * n, "trader": [t for _, t, _, _ in LEGS], + "side": pa.array([sd for *_, sd, _ in LEGS], pa.int8()), "sol": [v for *_, v in LEGS], "tokens": [10.0] * n, "price": [0.001] * n, + "quote_in_pool": [50.0] * n, "quote_mint": [WSOL] * n, "pool_id": ["P1"] * n}) + (tmp_path / "replay" / "2026-09-03").mkdir(parents=True) + pq.write_table(trades, tmp_path / "replay" / "2026-09-03" / "12_trades.parquet") + with transaction() as conn: + conn.execute("INSERT INTO token_insiders (mint, wallet, kind) VALUES (%s, 'I1', 'bundle') ON CONFLICT DO NOTHING", (MINT,)) + conn.execute("DELETE FROM pump_minutes WHERE mint = %s", (MINT,)) + yield tmp_path + with transaction() as conn: + conn.execute("DELETE FROM token_insiders WHERE mint = %s", (MINT,)); conn.execute("DELETE FROM pump_minutes WHERE mint = %s", (MINT,)) + conn.execute("DELETE FROM mature_days WHERE day = %s", (date(2026, 9, 3),)) + + +def _live(tmp_path): + agg = pumpstream.Aggregator() + for s, trader, side, sol in LEGS: + agg.ingest({"action": "buy" if side == 1 else "sell", "pool": "pump-amm", "mint": MINT, "timestamp": int((T0.timestamp() + s) * 1000), "quoteMint": WSOL, + "poolId": "P1", "price": 0.001, "quoteInPool": 50.0, "txSigner": trader, "quoteAmount": sol, "tokenAmount": 10.0}) + pumpstream._load_insiders(agg); pumpstream.load_skill(agg, T0.timestamp()) + agg.flush(int(T0.timestamp()) + 3600) + with transaction() as conn: + return {r["ts"]: r for r in conn.execute("SELECT * FROM pump_minutes WHERE mint = %s ORDER BY ts", (MINT,)).fetchall()} + + +def test_archive_and_live_wallet_columns_match(world): + assert mature.aggregate_day(date(2026, 9, 3)) > 0 + arch = {r["ts"]: r for r in pq.read_table(world / "mature" / "2026-09-03.parquet").to_pylist() if r["mint"] == MINT} + live = _live(world) + assert set(arch) == set(live) and len(arch) == 2 + for ts in arch: + a, l = arch[ts], live[ts] + for c in ("n_buyers", "wash_sol", "wash_buy_sol", "top_sell_sol", "insider_sell_sol", "n_traders", "buy_sol", "sell_sol"): + assert float(a[c]) == pytest.approx(float(l[c])), (ts, c) + assert [pytest.approx(v) for v in a["skill_buy"]] == [float(v) for v in l["skill_buy"]], ts + m0 = arch[T0] + assert m0["n_buyers"] == 3 and m0["wash_sol"] == pytest.approx(1.4) and m0["wash_buy_sol"] == pytest.approx(1.0) + assert m0["top_sell_sol"] == pytest.approx(3.0) and m0["insider_sell_sol"] == pytest.approx(3.0) + assert m0["skill_buy"][9] == pytest.approx(1.0) and m0["skill_buy"][3] == pytest.approx(2.0) and m0["skill_buy"][10] == pytest.approx(0.7) + + +def test_flow_window_derivations(): + fw = flow.FlowWindow(); t = 1_800_000_000.0 + fw.append(t, 3.7, 3.9, 3, 1.4, 1.0, 3.0, 3.0, [0, 0, 0, 2.0, 0, 0, 0, 0, 0, 1.0, 0.7]) + f = fw.features(t) + assert f["top_sell_share_1m"] == pytest.approx(3.0 / 3.9) and f["insider_sell_share_1m"] == pytest.approx(3.0 / 3.9) + assert f["wash_share_15m"] == pytest.approx(1.4 / 7.6) and f["org_imb_5m"] == pytest.approx(((3.7 - 1.0) - (3.9 - 0.4)) / ((3.7 - 1.0) + (3.9 - 0.4))) + assert f["skill_top9_15m"] == pytest.approx(1.0 / 3.7) and f["skill_top3_15m"] == pytest.approx(3.0 / 3.7) and f["skill_known_15m"] == pytest.approx(3.0 / 3.7) + fw.append(t + 60, 1.0, 0.0, 1, 0, 0, 0, 0, None) # a day without a skill table counts as unknown buyers + assert fw.features(t + 60)["buyers_ret"] == pytest.approx(0.6931471805599453 - __import__("math").log1p(3 / 5)) + mw = flow.MarketWindow(); mw.add(t, 5.0); mw.add(t + 3600, 1.0) + assert mw.value(t + 3600) == pytest.approx(__import__("math").log1p(1.0)) # the first minute left the hour + + +def test_skill_window_matches_flow_window(): + rng = np.random.default_rng(0); rows = [] + for m in ("A", "B", "C"): + t = 1_800_000_000 + for _ in range(80): + t += 60 * int(rng.integers(1, 6)) # traded minutes with gaps + rows.append((m, t, None if rng.random() < 0.2 else list(rng.exponential(1.0, flow.N_SKILL) * (rng.random(flow.N_SKILL) < 0.5)))) + rng.shuffle(rows) + got = flow.skill_features(np.array([r[0] for r in rows]), np.array([r[1] for r in rows], float), + np.array([r[2] if r[2] is not None else [0.0] * flow.N_SKILL for r in rows])) + for m in ("A", "B", "C"): + fw = flow.FlowWindow() + for i in sorted((i for i, r in enumerate(rows) if r[0] == m), key=lambda i: rows[i][1]): + fw.append(rows[i][1] + 60.0, 1.0, 0.0, 1, 0, 0, 0, 0, rows[i][2]) + f = fw.features(rows[i][1] + 60.0) + assert got[i] == pytest.approx([f[c] for c in flow.SKILL_COLS], abs=1e-6) + + +def test_candidate_skill_inputs_equal_the_archive_build(world, monkeypatch): + from fly_trader.train import decisions, progress, wallet_skill + monkeypatch.setattr(progress, "update", lambda *a, **k: None) + d = date(2026, 9, 3); table = mature.skill_table_for(d) + assert mature.aggregate_day(d) > 0 + agg = pq.read_table(world / "mature" / f"{d}.parquet") + same = mature.aggregate_table(mature._hour_files(d), table) + assert same.to_pylist() == agg.to_pylist() # the extracted SQL is aggregate_day's + monkeypatch.setattr(wallet_skill, "skill_table", lambda dd, h, L, day_dir=None: table if dd == d else None) + a = agg.to_pandas(); ts = mature._epoch_s(a["ts"]); n = len(a) + ds = decisions.DecisionSet(X=np.zeros((n, len(decisions.X_COLS)), np.float32), y=np.zeros(n, np.int8), fwd=np.zeros(n, np.float32), + fwd_pess=np.zeros(n, np.float32), day=np.array([d] * n, dtype=object), ts=ts, mint=a["mint"].to_numpy(), + cols=list(decisions.X_COLS), horizon_s=7200.0) + got = wallet_skill.candidate_skill_cols(ds, 30, 1) + fws = {} + for i, r in a.iterrows(): + fw = fws.setdefault(r["mint"], flow.FlowWindow()) + fw.append(ts[i] + 60.0, r["buy_sol"], r["sell_sol"], r["n_buyers"], r["wash_sol"], r["wash_buy_sol"], r["top_sell_sol"], r["insider_sell_sol"], + None if r["skill_buy"] is None else list(r["skill_buy"])) + f = fw.features(ts[i] + 60.0) + assert got[i] == pytest.approx([f[c] for c in flow.SKILL_COLS], abs=1e-6) + assert got[:, flow.SKILL_COLS.index("skill_known_15m")].max() > 0 # the candidate table was used + + +def test_passed_in_lookups_match_the_database(world): + """The skill fit reads the blocked pools and insiders once and hands them to every day instead of reconnecting for + each; the rows must be identical to the ones the database path builds.""" + from fly_trader.db.connection import transaction + from fly_trader.train.corpus_meta import blocked_pool_ids + d = date(2026, 9, 3); files = mature._hour_files(d) + from_db = mature.aggregate_table(files, mature.skill_table_for(d)) + with transaction() as conn: + blocked = blocked_pool_ids(conn) + ins: dict = {} + for r in conn.execute("SELECT mint, wallet FROM token_insiders").fetchall(): + ins.setdefault(r["mint"], []).append(r["wallet"]) + passed_in = mature.aggregate_table(files, mature.skill_table_for(d), blocked=blocked, insiders=ins) + assert from_db.to_pylist() == passed_in.to_pylist() + assert any(r["insider_sell_sol"] > 0 for r in passed_in.to_pylist()) # the insider join really fired diff --git a/tests/test_fly_calibration.py b/tests/test_fly_calibration.py new file mode 100644 index 0000000000000000000000000000000000000000..dc39fcb381907b6ba5eeced9716463ea93b76b3b --- /dev/null +++ b/tests/test_fly_calibration.py @@ -0,0 +1,39 @@ +"""The fly's daily calibration: the vectorised one-position-per-token rule equals the original loop, the line with the +most total profit over enough trades wins (quantile candidates included), and too few trades keep the previous line.""" +import numpy as np +import pytest + +from fly_trader.train import fly_calibrate +from fly_trader.train.decisions import taken_idx + + +def _loop(ts, mint, horizon_s, idx): + idx = np.asarray(idx, dtype=int) + idx = idx[np.lexsort((ts[idx], mint[idx]))]; out = []; last_mint = None; last_t = -1e18 + for i in idx: + if mint[i] != last_mint: + last_mint = mint[i]; last_t = -1e18 + if ts[i] >= last_t + horizon_s: + out.append(i); last_t = ts[i] + return np.asarray(out, dtype=int) + + +@pytest.mark.parametrize("seed", range(5)) +def test_taken_idx_equals_the_loop(seed): + rng = np.random.default_rng(seed); n = 3000 + ts = rng.integers(0, 20000, n).astype(float) * 60; mint = rng.choice([f"m{i}" for i in range(40)], n) + idx = rng.choice(n, 1200, replace=False) + assert np.array_equal(taken_idx(ts, mint, 1800.0, idx), _loop(ts, mint, 1800.0, idx)) + assert len(taken_idx(ts, mint, 1800.0, np.array([], dtype=int))) == 0 + + +def test_calibrate_picks_the_most_profitable_line_and_keeps_it_when_too_few_trades(): + rng = np.random.default_rng(0); n = 20000 + ts = np.arange(n, dtype=float) * 60; mint = np.array([f"m{i % 500}" for i in range(n)]) + scores = rng.normal(0, 0.001, n) # a compressed scale: fixed candidates alone never trade + rets = np.where(scores > np.quantile(scores, 0.98), 0.02, -0.01) + rng.normal(0, 0.001, n) + cal = fly_calibrate.calibrate(ts, mint, 1800.0, scores, rets) + assert cal.changed and cal.trades >= 100 and cal.mean > 0 and cal.line >= np.quantile(scores, 0.95) - 1e-12 + assert cal.line in fly_calibrate.candidates(scores) and cal.sizing + few = fly_calibrate.calibrate(ts[:50], mint[:50], 1800.0, scores[:50], rets[:50], prev_line=0.123, prev_sizing=[{"lo": 0.0, "kelly": 0.1}]) + assert few.line == 0.123 and not few.changed and few.sizing == [{"lo": 0.0, "kelly": 0.1}] diff --git a/tests/test_fly_channels.py b/tests/test_fly_channels.py new file mode 100644 index 0000000000000000000000000000000000000000..032d8b066ad8d11734c83c71b914b669065b557c --- /dev/null +++ b/tests/test_fly_channels.py @@ -0,0 +1,65 @@ +"""One dopamine channel per strategy: compartments come from the raw DAN→MBON mass; channels are balanced by KC→MBON +mass and keep both valences; a strategy's outcome changes only its own KC→MBON synapses; tags stay in due order; one +strategy behaves exactly as before.""" +import math + +import numpy as np +import pytest +import torch + +from fly_trader.brain import connectome as cn, plastic +from fly_trader.train.fly_selector import SCALE, FlyNet +from tests.test_fly_plastic import _graph, _parts + + +def test_compartments_from_raw_dan_mass(): + pops = {"KC": (0, 2), "MBON_APP": (2, 4), "MBON_AV": (4, 5), "MBON_OTHER": (5, 6), "DAN_PAM": (6, 8), "DAN_PPL1": (8, 9)} + post = torch.tensor([2, 2, 3, 4, 4]); pre = torch.tensor([6, 7, 7, 8, 6]); v = torch.tensor([5.0, 1.0, 3.0, 4.0, 1.0]) + comp, names = cn.mbon_compartments(torch.stack([post, pre]), v, pops, cell_type=np.array(["", "", "", "", "", "", "PAM01", "PAM02", "PPL101"])) + assert names == ["PAM01", "PAM02", "PPL101"] and list(comp) == [0, 1, 2, -1] # MBON 5 gets no DAN + + +def test_assign_channels_balance_valence_and_fallbacks(): + comp = np.array([0, 0, 1, 2, 3, 4, 5, -1]); mass = np.array([10, 10, 8, 6, 9, 5, 2, 7.0]); sign = np.array([1, 1, 1, 1, -1, -1, -1, 0]) + learn, read = plastic.assign_channels(comp, mass, sign, 2) + assert (learn.sum(0) <= 1).all() and not learn[:, 7].any() and read[:, 7].all() # unclaimed MBON: read by all, learned by none + for s in range(2): + assert learn[s, :4].any() and learn[s, 4:7].any() # both valences in every channel + l1, r1 = plastic.assign_channels(comp, mass, sign, 1) + assert l1.all() and r1.all() # one strategy: as before + lf, _ = plastic.assign_channels(np.full(8, -1), mass, sign, 3) + assert lf.sum() == 8 and (lf.sum(0) == 1).all() # no DAN data: each MBON its own compartment + + +@pytest.fixture() +def net(): + torch.manual_seed(0) + n = FlyNet(_graph(), obs_dim=5, device="cpu") + with torch.no_grad(): + n.mb_norm.running_mean.uniform_(-0.2, 0.2); n.mb_norm.running_var.uniform_(0.05, 0.3); n.c_free.uniform_(-0.3, 0.3) + return n.eval() + + +def test_a_strategys_outcome_changes_only_its_own_channel(net): + y_dn, u0, k = _parts(net) + sign = net.c_sign.numpy(); learn, read = plastic.assign_channels(np.arange(6), np.ones(6), np.where(sign == 0, 1, sign), 2) + bank = plastic.PlasticBank(net, [(0.05, math.inf)], SCALE, nu=1.0, step_cap=1e9, learn=learn, read=read) + r = torch.ones(len(y_dn)) * 0.05 + tags = plastic.Tags(list(range(len(r))), np.zeros(len(r)), y_dn, u0, k, torch.ones(1, len(r)), s=np.zeros(len(r), int)) + bank.update(tags, r, 0.0) + moved = (bank.D[0].abs().sum(0) > 0).numpy() + assert moved[learn[0]].any() and not moved[learn[1]].any() # channel 1 untouched by strategy 0's outcome + per = plastic.PlasticBank(net, [(0.0, math.inf), (0.05, math.inf)], SCALE, nu=1.0, step_cap=1e9, learn=learn, read=read) + per.D[1] = bank.D[0] + one = per.select_per_strategy([1, 0]) + assert torch.allclose(one.D[0][:, learn[0]], bank.D[0][:, learn[0]]) and (one.D[0][:, learn[1]] == 0).all() + + +def test_pending_tags_stay_in_due_order(net): + y_dn, u0, k = _parts(net, B=3) + q = plastic.PendingTags(net.n_kc, net.k_active) + q.push(["long"], np.zeros(1), 14400.0, y_dn[:1], u0[:1], k[:1], torch.ones(1, 1), s=2) + q.push(["short"], np.zeros(1), 600.0, y_dn[1:2], u0[1:2], k[1:2], torch.ones(1, 1), s=1) + t = q.pop_due(1000.0) + assert t.keys == ["short"] and list(t.s) == [1] and len(q) == 1 + assert q.pop_due(20000.0).keys == ["long"] diff --git a/tests/test_fly_distill.py b/tests/test_fly_distill.py new file mode 100644 index 0000000000000000000000000000000000000000..69435cd915f438ee3c0f54116cd8f81304a4d830 --- /dev/null +++ b/tests/test_fly_distill.py @@ -0,0 +1,198 @@ +"""The fly: FlyNet on a tiny stand-in connectome (with a mushroom body) learns to imitate a teacher, is built without +saturating or bottlenecked parts, starts at the mean target, keeps a sparse KC code, reaches its output through the MBON +readout, and is bootstrapped with its own buy line; the agreement / rank-correlation helpers are correct.""" +from datetime import date, datetime, timedelta, timezone +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +import torch +from torch import nn + +from fly_trader.train import fly_selector +from fly_trader.train.decisions import DecisionSet, _ranks, rank_corr +from fly_trader.train.scaling import RobustScaler + +POPS = {"KC": (0, 20), "MBON_APP": (20, 22), "MBON_AV": (22, 24), "MBON_OTHER": (24, 26), "ORN_FOOD": (26, 56), "DESCENDING": (70, 80)} + + +def _graph(seed=0): + """20 KCs, 6 MBONs, 30 sensory neurons (ORN_FOOD) → KCs and 14 interneurons → 10 descending neurons; MBONs → DNs; + KC→MBON synapses only in the dense block (as the build stores them).""" + g = torch.Generator().manual_seed(seed); pre, post = [], [] + for a, b in ((range(26, 56), range(0, 20)), (range(26, 56), range(56, 70)), (range(56, 70), range(70, 80)), (range(20, 26), range(70, 80)), + (range(26, 56), range(70, 80))): + for j in b: + for i in torch.randint(a.start, a.stop, (6,), generator=g).tolist(): + pre.append(i); post.append(j) + vals = (torch.rand(len(pre), generator=g) * 5 + 1) * torch.where(torch.rand(len(pre), generator=g) < 0.8, 1.0, -1.0) + W = torch.randint(1, 12, (20, 6), generator=g).float() * (torch.rand(20, 6, generator=g) < 0.4) + return SimpleNamespace(N=80, indices=torch.tensor([post, pre]), values_raw=vals, pop_ranges=dict(POPS), W_KM0=W, M_KM=W > 0, + device=torch.device("cpu"), s=0.05, spectral_radius_raw=None) + + +def _ds(n=6000, f=6, seed=0): + rng = np.random.default_rng(seed) + X = (rng.standard_t(3, size=(n, f)) * np.array([1, 2, 0.5, 300, 1, 1]) + 5).astype(np.float32) # fat tails, one huge-scale feature + days = np.array([date(2026, 9, 1 + i % 4) for i in range(n)], dtype=object) + return DecisionSet(X=X, y=np.zeros(n, np.int8), fwd=np.zeros(n, np.float32), fwd_pess=np.zeros(n, np.float32), day=days, ts=np.arange(n, dtype=np.float64) * 60, + mint=np.array([f"m{i % 50}" for i in range(n)]), cols=[f"f{i}" for i in range(f)], horizon_s=1800.0) + + +class Teacher: + def __init__(self, ds): + self.scaler = RobustScaler.fit(ds.X); self.threshold = 0.01 + + def score(self, X): + z = self.scaler.transform(X) + return 0.02 * z[:, 0] - 0.015 * z[:, 2] + 0.01 * z[:, 3] + + +@pytest.fixture(autouse=True) +def quiet(monkeypatch): + monkeypatch.setattr(fly_selector.prog, "update", lambda *a, **k: None) + + +def test_flynet_has_no_saturating_or_bottleneck_parts_and_starts_at_the_mean(): + ds = _ds(); t = Teacher(ds) + net = fly_selector.FlyNet(_graph(), obs_dim=len(ds.cols), device="cpu") + assert tuple(net.w_in.shape) == (30, len(ds.cols)) # every feature reaches every sensory neuron + assert isinstance(net.eff_norm, nn.BatchNorm1d) # homeostasis on the descending neurons + assert not any(isinstance(m, nn.Tanh) for m in net.dec.modules()) # the decoder cannot saturate + assert net.theta_km.numel() == int(_graph().M_KM.sum()) # only the synapses the connectome has + fly, info = fly_selector.train_fly(ds, np.ones(len(ds.y), bool), t, epochs=1, batch=500, graph=_graph(), device="cpu") + assert info["rows"] == len(ds.y) and info["epochs"][0]["rows"] == len(ds.y) # the full budget below the sample cap + start, _ = fly_selector.train_fly(ds, np.ones(len(ds.y), bool), t, epochs=0, graph=_graph(), device="cpu") + mean_target = float(np.clip(t.score(ds.X), -1, 1).mean()) + assert float(start.net.head.bias.detach()) == pytest.approx(mean_target * fly_selector.SCALE, rel=1e-4) # starts at the mean target + + +def test_kc_code_is_sparse_and_the_mbon_readout_is_the_plastic_path(): + torch.manual_seed(0) + net = fly_selector.FlyNet(_graph(), obs_dim=6, device="cpu").eval() + x = torch.randn(64, 6) + with torch.no_grad(): + y_dn, u0, k = net.forward_parts(x) + assert ((k > 0).sum(1) <= net.k_active).all() and (k >= 0).all() # APL-like winner-take-all, non-negative rates + assert torch.allclose(net.readout(y_dn, u0, k), net(x), atol=1e-5) # frozen fly = D None + assert torch.allclose(net.readout(y_dn, u0, k, torch.zeros(20, 6)), net(x), atol=1e-5) + D = torch.randn(20, 6) + moved = net.readout(y_dn, u0, k, D) - net(x) + assert moved.abs().max() > 0 # a KC→MBON change reaches the output + assert torch.allclose(net.readout(y_dn, u0, k, D * ~net.mask), net(x), atol=1e-5) # but only on existing synapses + assert (net.c[:2] > 0).all() and (net.c[2:4] < 0).all() # approach +, avoid − + + +def test_train_fly_imitates_the_teacher_on_the_same_signals(): + torch.manual_seed(0) + ds = _ds(); train = np.ones(len(ds.y), bool); t = Teacher(ds) + fly, info = fly_selector.train_fly(ds, train, t, epochs=6, batch=200, graph=_graph(), device="cpu") + assert not info["stopped"] and info["epochs"][-1]["mse"] < info["epochs"][0]["mse"] + s, g = fly.score(ds.X), t.score(ds.X) + assert rank_corr(s, g) > 0.9 # the fat-tailed feature does not blind it + b = np.polyfit(g, s, 1)[0] + assert 0.6 < b < 1.4 # on the teacher's scale, not squashed + assert fly.scaler is t.scaler # same signals + top_t, top_f = g >= np.quantile(g, 0.95), s >= np.quantile(s, 0.95) + assert (top_t & top_f).sum() / top_t.sum() > 0.6 # it picks what the teacher picks + d = fly_selector.diagnose(fly, ds.X[:2000], g[:2000]) + assert 0.0 <= d["mbon_share"] <= 1.0 and 0.0 <= d["kc_active"] <= fly_selector.KC_ACTIVE + 1e-6 and d["slope"] == pytest.approx(b, rel=0.2) + + +def test_normalisation_is_recalibrated_over_many_rows_not_the_last_batches(): + torch.manual_seed(0) + ds = _ds(); t = Teacher(ds) + fly, _ = fly_selector.train_fly(ds, np.ones(len(ds.y), bool), t, epochs=1, batch=500, graph=_graph(), device="cpu") + assert fly.net.eff_norm.momentum is None and int(fly.net.eff_norm.num_batches_tracked) == -(-len(ds.y) // fly_selector.BATCH) + + +def _market_ds(days=12, per_day=1000, seed=0): + """Rows with the universe columns and a label that follows f0: something a fly can calibrate a line on.""" + rng = np.random.default_rng(seed); n = days * per_day + X = rng.normal(size=(n, 6)).astype(np.float32); X[:, 4] = 0.0; X[:, 5] = np.log1p(24.0) # age_known 0: in the universe + fwd = (0.03 * X[:, 0] + rng.normal(scale=0.02, size=n)).astype(np.float32) + d0 = datetime(2026, 9, 1, tzinfo=timezone.utc); k = np.arange(n) + ts = np.array([(d0 + timedelta(days=int(i // per_day), minutes=int(i % per_day))).timestamp() for i in k]) + day = np.array([(d0 + timedelta(days=int(i // per_day))).date() for i in k], dtype=object) + return DecisionSet(X=X, y=(fwd > 0.03).astype(np.int8), fwd=fwd, fwd_pess=fwd, day=day, ts=ts, mint=np.array([f"m{i % 400}" for i in k]), + cols=["f0", "f1", "f2", "f3", "age_known", "log_age_h"], horizon_s=1800.0) + + +def test_bootstrap_gives_the_fly_its_own_line_from_labels_known_before_S(): + torch.manual_seed(0) + ds = _market_ds(); S = ds.days[-1] + timedelta(days=1) + fly, info = fly_selector.bootstrap(ds, S, graph=_graph(), device="cpu") + assert fly.threshold == info["line"] and fly.sizing == info["sizing"] + assert info["calibration"]["trades"] >= 100 or info["line"] == fly_selector.selector.MIN_EV + assert info["train_through"] == str(S - timedelta(days=9)) and info["calibration_days"] == [str(S - timedelta(days=7)), str(S - timedelta(days=1))] + assert set(info["diagnostics"]) >= {"mbon_share", "mbon_saturated", "kc_active", "slope"} and isinstance(info["gates_ok"], bool) + + +def test_the_fly_is_taught_by_the_selector_the_book_trades(monkeypatch): + """Until 2026-09-21 the bootstrap refit its own stack and distilled that. Selection is unstable enough that the + refit was a different trading system (deployed ev at 0.046 over 120 min vs a refit capitulation at 0.010 over 240), + so the fly was never a copy of the selector it was judged against.""" + import contextlib + + from fly_trader.agent import selector_session + ds = _market_ds(); train = ds.day < ds.days[-3] + ev = {"cols": ["f0", "f1"], "line": 0.046, "hold_min": 120, "thr": {}, "high": None} + monkeypatch.setattr(selector_session, "pinned_snapshot", lambda: 60) + monkeypatch.setattr(fly_selector.selector, "load_snapshot", lambda i: SimpleNamespace(stack={"strategies": {"ev": ev}, "combine": "score"})) + t, note, sid = fly_selector.deployed_teacher(ds, train) + assert sid == 60 and "#60" in note and t.strategies == ["ev"] and t.rules["ev"]["hold_min"] == 120 and t.lines["ev"] == 0.046 + + monkeypatch.setattr(fly_selector, "transaction", lambda: contextlib.nullcontext(None)) + monkeypatch.setattr(selector_session, "pinned_snapshot", lambda: None) + monkeypatch.setattr(fly_selector.selector, "latest_current", lambda conn: None) + assert fly_selector.deployed_teacher(ds, train) == (None, fly_selector.TEACHER_NOTE, None) # says so rather than pretending + + monkeypatch.setattr(selector_session, "pinned_snapshot", lambda: 61) + monkeypatch.setattr(fly_selector.selector, "load_snapshot", lambda i: SimpleNamespace(stack={"strategies": {"ev": {**ev, "cols": ["f0", "nope"]}}})) + t2, note2, sid2 = fly_selector.deployed_teacher(ds, train) + assert t2 is None and sid2 is None and "nope" in note2 # never distil inputs the corpus lacks + + +def test_training_stops_when_an_epoch_buys_less_than_its_own_noise(monkeypatch): + """One epoch left a per-row error of ~3.9 % net return against a 4.6 % line, so the ordering near the line was + mostly noise. The count is now the data's to choose: passes end when the held-out error stops moving by more than + its standard error, and the epoch that generalised best is the one kept.""" + monkeypatch.setattr(fly_selector, "VAL_MIN_ROWS", 100); monkeypatch.setattr(fly_selector, "VAL_ROWS", 500) + torch.manual_seed(0) + ds = _ds(); t = Teacher(ds) + fly, info = fly_selector.train_fly(ds, np.ones(len(ds.y), bool), t, epochs=12, batch=500, graph=_graph(), device="cpu") + assert info["val_rows"] == 500 and info["epoch_cap"] == 12 and fly_selector.MIN_EPOCHS <= info["epochs_run"] <= 12 + assert all("val_mse" in e and "val_se" in e for e in info["epochs"]) + assert info["val_mse"] == min(e["val_mse"] for e in info["epochs"]) # the best epoch, not the last + assert (info["early_stop"] is None) == (info["epochs_run"] == 12) + small, si = fly_selector.train_fly(ds, np.ones(len(ds.y), bool), t, epochs=0, graph=_graph(), device="cpu") + assert si["val_rows"] == 0 and si["early_stop"] is None and si["epochs_run"] == 0 # nothing to hold out, nothing to stop + + +def test_rank_corr_matches_spearman_with_ties(): + rng = np.random.default_rng(1) + a = rng.integers(0, 20, 500).astype(float); b = a + rng.normal(scale=5, size=500); b[::7] = b[0] + assert rank_corr(a, b) == pytest.approx(pd.Series(a).corr(pd.Series(b), method="spearman"), abs=1e-12) + x = rng.random(100) + assert rank_corr(x, np.exp(3 * x)) == pytest.approx(1.0) and rank_corr(x, -x) == pytest.approx(-1.0) + assert rank_corr(x, np.ones(100)) is None and rank_corr(np.array([1.0]), np.array([2.0])) is None + assert list(_ranks(np.array([3.0, 1.0, 3.0, 2.0]))) == [3.5, 1.0, 3.5, 2.0] + + +def test_robust_scaler_tames_fat_tails_and_keeps_order(): + X = np.c_[np.r_[np.ones(999), 1e6], np.arange(1000.0)] + sc = RobustScaler.fit(X); Z = sc.transform(X) + assert np.abs(Z).max() < 25 # a 1e6 outlier no longer swamps the units + assert (np.diff(Z[:, 1]) > 0).all() # monotone: tree splits are unchanged + assert np.allclose(RobustScaler.from_state(sc.state()).transform(X), Z) + + +def test_the_scaler_never_touches_the_callers_array(): + """transform() copies; only transform_into() writes, and the walk-forwards hand it arrays they own.""" + X = np.random.default_rng(0).normal(size=(64, 5)).astype(np.float32); before = X.copy() + sc = RobustScaler.fit(X) + Z = sc.transform(X) + assert np.array_equal(X, before) and not np.shares_memory(Z, X) + Y = X.copy() + assert sc.transform_into(Y) is Y and np.allclose(Y, Z) diff --git a/tests/test_fly_live.py b/tests/test_fly_live.py new file mode 100644 index 0000000000000000000000000000000000000000..5ca3d47876ef5bd1413c80250cb5f894390213c0 --- /dev/null +++ b/tests/test_fly_live.py @@ -0,0 +1,151 @@ +"""The fly on the bot wallet: entries mirrored from the paper book and sized from the wallet, exits retried then written off +as dead-bags, live wealth marked, and entries paused when live trails its paper mirror.""" +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest + +from fly_trader import config +from fly_trader.agent import fly_live, sizing +from fly_trader.chain.balances import Snapshot +from fly_trader.db.connection import transaction + + +class _Worker: + def __init__(self): + self.submitted, self.inflight = [], set() + + def submit(self, req): + self.submitted.append(req); self.inflight.add(req.mint); return True + + def pending(self): + return set(self.inflight) + + def drain(self): + return [] + + def stop(self): + pass + + +class _Rpc: + def __init__(self, lamports, tokens=None): + self.lamports, self.tokens = lamports, tokens or {} + + def get_balance(self, pk): + return self.lamports + + def get_token_accounts_by_owner(self, pk): + return [{"mint": m, "amount": a, "decimals": 6} for m, a in self.tokens.items()] + + +def _ctx(conn, m1): + return SimpleNamespace(conn=conn, m1=m1, m1_epoch=m1.timestamp(), prices={}, resqs={}, fees={}, last_resq=lambda m: 1000.0, mcap=lambda m, p: None) + + +@pytest.fixture() +def clean(): + def wipe(): + with transaction() as conn: + conn.execute("DELETE FROM positions WHERE mint LIKE 'LIVE_%%'"); conn.execute("DELETE FROM decisions WHERE mint LIKE 'LIVE_%%'") + conn.execute("DELETE FROM wealth_marks WHERE book = 'live'") + conn.execute("UPDATE circuit_state SET kill_switch = false, tripped = false, entries_paused = false, fail_count = 0 WHERE id = 1") + wipe(); yield; wipe() + + +def test_entries_are_sized_from_the_wallet_and_exits_retry_then_dead_bag(db_conn, clean, monkeypatch): + monkeypatch.setattr(config, "GAS_RESERVE_SOL", 0.3) + w = _Worker(); mirror = fly_live.LiveMirror(7200.0, broker=SimpleNamespace(rpc=_Rpc(5 * config.LAMPORTS_PER_SOL), pubkey="PK"), worker=w) + m1 = datetime(2026, 9, 20, 12, 0, tzinfo=timezone.utc); table = [{"lo": 0.0, "n": 100, "mean": 0.02, "win": 0.5, "kelly": 0.4}] + with transaction() as conn: + d = conn.execute("INSERT INTO decisions (kind, mint) VALUES ('fly_enter', 'LIVE_A') RETURNING id").fetchone()["id"] + for mint, age in (("LIVE_X", 7200 + 60), ("LIVE_D", 7200 + fly_live.DEAD_BAG_S + 60), ("LIVE_Y", 600)): + conn.execute("INSERT INTO positions (book, mint, pool, opened_at, qty, cost_sol, entry_price, status) VALUES ('live', %s, 'P', %s, 1000000, 0.2, 0.0000002, 'open')", + (mint, m1 - timedelta(seconds=age))) + beat = conn.execute("INSERT INTO beats (beat_no) VALUES (1) RETURNING id").fetchone()["id"] + out = mirror.minute(_ctx(conn, m1), run_id=str(uuid.uuid4()), beat_id=beat, line=0.01, table=table, + entries=[{"mint": "LIVE_A", "decision_id": d, "score": 0.03, "info": {"resq": 1000.0, "pool": "PA", "decimals": 6}}]) + dead = conn.execute("SELECT status, forced_exit_kind, realized_sol FROM positions WHERE book = 'live' AND mint = 'LIVE_D'").fetchone() + mark = conn.execute("SELECT wealth FROM wealth_marks WHERE book = 'live' AND beat_id = %s", (beat,)).fetchone() + targets = conn.execute("SELECT book_targets FROM decisions WHERE id = %s", (d,)).fetchone()["book_targets"] + sells = [r for r in w.submitted if r.side == "sell"]; buys = [r for r in w.submitted if r.side == "buy"] + assert [r.mint for r in sells] == ["LIVE_X"] and sells[0].slippage_bps == config.SLIPPAGE_EXIT_BPS and sells[0].amount_in == 1000000 + assert dead["status"] == "closed" and dead["forced_exit_kind"] == "dead_bag" and dead["realized_sol"] == pytest.approx(-0.2) + bankroll = 5.0 + 0.2 + 0.2 # wallet SOL + open live positions at cost (the dead-bag is gone) + expect, _ = sizing.size_position(0.03, 0.01, table, bankroll, 5.0, 1000.0) + assert len(buys) == 1 and buys[0].mint == "LIVE_A" and buys[0].amount_in == int(expect * config.LAMPORTS_PER_SOL) and buys[0].decision_id == d + assert "live" in targets and out["entered"] == 1 and out["dead_bags"] == 1 and mark is not None + + +def test_blocked_entries_and_the_gap_pause(db_conn, clean): + w = _Worker(); mirror = fly_live.LiveMirror(7200.0, broker=SimpleNamespace(rpc=_Rpc(5 * config.LAMPORTS_PER_SOL), pubkey="PK"), worker=w) + m1 = datetime(2026, 9, 21, 12, 0, tzinfo=timezone.utc) + with transaction() as conn: + for i in range(fly_live.GAP_TRADES): + d = conn.execute("INSERT INTO decisions (kind, mint) VALUES ('fly_enter', %s) RETURNING id", (f"LIVE_G{i}",)).fetchone()["id"] + for book, r in (("live", -0.04), ("paper_fly", 0.0)): + conn.execute("INSERT INTO positions (book, mint, entry_decision_id, qty, cost_sol, realized_sol, status, closed_at) VALUES (%s,%s,%s,0,0.1,%s,'closed',%s)", + (book, f"LIVE_G{i}", d, 0.1 * r, m1 - timedelta(minutes=i))) + beat = conn.execute("INSERT INTO beats (beat_no) VALUES (1) RETURNING id").fetchone()["id"] + out = mirror.minute(_ctx(conn, m1), run_id=str(uuid.uuid4()), beat_id=beat, line=0.01, table=[{"lo": 0.0, "kelly": 0.4, "n": 1, "mean": 0.1, "win": 1}], entries=[]) + paused = conn.execute("SELECT entries_paused FROM circuit_state WHERE id = 1").fetchone()["entries_paused"] + assert out["gap"]["paused"] and paused + d = conn.execute("INSERT INTO decisions (kind, mint) VALUES ('fly_enter', 'LIVE_B') RETURNING id").fetchone()["id"] + out2 = mirror.minute(_ctx(conn, m1 + timedelta(minutes=1)), run_id=str(uuid.uuid4()), beat_id=beat + 1000000, line=0.01, + table=[{"lo": 0.0, "kelly": 0.4, "n": 1, "mean": 0.1, "win": 1}], + entries=[{"mint": "LIVE_B", "decision_id": d, "score": 0.05, "info": {"resq": 1000.0, "pool": "PB", "decimals": 6}}]) + assert out2["entered"] == 0 and out2["blocked"] == "paused" and not [r for r in w.submitted if r.side == "buy"] + + +def test_a_tripped_kill_switch_liquidates_when_asked_and_only_then(db_conn, clean, monkeypatch): + """The kill switch blocked entries and kept holding. With KILL_SWITCH_LIQUIDATE every open live position is sold at the + forced slippage the minute it trips, retried while it stays on; a position already in flight is left to the worker.""" + w = _Worker(); mirror = fly_live.LiveMirror(7200.0, broker=SimpleNamespace(rpc=_Rpc(5 * config.LAMPORTS_PER_SOL), pubkey="PK"), worker=w) + m1 = datetime(2026, 9, 22, 12, 0, tzinfo=timezone.utc) + with transaction() as conn: + for mint, age in (("LIVE_K1", 600), ("LIVE_K2", 7200 + 60)): # one young, one due for its scheduled exit + conn.execute("INSERT INTO positions (book, mint, pool, opened_at, qty, cost_sol, entry_price, status) VALUES ('live', %s, 'P', %s, 1000000, 0.2, 0.0000002, 'open')", + (mint, m1 - timedelta(seconds=age))) + beat = conn.execute("INSERT INTO beats (beat_no) VALUES (1) RETURNING id").fetchone()["id"] + conn.execute("INSERT INTO wealth_marks (beat_id, book, ts, sol_free, positions_value, exit_cost, wealth, peak, drawdown, exposure, n_open) " + "VALUES (%s, 'live', %s, 100, 0, 0, 100, 100, 0, 0, 0)", (beat + 500000, m1 - timedelta(minutes=5))) # a peak far above today's 5 SOL + monkeypatch.setattr(config, "KILL_SWITCH_LIQUIDATE", False) + out = mirror.minute(_ctx(conn, m1), run_id=str(uuid.uuid4()), beat_id=beat, line=0.01, table=[], entries=[]) + killed = conn.execute("SELECT kill_switch FROM circuit_state WHERE id = 1").fetchone()["kill_switch"] + assert killed and out["liquidated"] == 0 and [r.mint for r in w.submitted] == ["LIVE_K2"] # default: halt only, the scheduled exit alone + w2 = _Worker(); mirror.worker = w2 + with transaction() as conn: + monkeypatch.setattr(config, "KILL_SWITCH_LIQUIDATE", True) + out = mirror.minute(_ctx(conn, m1 + timedelta(minutes=1)), run_id=str(uuid.uuid4()), beat_id=beat + 1, line=0.01, table=[], entries=[]) + reasons = {r["mint"]: r["reason"] for r in conn.execute("SELECT mint, reason FROM decisions WHERE mint LIKE 'LIVE_K%%' AND kind = 'fly_exit'").fetchall()} + sells = {r.mint: r for r in w2.submitted} + assert out["liquidated"] == 1 and set(sells) == {"LIVE_K1", "LIVE_K2"} + assert sells["LIVE_K1"].slippage_bps == config.SLIPPAGE_FORCED_BPS and reasons["LIVE_K1"] == "kill switch: liquidating" + assert sells["LIVE_K2"].slippage_bps == config.SLIPPAGE_EXIT_BPS # the scheduled exit went first; not re-submitted + + +def test_the_handover_gate_reads_its_thresholds_from_config(db_conn, monkeypatch): + """HANDOVER_DAYS=0 counts the whole race (a zero-length window would count nothing), HANDOVER_MIN_TRADES is the trade + count, and HANDOVER_BEAT_SELECTOR=0 drops the race against the paper selector.""" + from fly_trader.agent import fly_session, handover + now = datetime(2026, 9, 22, 12, 0, tzinfo=timezone.utc); race = now - timedelta(days=2) + stub = SimpleNamespace(race_started_at=race.timestamp(), boot_id=62) + with transaction() as conn: + conn.execute("DELETE FROM ui_settings WHERE key = %s", (handover.KEY,)); conn.execute("DELETE FROM positions WHERE mint LIKE 'HAND_%%'") + for i in range(20): + conn.execute("INSERT INTO positions (book, mint, qty, cost_sol, realized_sol, status, closed_at) VALUES (%s, %s, 0, 0.1, -0.01, 'closed', %s)", + (fly_session.BOOK, f"HAND_{i}", race + timedelta(hours=i))) # 20 losing trades: no selector to beat + try: + monkeypatch.setattr(config, "HANDOVER_DAYS", 14); monkeypatch.setattr(config, "HANDOVER_MIN_TRADES", 20); monkeypatch.setattr(config, "HANDOVER_BEAT_SELECTOR", True) + fly_session.FlyBook._handover_check(stub, conn, now.timestamp()) + assert handover.state(conn) is None # two days into a 14-day race + monkeypatch.setattr(config, "HANDOVER_DAYS", 0) + fly_session.FlyBook._handover_check(stub, conn, now.timestamp()) + assert handover.state(conn) is None # loses to a flat selector + monkeypatch.setattr(config, "HANDOVER_BEAT_SELECTOR", False) + fly_session.FlyBook._handover_check(stub, conn, now.timestamp()) + st = handover.state(conn) + assert st and st["fly_trades"] == 20 and st["days"] == 0 and st["beat_selector"] is False + finally: + conn.execute("DELETE FROM ui_settings WHERE key = %s", (handover.KEY,)); conn.execute("DELETE FROM positions WHERE mint LIKE 'HAND_%%'") diff --git a/tests/test_fly_plastic.py b/tests/test_fly_plastic.py new file mode 100644 index 0000000000000000000000000000000000000000..3a9267e06eebb6204976dadf5596a7a6eef027f7 --- /dev/null +++ b/tests/test_fly_plastic.py @@ -0,0 +1,120 @@ +"""Mushroom-body plasticity: the local three-factor rule is exactly gradient descent on the weighted squared prediction +error, the shadow is the bootstrap fly, changes stay on connectome synapses within their bounds and step cap, decay has +the exact half-life, and tags come back as they were pushed, only once due.""" +import math +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from fly_trader.brain import plastic +from fly_trader.train.fly_selector import SCALE, FlyNet + +POPS = {"KC": (0, 20), "MBON_APP": (20, 22), "MBON_AV": (22, 24), "MBON_OTHER": (24, 26), "ORN_FOOD": (26, 56), "DESCENDING": (70, 80)} + + +def _graph(seed=0): + g = torch.Generator().manual_seed(seed); pre, post = [], [] + for a, b in ((range(26, 56), range(0, 20)), (range(26, 56), range(56, 70)), (range(56, 70), range(70, 80)), (range(20, 26), range(70, 80))): + for j in b: + for i in torch.randint(a.start, a.stop, (6,), generator=g).tolist(): + pre.append(i); post.append(j) + vals = (torch.rand(len(pre), generator=g) * 5 + 1) * torch.where(torch.rand(len(pre), generator=g) < 0.8, 1.0, -1.0) + W = torch.randint(1, 12, (20, 6), generator=g).float() * (torch.rand(20, 6, generator=g) < 0.4) + return SimpleNamespace(N=80, indices=torch.tensor([post, pre]), values_raw=vals, pop_ranges=dict(POPS), W_KM0=W, M_KM=W > 0, + device=torch.device("cpu"), s=0.05, spectral_radius_raw=None) + + +@pytest.fixture() +def net(): + torch.manual_seed(0) + n = FlyNet(_graph(), obs_dim=5, device="cpu") + with torch.no_grad(): # non-trivial frozen normalisation and readout + n.mb_norm.running_mean.uniform_(-0.2, 0.2); n.mb_norm.running_var.uniform_(0.05, 0.3) + n.c_free.uniform_(-0.3, 0.3); n.gain.uniform_(0.8, 1.2) + return n.eval() + + +def _parts(net, B=32, seed=1): + torch.manual_seed(seed) + with torch.no_grad(): + return net.forward_parts(torch.randn(B, 5) * 2) + + +def test_local_rule_equals_autograd_gradient_descent(net): + y_dn, u0, k = _parts(net) + bank = plastic.PlasticBank(net, [(0.005, math.inf)], SCALE, nu=1.0, step_cap=1e9) + bank.D[0] = (torch.rand_like(bank.w0) - 0.5) * 0.5 * bank.w0 # start away from zero, well inside [−w0, 2·w0] + D0 = bank.D[0].clone() + with torch.no_grad(): + yhat = net.readout(y_dn, u0, k, D0) / SCALE + r = yhat + torch.randn_like(yhat) * 0.01 # errors well inside the clips + w = torch.where(torch.rand(len(r)) < 0.3, plastic.TOP_WEIGHT, 1.0) + Dp = D0.clone().requires_grad_(True) + loss = 0.5 * (w * (r - net.readout(y_dn, u0, k, Dp) / SCALE) ** 2).sum() + loss.backward() + expected = D0 - 0.005 * Dp.grad * bank.mask + tags = plastic.Tags(keys=list(range(len(r))), ts=np.zeros(len(r)), y_dn=y_dn, u0=u0, k=k, w=w[None]) + bank.update(tags, r, t=0.0) + assert torch.allclose(bank.D[0], expected, atol=1e-7, rtol=1e-4) + + +def test_shadow_is_the_bootstrap_fly_and_predictions_match_the_net(net): + y_dn, u0, k = _parts(net) + bank = plastic.PlasticBank(net, [(1e-3, 7.0), (0.0, math.inf)], SCALE) + with torch.no_grad(): + assert torch.allclose(bank.frozen(y_dn, u0), net.readout(y_dn, u0, k) / SCALE, atol=1e-6) + bank.D[0] = torch.rand_like(bank.w0) * bank.w0 + s, _ = bank.predict(y_dn, u0, k) + assert torch.allclose(s[0], net.readout(y_dn, u0, k, bank.D[0]) / SCALE, atol=1e-6) + assert torch.allclose(s[1], bank.frozen(y_dn, u0), atol=1e-6) + + +def test_changes_stay_on_synapses_within_bounds_and_step_cap(net): + y_dn, u0, k = _parts(net) + bank = plastic.PlasticBank(net, [(1e6, math.inf)], SCALE, nu=1.0) # absurd rate: the caps must hold + r = torch.ones(len(y_dn)); w = torch.ones(1, len(r)) + tags = plastic.Tags(list(range(len(r))), np.zeros(len(r)), y_dn, u0, k, w) + before = bank.D.clone(); st = bank.update(tags, r, 0.0) + assert st["capped"] == [True] + assert float(torch.linalg.norm(bank.D - before)) <= plastic.STEP_CAP * bank.w0_norm * (1 + 1e-5) + for _ in range(3000): + bank.update(tags, r, 0.0) + assert (bank.D[:, ~bank.mask.bool()] == 0).all() + assert (bank.D >= -bank.w0 - 1e-7).all() and (bank.D <= 2 * bank.w0 + 1e-7).all() + + +def test_decay_has_the_exact_half_life(net): + bank = plastic.PlasticBank(net, [(1e-3, 3.0), (1e-3, math.inf)], SCALE) + bank.D[:] = bank.w0 * 0.5; bank.decay_to(100.0); bank.decay_to(100.0 + 3 * plastic.DAY_S) + assert torch.allclose(bank.D[0], bank.w0 * 0.25) and torch.allclose(bank.D[1], bank.w0 * 0.5) + + +def test_bias_toward_depression_is_optional_and_state_round_trips(net): + y_dn, u0, k = _parts(net) + bank = plastic.PlasticBank(net, [(0.05, 7.0)], SCALE, nu=1.0, beta_pot=0.5, step_cap=1e9) + bank.update(plastic.Tags(list(range(len(y_dn))), np.zeros(len(y_dn)), y_dn, u0, k, torch.ones(1, len(y_dn))), torch.randn(len(y_dn)) * 0.05, 5.0) + other = plastic.PlasticBank(net, [(0.05, 7.0)], SCALE) + other.load_state(bank.state()) + assert torch.allclose(other.D, bank.D) and other.t_last == 5.0 and other.nu == 1.0 and other.beta_pot == 0.5 + with pytest.raises(ValueError): + plastic.PlasticBank(net, [(0.1, 7.0)], SCALE).load_state(bank.state()) + one = plastic.PlasticBank(net, [(0.0, 1.0), (0.05, 7.0)], SCALE); one.D[1] = bank.D[0] + assert torch.allclose(one.select(1).D[0], bank.D[0]) + + +def test_pending_tags_return_as_pushed_and_only_when_due(net): + y_dn, u0, k = _parts(net, B=10) + q = plastic.PendingTags(net.n_kc, net.k_active) + q.push([("a", i) for i in range(4)], np.full(4, 0.0), 100.0, y_dn[:4], u0[:4], k[:4], torch.ones(2, 4)) + q.push([("b", i) for i in range(6)], np.full(6, 60.0), 160.0, y_dn[4:], u0[4:], k[4:], torch.full((2, 6), 10.0)) + assert len(q) == 10 and q.pop_due(99.0) is None + t = q.pop_due(130.0) + assert t.keys == [("a", i) for i in range(4)] and len(q) == 6 + assert torch.allclose(t.k, k[:4]) and torch.allclose(t.u0, u0[:4]) and tuple(t.w.shape) == (2, 4) + t2 = q.pop_due(1e9) + assert len(t2) == 6 and len(q) == 0 and float(t2.w[0, 0]) == 10.0 + assert plastic.row_weights(torch.tensor([[0.1, 0.3]]), torch.tensor([0.2])).tolist() == [[0.0, 1.0]] # below the line teaches nothing + two = plastic.row_weights(torch.tensor([[0.1, 0.3], [0.1, 0.3]]), torch.tensor([0.2, 0.5])) + assert two.tolist() == [[0.0, 1.0], [0.0, 0.0]] # each configuration judges against its own line diff --git a/tests/test_fly_replay.py b/tests/test_fly_replay.py new file mode 100644 index 0000000000000000000000000000000000000000..c7f9186ee02d086bce2ee2863f3fb9311793733e --- /dev/null +++ b/tests/test_fly_replay.py @@ -0,0 +1,106 @@ +"""The fly replay: bootstrapped once, it learns minute by minute from labels it could have known at the time, its +chosen configuration does not depend on the evaluation half, and the verdict uses the selector's deploy rule. +Also the rollback checks it counts.""" +import math + +import numpy as np +import pytest +import torch + +from fly_trader.brain import plastic +from fly_trader.train import fly_governance as gov, fly_replay, fly_selector +from tests.test_fly_distill import _graph, _market_ds + + +@pytest.fixture(autouse=True) +def quiet(monkeypatch): + monkeypatch.setattr(fly_selector.prog, "update", lambda *a, **k: None) + monkeypatch.setattr(fly_replay.prog, "update", lambda *a, **k: None) + + +def _boot(ds, start_day): + torch.manual_seed(0) + return fly_selector.bootstrap(ds, ds.days[start_day], graph=_graph(), device="cpu") + + +def test_replay_learns_only_from_known_labels_and_judges_with_the_deploy_rule(monkeypatch): + ds = _market_ds(days=16, per_day=600); fly, boot = _boot(ds, 9) + seen = [] + real = plastic.PlasticBank.update + + def spy(self, tags, r, t): + assert (tags.ts + ds.horizon_s + fly_replay.LABEL_LAG_S <= t + 1e-6).all() # never a label from the future + seen.append(len(tags)); return real(self, tags, r, t) + + monkeypatch.setattr(plastic.PlasticBank, "update", spy) + out = fly_replay.run(ds=ds, fly=fly, boot=boot, start_day=9, configs=[(0.0, math.inf), (1e-3, 3.0)], save_verdict=False) + assert sum(seen) > 0 and out["configs"][0]["alpha"] == 0.0 and len(out["configs"]) == 2 + assert isinstance(out["passed"], bool) and "reason" in out and out["data"] == fly_selector.FLY_VERSION + assert out["learning"] and sum(d["n"] for d in out["learning"]) > 0 # the replay records what its learning did + assert all(len(d["drift"]) == len(out["configs"]) for d in out["learning"]) # per configuration, every day + if out.get("chosen"): + assert out["chosen"]["alpha"] > 0 # the frozen fly is a comparison, never the pick + + +def test_choice_does_not_depend_on_evaluation_half_labels(): + ds = _market_ds(days=16, per_day=600); fly, boot = _boot(ds, 9) + cfg = [(0.0, math.inf), (1e-3, 1.0), (3e-3, 30.0)] + a = fly_replay.run(ds=ds, fly=fly, boot=boot, start_day=9, configs=cfg, save_verdict=False) + days_r = ds.days[9:]; ev = np.isin(ds.day, days_r[len(days_r) // 2:]) + ds.fwd_pess[ev] = -ds.fwd_pess[ev] # the evaluation half's future changes + b = fly_replay.run(ds=ds, fly=fly, boot=boot, start_day=9, configs=cfg, save_verdict=False) + assert [t["total"] for t in a["configs"]] == pytest.approx([t["total"] for t in b["configs"]]) + assert (a.get("chosen") or {}).get("config") == (b.get("chosen") or {}).get("config") + + +def test_rollback_checks(): + rng = np.random.default_rng(0) + good, bad = rng.normal(0.01, 0.02, 200), rng.normal(-0.01, 0.02, 200) + assert gov.shadow_check(bad, good)[0] and not gov.shadow_check(good, bad)[0] + assert not gov.shadow_check(bad[:10], good)[0] # too few trades to judge + s = rng.normal(size=10000) + assert gov.ic_check(s, -s + rng.normal(size=10000))[0] and not gov.ic_check(s, s)[0] + assert gov.ic_check(s[:100], -s[:100]) == (False, None) + assert gov.drift_check(0.6) and not gov.drift_check(0.4) + out = gov.check((bad, good), (s, -s), 0.7) + assert out["triggers"] == ["shadow", "ic", "drift"] + + +def test_unresolved_rows_are_not_trades_and_a_nan_verdict_still_stores(): + """A 240-minute hold has no label at the end of the corpus (decisions.build leaves NaN). Counting those rows put + n=3195 mean=NaN in a finished replay's own judgement and then lost five hours of work to a JSON error.""" + import json + import numpy as np + from fly_trader.train import progress as prog + ds = _market_ds(days=14, per_day=400) + lab = ds.fwd_pess.copy(); lab[-200:] = np.nan # the tail has no resolved label + ds.fwd_h = {30: lab} + order = np.argsort(ds.ts, kind="stable"); ts_o = ds.ts[order] + scores = np.full((1, 1, len(order)), 1.0, np.float32) + pick, hold, ret = fly_replay._book(ds, order, ts_o, scores, {}, [0], [0.0], [30], np.ones(len(order), bool)) + assert pick.any() and np.isfinite(ret[pick]).all() # every taken row has a known outcome + assert not pick[np.flatnonzero(~np.isfinite(lab))].any() # and the unresolved tail was not taken + assert json.loads(json.dumps(prog._finite({"frozen": {"n": 3195, "mean": float("nan")}})))["frozen"]["mean"] is None + + +def test_every_configuration_shares_the_frozen_flys_line(monkeypatch): + """Per-configuration lines let the frozen arm take 4,216 selection-half trades to the chosen arm's 2,609 with the + weights no more than 5 % apart: the arms traded different slices because of their lines, not their weights. One line, + calibrated on the frozen fly, leaves the weights as the only difference between arms.""" + from types import SimpleNamespace + calls = [] + + def fake_cal(ts, mint, hold_s, sc, y, prev_line=None, prev_sizing=None): + calls.append(len(sc)); return SimpleNamespace(line=float(np.mean(sc)), sizing=[{"lo": 0.0, "kelly": 0.1}]) + monkeypatch.setattr(fly_replay.fly_calibrate, "calibrate", fake_cal) + n, C, NS = 200, 3, 2 + ts = np.arange(n, dtype=np.float64) * 60.0 + ds = SimpleNamespace(ts=ts, mint=np.array([f"m{i % 7}" for i in range(n)]), fwd_h={}, fwd_pess=np.zeros(n, np.float32)) + scores = np.random.default_rng(0).normal(size=(C, NS, n)) # every configuration scores differently + lines = np.full((C, NS), 0.02); sizings = [[[] for _ in range(NS)] for _ in range(C)]; line_log = {} + fly_replay._recalibrate(ds, np.arange(n), ts, scores, lines, sizings, float(ts[-1]) + 61.0, [60.0, 60.0], [10, 30], line_log) + assert calls == [n, n] # once per strategy, on every resolved row + for s in range(NS): + assert lines[0, s] == pytest.approx(np.mean(scores[0, s])) # the frozen configuration's scores + assert (lines[:, s] == lines[0, s]).all() and all(line_log[(c, s)][-1] == (float(ts[-1]) + 61.0, lines[0, s]) for c in range(C)) + assert all(sizings[c][s] == sizings[0][s] for c in range(C)) diff --git a/tests/test_fly_session.py b/tests/test_fly_session.py new file mode 100644 index 0000000000000000000000000000000000000000..91a6de54a4d63303da262d3f44d8b24c3e5725bb --- /dev/null +++ b/tests/test_fly_session.py @@ -0,0 +1,62 @@ +"""The live fly learns from exactly the label it was trained on: its resolver equals ``decisions.build``'s ``fwd_pess`` +row for row (next-open entry within two minutes else the close, last close at or before t + hold, exit cost at both +ends), drops an exit after a 50× jump or without the row's own bar; tags can be filtered.""" +from datetime import datetime, timedelta, timezone + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest +import torch + +from fly_trader.agent.fly_session import resolve_label +from fly_trader.brain import plastic +from fly_trader.market.features import FEATURES +from fly_trader.train import decisions, mature + + +def _part(tmp_path): + rng = np.random.default_rng(0) + base = datetime(2026, 9, 1, tzinfo=timezone.utc) + rows = [] + for mint in ("Apump", "Bpump"): + offs = sorted(rng.choice(400, 220, replace=False).tolist()) # traded minutes with irregular gaps (some > 2 minutes) + price = 1.0 + for o in offs: + price *= float(np.exp(rng.normal(0, 0.02))) + rows.append({**{f: 0.0 for f in FEATURES}, "mint": mint, "ts": base + timedelta(minutes=int(o)), "open": price * float(np.exp(rng.normal(0, 0.005))), + "close": price, "resq": 50.0, "age_h": 30.0, "traders_15m": 10.0, "traders_1h": 40.0, "n_trades_1m": 3.0, "logvol_15m": 3.0, + "exit_cost_0p1": float(rng.uniform(0.005, 0.02))}) + df = pd.DataFrame(rows) + root = tmp_path / "feat" + mature.write_part(pa.Table.from_pandas(df, preserve_index=False), root / "2026-09-01" / "part.parquet", mature.FEATURE_VERSION, {"fly_agg": mature.AGG_VERSION}) + return df, root + + +def test_resolver_equals_the_training_label(tmp_path): + df, root = _part(tmp_path) + ds = decisions.build(days=None, horizon_min=10, feature_dir=root) + assert len(ds.y) > 100 + from fly_trader.market.exit_cost import cost_at_size # the engine reprices before it hands a bar to the fly + bars = {m: [(t.timestamp(), o, c, cost_at_size(e, rq)) for t, o, c, e, rq in + zip(g["ts"], g["open"], g["close"], g["exit_cost_0p1"], g["resq"])] + for m, g in df.sort_values("ts").groupby("mint")} + got = np.array([resolve_label(bars[m], t, ds.horizon_s) for m, t in zip(ds.mint, ds.ts)], dtype=np.float64) + assert np.allclose(got, ds.fwd_pess, rtol=1e-5, atol=1e-6) + gaps = np.array([next((b[0] for b in bars[m] if b[0] > t), np.inf) - t for m, t in zip(ds.mint, ds.ts)]) + assert (gaps > 120).any() and (gaps <= 120).any() # both entry rules were exercised + + +def test_resolver_refuses_jumps_and_missing_rows(): + bars = [(0.0, 1.0, 1.0, 0.01), (60.0, 1.0, 1.0, 0.01), (120.0, 1.0, 60.0, 0.01)] + assert resolve_label(bars, 0.0, 120.0) is None # a 60× jump into the exit minute + assert resolve_label(bars, 30.0, 120.0) is None # the row's own minute is not in the bars + ok = [(0.0, 1.0, 1.0, 0.01), (300.0, 2.0, 1.1, 0.02)] + assert resolve_label(ok, 0.0, 600.0) == pytest.approx(1.1 / 1.0 * 0.99 * 0.98 - 1) # next trade 5 min later: entry at the close + + +def test_tags_subset_keeps_the_chosen_rows(): + k = torch.rand(3, 8); t = plastic.Tags(keys=["a", "b", "c"], ts=np.array([0.0, 1.0, 2.0]), y_dn=torch.tensor([1.0, 2.0, 3.0]), u0=torch.rand(3, 4), k=k, + w=torch.tensor([[1.0, 10.0, 1.0]])) + s = t.subset(np.array([True, False, True])) + assert s.keys == ["a", "c"] and list(s.ts) == [0.0, 2.0] and s.y_dn.tolist() == [1.0, 3.0] and torch.equal(s.k, k[[0, 2]]) and s.w.tolist() == [[1.0, 1.0]] diff --git a/tests/test_fly_strategies.py b/tests/test_fly_strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..024f363b8aa146c4ce722afe67616024e3eb2c0a --- /dev/null +++ b/tests/test_fly_strategies.py @@ -0,0 +1,111 @@ +"""The fly with several strategies: one head per strategy distilled from the stack's per-strategy targets on that +strategy's candidates; each strategy calibrates its own line; in the replay a strategy's tags are due only at its own +hold and teach only its own channel; the verdict carries one configuration per strategy; save/load keeps the rules.""" +import math +from datetime import timedelta + +import numpy as np +import pytest +import torch + +from fly_trader.brain import plastic +from fly_trader.train import fly_replay, fly_selector +from tests.test_fly_distill import _graph, _market_ds + +HOLDS = {"ev": 30, "capitulation": 10} + + +@pytest.fixture(autouse=True) +def quiet(monkeypatch): + monkeypatch.setattr(fly_selector.prog, "update", lambda *a, **k: None) + monkeypatch.setattr(fly_replay.prog, "update", lambda *a, **k: None) + + +def _ds(days=16, per_day=600): + ds = _market_ds(days=days, per_day=per_day) + rng = np.random.default_rng(3) + ret = (0.01 * rng.normal(size=len(ds.y))).astype(np.float32) + ds.X = np.column_stack([ds.X, ret, np.zeros(len(ds.y), np.float32)]).astype(np.float32); ds.cols = ds.cols + ["ret_1m", "insider_sell_share_1m"] + ds.fwd_h = {30: ds.fwd_pess, 10: (-2.0 * ret + rng.normal(scale=0.01, size=len(ret))).astype(np.float32)} + return ds + + +class _Teacher(fly_selector.StackTeacher): + """A stand-in for the fitted stack: ev follows f0, capitulation (ret_1m < 0) follows −ret_1m; rows below the line blocked.""" + + def __init__(self, ds): + self.cols = list(ds.cols); self.scaler = fly_selector.RobustScaler.fit(ds.X, seed=7); self.strategies = list(HOLDS) + self.rules = {k: {"thr": {}, "high": None, "hold_min": h} for k, h in HOLDS.items()} + self.lines = {"ev": 0.02, "capitulation": 0.01}; self.threshold = 0.02; self.combine = "score"; self.eps = {} + + def targets(self, X, ts): + c = self.cols; ev = 0.03 * X[:, c.index("f0")]; cap = -2.0 * X[:, c.index("ret_1m")] + T = np.column_stack([ev, np.where(X[:, c.index("ret_1m")] < 0, cap, np.nan)]) + A = np.isfinite(T) & (T >= np.array([self.lines["ev"], self.lines["capitulation"]])) + return T, A + + +def _boot(ds, day=9, epochs=1): + torch.manual_seed(0) + return fly_selector.bootstrap(ds, ds.days[day], epochs=epochs, graph=_graph(), device="cpu", teacher=_Teacher(ds)) + + +def test_one_head_per_strategy_with_its_own_line(tmp_path, db_conn, monkeypatch): + ds = _ds(); fly, info = _boot(ds, epochs=40) # 600 training rows: enough steps to learn + assert fly.strategies == ["ev", "capitulation"] and fly.net.n_strategies == 2 and len(fly.net.heads) == 2 + assert set(info["calibration"]["per_strategy"]) == {"ev", "capitulation"} and info["lines"] == fly.lines + trig = fly.triggers(ds.X[:500], ds.cols) + assert trig[:, 0].all() and (trig[:, 1] == (ds.X[:500, ds.cols.index("ret_1m")] < 0)).all() + V = fly.score_all(ds.X[:2000]) + c0 = np.corrcoef(V[:, 0], ds.X[:2000, 0])[0, 1]; c1 = np.corrcoef(V[:, 1], -ds.X[:2000, ds.cols.index("ret_1m")])[0, 1] + assert c0 > 0.3 and c1 > 0.3 # each head learned its own strategy's value + monkeypatch.setattr(fly_selector.config, "BRAIN_DIR", tmp_path) + path, sid = fly_selector.save(fly, {}) + with fly_selector.transaction() as conn: + conn.execute("DELETE FROM brain_snapshots WHERE id = %s", (sid,)) + back = fly_selector.load(path, graph=_graph(), device="cpu") + assert back.rules == fly.rules and back.lines == pytest.approx(fly.lines) and np.allclose(back.score_all(ds.X[:200]), V[:200], atol=1e-5) + d = fly_selector.fly_decide(fly, ds.X[:2000], ds.cols, V) + for i in np.flatnonzero(d["strategy"] != None): # noqa: E711 + j = fly.strategies.index(d["strategy"][i]) + assert trig_ok(fly, ds, i, j) and V[i, j] >= fly.lines[d["strategy"][i]] and d["hold_s"][i] == HOLDS[d["strategy"][i]] * 60.0 + + +def trig_ok(fly, ds, i, j): + return bool(fly.triggers(ds.X[i:i + 1], ds.cols)[0, j]) + + +def test_replay_tags_are_due_at_their_own_hold_and_teach_their_own_channel(monkeypatch): + ds = _ds(); fly, boot = _boot(ds) + learn = fly.net.learn.cpu().numpy(); seen = {0: 0, 1: 0} + real = plastic.PlasticBank.update + + def spy(self, tags, r, t): + holds = np.array([HOLDS[fly.strategies[s]] * 60.0 for s in tags.s]) + assert (tags.ts + holds + fly_replay.LABEL_LAG_S <= t + 1e-6).all() # never a label from the future, per hold + for (row, s), s2 in zip(tags.keys, tags.s): + assert s == s2; seen[int(s)] += 1 + if s == 1: + assert ds.X[row, ds.cols.index("ret_1m")] < 0 # capitulation tags only on its candidates + self.decay_to(t); before = self.D.clone(); out = real(self, tags, r, t) # forgetting (time) moves every channel; learning must not + moved = ((self.D - before).abs().sum((0, 1)) > 0).cpu().numpy() + present = set(int(x) for x in tags.s) + others = np.zeros(learn.shape[1], bool) + for s3 in set(range(learn.shape[0])) - present: + others |= learn[s3] + assert not (moved & others).any() # a strategy's outcome leaves the others' columns alone + return out + + monkeypatch.setattr(plastic.PlasticBank, "update", spy) + out = fly_replay.run(ds=ds, fly=fly, boot=boot, start_day=9, configs=[(0.0, math.inf), (1e-3, 3.0)], save_verdict=False) + assert seen[0] > 0 and seen[1] > 0 + assert set(out["per_strategy"]) == {"ev", "capitulation"} and out["strategies"] == ["ev", "capitulation"] + assert {v["hold_min"] for v in out["per_strategy"].values()} == {30, 10} and isinstance(out["passed"], bool) + for v in out["per_strategy"].values(): + assert v["chosen"] is None or v["chosen"]["alpha"] > 0 + + +def test_bootstrap_line_uses_labels_known_before_S(): + ds = _ds(); S = ds.days[9]; fly, info = _boot(ds) + assert info["calibration_days"] == [str(S - timedelta(days=7)), str(S - timedelta(days=1))] + assert all(v["hold_min"] == HOLDS[k] for k, v in info["calibration"]["per_strategy"].items()) diff --git a/tests/test_guards.py b/tests/test_guards.py new file mode 100644 index 0000000000000000000000000000000000000000..f2db1bccf95e34f6b269124e371381d4ad35c6dc --- /dev/null +++ b/tests/test_guards.py @@ -0,0 +1,28 @@ +"""Small guards found by the review: finite JSON for status/events, worker-thread log routing.""" +import logging +import math +import threading + +from fly_trader.db import apilog +from fly_trader.logging_setup import _ThreadFilter +from fly_trader.train import progress + + +def test_finite_json_guards(): + x = {"a": float("inf"), "b": [1.0, float("nan"), {"c": -math.inf}], "d": 2.5} + assert progress._finite(x) == {"a": None, "b": [1.0, None, {"c": None}], "d": 2.5} + assert apilog._finite(x) == progress._finite(x) + + +def test_thread_filter_routes_helper_threads(): + got = [] + def _in_worker(): + f = _ThreadFilter("replay") + for tn in ("replay", "replay-dl-0", "replay-assemble", "corpus", "MainThread"): + rec = logging.LogRecord("x", logging.INFO, "", 0, "m", (), None); rec.threadName = tn + got.append((tn, f.filter(rec))) + t = threading.Thread(target=_in_worker, name="replay"); t.start(); t.join() + assert dict(got) == {"replay": True, "replay-dl-0": True, "replay-assemble": True, "corpus": False, "MainThread": False} + f = _ThreadFilter("capture") # created on the main thread (CLI use): MainThread records pass + rec = logging.LogRecord("x", logging.INFO, "", 0, "m", (), None); rec.threadName = "MainThread" + assert f.filter(rec) is True diff --git a/tests/test_keys.py b/tests/test_keys.py new file mode 100644 index 0000000000000000000000000000000000000000..27948d89d6f06fd16fdf588fa5e50881257fe36a --- /dev/null +++ b/tests/test_keys.py @@ -0,0 +1,88 @@ +"""Key parsing accepts both on-disk forms; failures never leak secret material; wallet creation is +tested only against a temporary .env (the real bot wallet is never created here).""" +import json +import os +import stat + +import base58 +import pytest +from solders.keypair import Keypair + +from fly_trader.chain import keys + + +def test_base58_and_json_array_forms_agree(): + kp = Keypair() + b58 = base58.b58encode(bytes(kp)).decode() + arr = json.dumps(list(bytes(kp))) + assert keys.bot_pubkey(b58) == keys.bot_pubkey(arr) == str(kp.pubkey()) + assert keys.load_keypair(b58).pubkey() == kp.pubkey() + assert len(b58) in (87, 88) # what the logging scrubber redacts + + +def test_bad_key_errors_never_contain_material(): + marker = "SECRETMARKER" + with pytest.raises(keys.WalletKeyError) as ei: + keys.load_keypair(marker + "0OIl-not-base58") + assert marker not in str(ei.value) + + with pytest.raises(keys.WalletKeyError) as ei: + keys.load_keypair(json.dumps([7] * 63)) + assert "7, 7" not in str(ei.value) and "[7" not in str(ei.value) + assert "63" in str(ei.value) # the length is safe to report + + kp = Keypair() + b58 = base58.b58encode(bytes(kp)).decode() + truncated = b58[:-3] + with pytest.raises(keys.WalletKeyError) as ei: + keys.load_keypair(truncated) + assert b58[:16] not in str(ei.value) + + +def test_missing_key(monkeypatch): + monkeypatch.delenv("BOT_PRIVATE_KEY", raising=False) + with pytest.raises(keys.WalletKeyError, match="not set"): + keys.load_keypair() + assert keys.key_available() is False + + +def test_create_wallet_refuses_when_env_file_has_key(tmp_path, monkeypatch): + monkeypatch.delenv("BOT_PRIVATE_KEY", raising=False) + env = tmp_path / ".env" + env.write_text("FOO=1\nexport BOT_PRIVATE_KEY=placeholder\n") + with pytest.raises(keys.WalletExists): + keys.create_wallet(env_path=env) + assert env.read_text().count("BOT_PRIVATE_KEY") == 1 + + +def test_create_wallet_into_temporary_env(tmp_path, monkeypatch, db_conn): + monkeypatch.delenv("BOT_PRIVATE_KEY", raising=False) + env = tmp_path / ".env" + env.write_bytes(b"FOO=1") # no trailing newline, like the operator's file + pubkey = keys.create_wallet(env_path=env) + text = env.read_text() + assert text.startswith("FOO=1\nBOT_PRIVATE_KEY=") and text.endswith("\n") + assert stat.S_IMODE(env.stat().st_mode) == 0o600 + line = next(l for l in text.splitlines() if l.startswith("BOT_PRIVATE_KEY=")) + assert keys.bot_pubkey(line.split("=", 1)[1]) == pubkey + assert "BOT_PRIVATE_KEY" not in os.environ + with pytest.raises(keys.WalletExists): + keys.create_wallet(env_path=env) + n = db_conn.execute("SELECT count(*) AS n FROM wallet_events WHERE kind = 'created' AND pubkey = %s", (pubkey,)).fetchone()["n"] + assert n == 1 + + +def test_show_wallet_uses_rpc_and_records_snapshot(monkeypatch, db_conn): + kp = Keypair() + monkeypatch.setattr(keys, "load_keypair", lambda raw=None: kp) + + class Rpc: + def get_balance(self, pk): + assert pk == str(kp.pubkey()) + return 1_234_567_890 + + line = keys.show_wallet(rpc=Rpc()) + assert str(kp.pubkey()) in line and "1.234567890" in line + row = db_conn.execute("SELECT detail FROM wallet_events WHERE kind = 'balance_snapshot' AND pubkey = %s", + (str(kp.pubkey()),)).fetchone() + assert row["detail"]["lamports"] == 1_234_567_890 diff --git a/tests/test_minute_engine.py b/tests/test_minute_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..703c11f661691bc276718f115e78d4656e1de8b6 --- /dev/null +++ b/tests/test_minute_engine.py @@ -0,0 +1,115 @@ +"""The minute engine: readiness on the stream, minute aggregation, the full feature vector built once per mint whatever +the number of books, the training eligibility rules, and each book trading in its own savepoint.""" +import json +import math +import time +from datetime import datetime, timezone + +import numpy as np + +from fly_trader.agent import minute_engine as me +from fly_trader.db.connection import transaction +from fly_trader.train.decisions import X_COLS + + +def _engine(books=()): + return me.MinuteEngine(list(books)) + + +def test_ready_through_waits_for_the_stream(db_conn): + s = _engine() + m0 = math.floor(time.time() / 60) * 60 - 60 + with db_conn.cursor() as cur: + cur.execute("INSERT INTO ui_settings (key, value) VALUES ('pumpstream_status', %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + (json.dumps({"flushed_through": datetime.fromtimestamp(m0 - 60, timezone.utc).isoformat()}),)) + db_conn.commit() + assert s.ready_through(m0 + 60) == m0 # minute m0 is not written yet: it waits, however late + with db_conn.cursor() as cur: + cur.execute("UPDATE ui_settings SET value = %s WHERE key = 'pumpstream_status'", (json.dumps({"flushed_through": datetime.fromtimestamp(m0, timezone.utc).isoformat()}),)) + db_conn.commit() + assert s.ready_through(m0 + 60) == m0 + 60 + + +def test_scale_break_blocks_the_mint_like_training(db_conn): + s = _engine(); t = 1_800_000_000.0 + a = {"open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0, "buy": 1.0, "sell": 0.0, "nb": 1, "ns": 0, "n_traders": 1, "resq": 50.0, "pool": "P", "program_label": "Pump.fun Amm"} + assert s._features(db_conn, "Bkpump", a, t)[1]["broken"] is False + assert s._features(db_conn, "Bkpump", {**a, "close": 80.0}, t + 60)[1]["broken"] is True + assert s._features(db_conn, "Bkpump", a, t + 120)[1]["broken"] is True # from that minute on, as in decisions.build + + +def test_aggregate_and_full_feature_vector(db_conn): + s = _engine() + m0 = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) + with db_conn.cursor() as cur: + cur.execute("DELETE FROM pump_minutes WHERE mint = 'Zpump'") + cur.execute("INSERT INTO pump_minutes (mint, ts, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol) VALUES " + "('Zpump', %s, 'POOL', 1.0, 1.2, 0.9, 1.1, 6.0, 2.0, 3, 1, 4, 55.0)", (m0,)) + db_conn.commit() + agg = s._aggregate(db_conn, m0, m0.replace(minute=1)) + assert list(agg) == ["Zpump"] and agg["Zpump"]["resq"] == 55.0 and agg["Zpump"]["n_traders"] == 4 + x, info = s._features(db_conn, "Zpump", agg["Zpump"], m0.timestamp() + 60) + assert x.shape == (len(X_COLS),) and np.isfinite(x).all() + assert info["resq"] == 55.0 and info["price"] == 1.1 and info["open"] == 1.0 and info["ec"] > 0 + assert x[X_COLS.index("n_trades_1m")] == 4.0 and x[X_COLS.index("traders_15m")] == 4.0 + assert x[X_COLS.index("meta_known")] == 0.0 and x[X_COLS.index("age_known")] == 0.0 + + +def test_graduation_time_comes_from_corpus_meta(db_conn): + s = _engine() + g = datetime(2026, 9, 1, 10, 0, tzinfo=timezone.utc) + with db_conn.cursor() as cur: + cur.execute("DELETE FROM corpus_meta WHERE mint = 'Ypump'") + cur.execute("INSERT INTO corpus_meta (mint, graduated_at, ttg_min, dev_sol) VALUES ('Ypump', %s, 45.0, 1.5)", (g,)) + db_conn.commit() + a = {"open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0, "buy": 6.0, "sell": 0.0, "nb": 2, "ns": 0, "n_traders": 2, "resq": 60.0, "pool": "P", "program_label": "Pump.fun Amm"} + x, info = s._features(db_conn, "Ypump", a, g.timestamp() + 2 * 3600) + assert info["age_h"] == 2.0 + assert x[X_COLS.index("age_known")] == 1.0 and x[X_COLS.index("meta_known")] == 1.0 + assert x[X_COLS.index("ttg_min")] == 45.0 and x[X_COLS.index("dev_sol")] == 1.5 + with db_conn.cursor() as cur: + cur.execute("DELETE FROM corpus_meta WHERE mint = 'Ypump'") + db_conn.commit() + + +class _Book: + def __init__(self, name, fail=False): + self.name, self.fail, self.done, self.seen = name, fail, False, None + + def on_bars(self, t, bars): + self.bars = bars + + def on_minute(self, ctx): + ctx.conn.execute("INSERT INTO events (level, source, message) VALUES ('info', 'engine_test', %s)", (self.name,)) + self.seen = (ctx.X.shape, list(ctx.mints)) + if self.fail: + raise RuntimeError("boom") + return {"ok": True} + + def open_mints(self, conn): + return set() + + def finish(self): + pass + + +def test_books_share_one_feature_pass_and_trade_in_their_own_savepoints(db_conn, monkeypatch): + m0 = datetime(2026, 9, 2, 12, 0, tzinfo=timezone.utc) + with db_conn.cursor() as cur: + cur.execute("DELETE FROM pump_minutes WHERE mint = 'Wpump'"); cur.execute("DELETE FROM events WHERE source = 'engine_test'") + cur.execute("INSERT INTO pump_minutes (mint, ts, pool_id, open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol) VALUES " + "('Wpump', %s, 'POOL', 1.0, 1.2, 0.9, 1.1, 6.0, 2.0, 3, 1, 4, 55.0)", (m0,)) + db_conn.commit() + bad, good = _Book("bad", fail=True), _Book("good") + e = _engine([bad, good]); calls = [] + real = e._features + monkeypatch.setattr(e, "_features", lambda *a: calls.append(a[1]) or real(*a)) + monkeypatch.setattr(e, "stream_fresh", lambda m0_epoch: True) + out = e.run_minute(m0.timestamp() + 60) + assert calls == ["Wpump"] # one feature pass per mint, two books + assert good.seen == ((1, len(X_COLS)), ["Wpump"]) and good.bars["Wpump"][0] == m0.timestamp() + assert "good" in out and "bad" not in out + with transaction() as conn: + left = [r["message"] for r in conn.execute("SELECT message FROM events WHERE source = 'engine_test'").fetchall()] + conn.execute("DELETE FROM events WHERE source = 'engine_test'"); conn.execute("DELETE FROM pump_minutes WHERE mint = 'Wpump'") + assert left == ["good"] # the failing book's writes were rolled back alone diff --git a/tests/test_multi_hold.py b/tests/test_multi_hold.py new file mode 100644 index 0000000000000000000000000000000000000000..a397ccf890e099a73d8184044b4b9b03a738770e --- /dev/null +++ b/tests/test_multi_hold.py @@ -0,0 +1,100 @@ +"""Per-strategy holds: labels for several holds computed at once equal single-hold builds, and the one-position-per-token +rule with per-row holds equals the scalar rule when holds are equal and respects each position's own exit otherwise.""" +from datetime import date, datetime, timedelta, timezone + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +from fly_trader.market.features import FEATURES +from fly_trader.train import decisions, mature +from fly_trader.train.decisions import taken_idx +from fly_trader.train.flow import FLOW_COLS, MARKET_COLS, SKILL_COLS + + +def _part(tmp_path): + rng = np.random.default_rng(1); base = datetime(2026, 9, 1, tzinfo=timezone.utc); rows = [] + for mint in ("Apump", "Bpump"): + price = 1.0 + for o in sorted(rng.choice(600, 300, replace=False).tolist()): + price *= float(np.exp(rng.normal(0, 0.02))) + rows.append({**{f: 0.0 for f in FEATURES + FLOW_COLS + SKILL_COLS + MARKET_COLS}, "mint": mint, "ts": base + timedelta(minutes=int(o)), + "open": price * float(np.exp(rng.normal(0, 0.004))), "close": price, "resq": 50.0, "age_h": 30.0, "traders_15m": 10.0, "traders_1h": 40.0, + "n_trades_1m": 3.0, "logvol_15m": 3.0, "exit_cost_0p1": float(rng.uniform(0.005, 0.02))}) + root = tmp_path / "feat" + mature.write_part(pa.Table.from_pandas(pd.DataFrame(rows), preserve_index=False), root / "2026-09-01" / "part.parquet", mature.FEATURE_VERSION, {"fly_agg": mature.AGG_VERSION}) + return root + + +def test_multi_hold_labels_equal_single_hold_builds(tmp_path): + root = _part(tmp_path) + ds = decisions.build(days=None, horizon_min=30, feature_dir=root, holds=[10, 30, 60]) + assert np.allclose(ds.fwd_h[30], ds.fwd_pess, equal_nan=True) + for H in (10, 60): + one = decisions.build(days=None, horizon_min=H, feature_dir=root) + key = pd.Series(one.fwd_pess, index=pd.MultiIndex.from_arrays([one.mint, one.ts])) + got = pd.Series(ds.fwd_h[H], index=pd.MultiIndex.from_arrays([ds.mint, ds.ts])) + common = key.index.intersection(got.index) + assert len(common) > 100 and np.allclose(got[common].to_numpy(), key[common].to_numpy(), equal_nan=True) + + +def test_per_row_holds_in_taken_idx(): + rng = np.random.default_rng(0); n = 2000 + ts = rng.integers(0, 20000, n).astype(float) * 60; mint = rng.choice([f"m{i}" for i in range(30)], n); idx = rng.choice(n, 900, replace=False) + assert np.array_equal(taken_idx(ts, mint, np.full(n, 1800.0), idx), taken_idx(ts, mint, 1800.0, idx)) + hold = np.where(rng.random(n) < 0.5, 600.0, 14400.0) + got = taken_idx(ts, mint, hold, idx) + by = {} + for i in got: # no taken entry falls inside an earlier position's own hold + by.setdefault(mint[i], []).append(i) + for m, ii in by.items(): + ii = sorted(ii, key=lambda i: ts[i]) + for a, b in zip(ii, ii[1:]): + assert ts[b] >= ts[a] + hold[a] + + +def test_trades_and_random_with_per_row_holds(tmp_path): + root = _part(tmp_path) + ds = decisions.build(days=None, horizon_min=30, feature_dir=root, holds=[10, 30]) + pick = np.isfinite(ds.fwd_h[10]); hold = np.full(len(ds.y), 600.0) + r = decisions.trades_from_picks(ds, pick, returns=ds.fwd_h[10], hold_s=hold) + assert len(r) == len(taken_idx(ds.ts, ds.mint, hold, np.flatnonzero(pick))) and len(r) > len(decisions.trades_from_picks(ds, pick)) + ev = decisions.evaluate(ds, np.where(pick, 1.0, np.nan), pick, 0.5, hold_s=hold, returns=ds.fwd_h[10]) + assert ev["pooled"]["n"] == len(r) + + +def test_the_label_is_costed_at_the_size_the_book_trades(): + """exit_cost_0p1 is priced for 0.1 SOL; positions are Kelly-sized well above that and impact grows with size, so the + label must reprice both sides or every backtest return is optimistic.""" + import pandas as pd + from fly_trader import config + from fly_trader.market.exit_cost import exit_cost_fraction + from fly_trader.train.decisions import _label_exit_cost + resq = np.array([25.0, 150.0, 2000.0]) + ec01 = np.array([exit_cost_fraction(0.1, r, 5e5, None) for r in resq]) + got = _label_exit_cost(pd.DataFrame({"exit_cost_0p1": ec01, "resq": resq})) + size = config.LABEL_SIZE_SOL or config.MAX_POSITION_FRACTION * config.CAPITAL_SOL + want = [exit_cost_fraction(min(size, config.MAX_POOL_SHARE * r), r, 5e5, None) for r in resq] + assert got == pytest.approx(want, abs=2e-4) # matches the live cost model at the traded size + assert (got > ec01).all() # and is strictly dearer than the 0.1 SOL assumption + blind = _label_exit_cost(pd.DataFrame({"exit_cost_0p1": [0.01], "resq": [float("nan")]})) + assert blind[0] == 1.0 # unknown liquidity: assume the position cannot be exited + + +def test_subset_keeps_days_rows_and_every_hold(): + """The fly's teacher is built with DecisionSet.subset; if it dropped days or per-hold labels the bootstrap would fit + on a silently smaller corpus than intended.""" + from fly_trader.train.decisions import X_COLS, DecisionSet + n = 600 + days = np.array([date(2026, 7, 1) + timedelta(days=int(i // 100)) for i in range(n)], dtype=object) + ds = DecisionSet(X=np.zeros((n, len(X_COLS)), np.float32), y=np.zeros(n, np.int8), fwd=np.arange(n, dtype=np.float32), + fwd_pess=np.arange(n, dtype=np.float32), day=days, ts=np.arange(n, dtype=float) * 60.0, + mint=np.array([f"m{i}" for i in range(n)]), cols=list(X_COLS), horizon_s=1800.0, + fwd_h={h: np.arange(n, dtype=np.float32) + h for h in (10, 120)}) + m = ds.day < date(2026, 7, 4) + sub = ds.subset(m) + assert len(sub.y) == int(m.sum()) and sub.days == sorted(set(days[m].tolist())) + assert sorted(sub.fwd_h) == [10, 120] and all(len(v) == len(sub.y) for v in sub.fwd_h.values()) + assert sub.fwd_h[120][0] == pytest.approx(ds.fwd_h[120][np.flatnonzero(m)[0]]) # labels follow their rows + assert sub.cols == ds.cols and sub.horizon_s == ds.horizon_s diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..125e540b1bf207ae76d046f754bc44c974a67496 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,81 @@ +"""Training pipeline: schedule, waiting for data, the deploy decision and the build-completeness check.""" +from datetime import date, datetime, timedelta, timezone + +import pyarrow as pa +import pytest + +from fly_trader.train import mature, pipeline +from fly_trader.train.selector import DATA_VERSION, deploy_decision, is_deployable + + +def test_schedule_is_seven_days_or_on_request(): + now = datetime(2026, 9, 14, 12, tzinfo=timezone.utc) + assert pipeline.due({}, now) # never ran + assert not pipeline.due({"last_run_at": (now - timedelta(days=6)).isoformat()}, now) + assert pipeline.due({"last_run_at": (now - timedelta(days=7)).isoformat()}, now) + assert pipeline.due({"last_run_at": now.isoformat(), "run_requested": True}, now) # Retrain now + assert not pipeline.due({"run_requested": True, "retry_after": (now + timedelta(minutes=5)).isoformat()}, now) + assert not pipeline.due({"last_run_at": (now - timedelta(days=30)).isoformat()}, now, handed_over=True) # the fly holds the seat: no schedule + assert not pipeline.due({}, now, handed_over=True) + assert pipeline.due({"run_requested": True}, now, handed_over=True) # Retrain now still works + + +def test_run_waits_for_the_feature_build(monkeypatch): + monkeypatch.setattr(mature, "build_complete", lambda: (False, "3 day(s) of features still to build")) + with pytest.raises(pipeline.NotReady, match="features still to build"): + pipeline.run() + + +def _patched_run(monkeypatch, saved, current_fly): + calls = [] + monkeypatch.setattr(mature, "build_complete", lambda: (True, "ok")) + monkeypatch.setattr(pipeline, "state", lambda: dict(saved)) + monkeypatch.setattr(pipeline, "_save", lambda **kv: saved.update(kv) or dict(saved)) + monkeypatch.setattr(pipeline, "record_event", lambda *a, **k: None) + monkeypatch.setattr(pipeline, "_current_fly", lambda: current_fly) + from fly_trader.train import fly_selector, selector + monkeypatch.setattr(selector, "main", lambda stop_event=None: calls.append("selector") or {"snapshot_id": 7, "deployable": False, "line": 0.005}) + monkeypatch.setattr(fly_selector, "main", lambda stop_event=None: calls.append("fly") or {"snapshot_id": 8}) # never given the selector's line + return calls + + +def test_run_trains_selector_and_bootstraps_a_fly_only_when_none_is_current(monkeypatch): + saved = {}; calls = _patched_run(monkeypatch, saved, current_fly=None) + out = pipeline.run() + assert calls == ["selector", "fly"] and out["selector_snapshot"] == 7 and out["fly_snapshot"] == 8 and out["stage"] == "done" + assert datetime.fromisoformat(out["next_run_at"]) - datetime.fromisoformat(out["last_run_at"]) == timedelta(days=7) + saved = {"fly_snapshot": 5}; calls = _patched_run(monkeypatch, saved, current_fly={"id": 5}) + out = pipeline.run() + assert calls == ["selector"] and out["fly_snapshot"] == 5 # a weekly run never replaces the fly + saved = {"fly_snapshot": 5, "fly_requested": True}; calls = _patched_run(monkeypatch, saved, current_fly={"id": 5}) + out = pipeline.run() + assert calls == ["selector", "fly"] and out["fly_snapshot"] == 8 and not out["fly_requested"] # re-bootstrap on request + + +def test_only_a_profitable_selector_that_beats_random_is_deployable(): + assert deploy_decision({"n": 500, "mean": 0.01}, {"mean": -0.03})[0] + assert not deploy_decision({"n": 500, "mean": -0.005}, {"mean": -0.03})[0] # lost money + assert not deploy_decision({"n": 500, "mean": 0.01}, {"mean": 0.02})[0] # no better than random + assert not deploy_decision({"n": 20, "mean": 0.05}, {"mean": -0.03})[0] # too few trades + assert is_deployable({"data": DATA_VERSION, "deployable": True}) and not is_deployable({"data": {"agg": 1}, "deployable": True}) + legacy = {**{k: v for k, v in DATA_VERSION.items() if k != "cols"}, "fly": "flynet-1"} # saved before the split and the column hash + assert is_deployable({"data": legacy, "deployable": True}) + assert not is_deployable({"data": {**DATA_VERSION, "cols": "00000000"}, "deployable": True}) # other input columns + assert not is_deployable({"data": {**legacy, "features": -1}, "deployable": True}) + + +def test_build_complete_needs_every_feature_day(tmp_path, monkeypatch): + monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature"); monkeypatch.setattr(mature, "MATURE_FEAT_DIR", tmp_path / "feat") + from fly_trader import config + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay") # the real archive's pool_id gap must not leak in + monkeypatch.setattr(mature, "SKILL_DIR", tmp_path / "skill") # nor the frozen wallet-skill tables: this test is about missing parts + monkeypatch.setattr(mature, "days_ready", lambda: []) + t = pa.table({"mint": ["A"], "close": [1.0]}) + for d in ("2026-09-01", "2026-09-02"): + mature.write_part(t, tmp_path / "mature" / f"{d}.parquet", mature.AGG_VERSION) + ok, why = mature.build_complete() + assert not ok and "features still to build" in why # 09-02 has no feature part + mature.write_part(t, tmp_path / "feat" / "2026-09-02" / "part.parquet", mature.FEATURE_VERSION, {"fly_agg": mature.AGG_VERSION}) + assert mature.build_complete()[0] + mature.write_part(t, tmp_path / "feat" / "2026-09-02" / "part.parquet", mature.FEATURE_VERSION) # built before the aggregation stamp + assert not mature.build_complete()[0] diff --git a/tests/test_pool_id_guard.py b/tests/test_pool_id_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..a07498baf452c1f635d425d6fa020c18fdf86bd3 --- /dev/null +++ b/tests/test_pool_id_guard.py @@ -0,0 +1,33 @@ +"""The pool_id parity guard: an archive day with an hour file lacking the column is never aggregated (the live stream's +blocked-pool filter and dominant-pool choice need it), build_complete says why, and refetch marks those hours.""" +from datetime import datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq + +from fly_trader import config +from fly_trader.db.connection import transaction +from fly_trader.ingest import replay_pull +from fly_trader.train import mature + + +def _hour(path, with_pool_id: bool): + cols = {"ts": pa.array([datetime(2026, 9, 1, 0, 0, 5, tzinfo=timezone.utc)], pa.timestamp("ms", tz="UTC")), "slot": [1], "pool": ["pump-amm"], + "mint": ["Apump"], "trader": ["W"], "side": pa.array([1], pa.int8()), "sol": [1.0], "tokens": [10.0], "price": [0.1], "quote_in_pool": [50.0]} + if with_pool_id: + cols["pool_id"] = ["P1"] + path.parent.mkdir(parents=True, exist_ok=True); pq.write_table(pa.table(cols), path) + + +def test_day_without_pool_id_is_not_aggregated_and_hours_are_marked(tmp_path, monkeypatch, db_conn): + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay"); monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature") + _hour(tmp_path / "replay" / "2026-09-01" / "00_trades.parquet", True) + _hour(tmp_path / "replay" / "2026-09-01" / "01_trades.parquet", False) + _hour(tmp_path / "replay" / "2026-09-02" / "00_trades.parquet", True) + assert mature.days_missing_pool_id() == ["2026-09-01"] + assert mature.aggregate_day(datetime(2026, 9, 1).date()) == 0 and not (tmp_path / "mature" / "2026-09-01.parquet").exists() + assert replay_pull.refetch_missing_pool_id() == 1 + with transaction() as conn: + r = conn.execute("SELECT status FROM replay_hours WHERE hour = %s", (datetime(2026, 9, 1, 1, tzinfo=timezone.utc),)).fetchone() + conn.execute("DELETE FROM replay_hours WHERE hour = %s", (datetime(2026, 9, 1, 1, tzinfo=timezone.utc),)) + assert r["status"] == "refetch" diff --git a/tests/test_position_hold.py b/tests/test_position_hold.py new file mode 100644 index 0000000000000000000000000000000000000000..196d601d595621a01a8f7b6367b9c05b1567c70c --- /dev/null +++ b/tests/test_position_hold.py @@ -0,0 +1,55 @@ +"""Per-strategy holds in the book: a 10-minute and a 240-minute position exit at their own times; a row a filter +blocks is recorded with the filter's reason; a call without the new arguments behaves as before.""" +import uuid +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import numpy as np + +from fly_trader import config +from fly_trader.agent import paper_trading +from fly_trader.db.connection import transaction +from fly_trader.execution import ledger +from fly_trader.execution.broker_paper import PaperBroker + +BOOK = "replay_hold_test" + + +def _ctx(conn, m1, mints, price=1.0): + return SimpleNamespace(conn=conn, m1=m1, m1_epoch=m1.timestamp(), agg={m: {} for m in mints}, prices={m: price for m in mints}, resqs={m: 1000.0 for m in mints}, + fees={}, last_resq=lambda m: 1000.0, mcap=lambda m, p: 1e6) + + +def _info(m): + return {"resq": 1000.0, "pool": "P" + m, "age_h": 10.0, "price": 1.0, "mcap": 1e6, "decimals": 6, "program_label": None, "fee_rate": None} + + +def test_per_position_holds_and_filter_reasons(db_conn, monkeypatch): + monkeypatch.setattr(config, "CAPITAL_SOL", 5.0) + table = [{"lo": 0.0, "n": 100, "mean": 0.02, "win": 0.7, "kelly": 0.4}] + run = str(uuid.uuid4()); broker = PaperBroker(BOOK); m0 = datetime(2026, 9, 10, 12, 0, tzinfo=timezone.utc) + with transaction() as conn: + conn.execute("DELETE FROM positions WHERE book = %s", (BOOK,)); conn.execute("DELETE FROM wealth_marks WHERE book = %s", (BOOK,)) + conn.execute("UPDATE circuit_state SET kill_switch = false, entries_paused = false WHERE id = 1") + mints = ["SHORTpump", "LONGpump", "BLOCKpump"] + st = paper_trading.trade_minute(_ctx(conn, m0, mints), book=BOOK, run_id=run, beat_no=1, broker=broker, kind="selector", mints=mints, + infos=[_info(m) for m in mints], scores=np.array([0.05, 0.05, 0.05]), threshold=np.array([0.01, 0.01, 0.01]), table=None, + horizon_s=7200.0, holds=np.array([600.0, 14400.0, 600.0]), strategies=np.array(["capitulation", "breakout", "ev"], dtype=object), + allow=np.array([True, True, False]), reasons=np.array(["trade", "trade", "dump veto"], dtype=object), tables=[table] * 3) + assert st["entered"] == 2 + blk = conn.execute("SELECT rail FROM decisions WHERE run_id = %s AND kind = 'blocked' AND mint = 'BLOCKpump'", (run,)).fetchone() + assert blk["rail"] == "dump veto" + holds = {p["mint"]: (p["hold_s"], p["strategy"]) for p in ledger.open_positions(conn, BOOK)} + assert holds == {"SHORTpump": (600.0, "capitulation"), "LONGpump": (14400.0, "breakout")} + for minutes, left in ((11, {"LONGpump"}), (241, set())): + m1 = m0 + timedelta(minutes=minutes) + paper_trading.trade_minute(_ctx(conn, m1, mints), book=BOOK, run_id=run, beat_no=2, broker=broker, kind="selector", mints=[], infos=[], scores=np.array([]), + threshold=0.01, table=None, horizon_s=7200.0) + assert {p["mint"] for p in ledger.open_positions(conn, BOOK)} == left + # the scalar call of the single-strategy engine: the book's horizon applies + paper_trading.trade_minute(_ctx(conn, m0 + timedelta(minutes=300), ["OLDpump"]), book=BOOK, run_id=run, beat_no=3, broker=broker, kind="selector", + mints=["OLDpump"], infos=[_info("OLDpump")], scores=np.array([0.05]), threshold=0.01, table=table, horizon_s=7200.0) + p = ledger.open_positions(conn, BOOK)[0] + assert p["mint"] == "OLDpump" and p["hold_s"] is None and p["strategy"] is None + conn.execute("DELETE FROM positions WHERE book = %s", (BOOK,)); conn.execute("DELETE FROM wealth_marks WHERE book = %s", (BOOK,)) + conn.execute("DELETE FROM decisions WHERE run_id = %s", (run,)); conn.execute("DELETE FROM fills WHERE book = %s", (BOOK,)) diff --git a/tests/test_pumpstream.py b/tests/test_pumpstream.py new file mode 100644 index 0000000000000000000000000000000000000000..818310c45cf8f85ce0e640ec86c2854c7bcfeba2 --- /dev/null +++ b/tests/test_pumpstream.py @@ -0,0 +1,91 @@ +"""PumpAPI stream aggregation: filters, minute boundaries, idempotent flush.""" +from datetime import datetime, timezone + +from fly_trader.ingest import pumpstream + +WSOL = "So11111111111111111111111111111111111111112" + + +def _ev(action, mint, ts_ms, price, sol, trader, pool="pump-amm", quote=WSOL, resq=80.0): + return {"action": action, "pool": pool, "mint": mint, "timestamp": ts_ms, "price": price, "quoteMint": quote, "quoteInPool": resq, + "txSigner": trader, "breakdown": [{"action": action, "trader": trader, "quoteAmount": sol, "tokenAmount": 1.0}], "poolId": "P1"} + + +def test_aggregation_and_flush(db_conn): + agg = pumpstream.Aggregator() + m = 1_800_000_000 # a minute boundary (seconds) + agg.ingest(_ev("buy", "Apump", (m + 1) * 1000, 1.0, 2.0, "t1")) + agg.ingest(_ev("sell", "Apump", (m + 30) * 1000, 0.9, 1.0, "t2")) + agg.ingest(_ev("buy", "Apump", (m + 59) * 1000, 1.1, 3.0, "t1")) + agg.ingest(_ev("buy", "Apump", (m + 61) * 1000, 1.2, 1.0, "t3")) # next minute + agg.ingest(_ev("buy", "Bpump", (m + 5) * 1000, 1.0, 1.0, "t9", quote="USDC")) # not SOL-quoted: ignored + agg.ingest(_ev("buy", "Cother", (m + 5) * 1000, 1.0, 1.0, "t9")) # not pump.fun-origin (no "pump" suffix): ignored + agg.ingest(_ev("buy", "Dpump", (m + 5) * 1000, 1.0, 1.0, "t9", pool="pump")) # bonding curve: not a PumpSwap minute + assert set(agg.minutes) == {m, m + 60} and list(agg.minutes[m]) == ["Apump"] + r = agg.row(m, "Apump") + assert (r.open, r.high, r.low, r.close) == (1.0, 1.1, 0.9, 1.1) and (r.buy, r.sell, r.nb, r.ns) == (5.0, 1.0, 2, 1) and len(r.traders) == 2 + with db_conn.cursor() as cur: + cur.execute("DELETE FROM pump_minutes WHERE mint = 'Apump'") + db_conn.commit() + assert agg.flush(m) == 0 # the current minute is still open + assert agg.flush(m + 60) == 1 # the minute that closed at m+60 is written + assert agg.flushed_through == datetime.fromtimestamp(m, timezone.utc) + row = db_conn.execute("SELECT open, high, low, close, buy_sol, sell_sol, n_buys, n_sells, n_traders, resq_sol FROM pump_minutes WHERE mint = 'Apump'").fetchone() + assert (row["open"], row["close"], row["buy_sol"], row["n_traders"], row["resq_sol"]) == (1.0, 1.1, 5.0, 2, 80.0) + assert m + 60 in agg.minutes and m not in agg.minutes + + +def test_band_dominant_pool_and_watermark(): + agg = pumpstream.Aggregator() + m = 1_800_000_000 + for k in range(5): + agg.ingest(_ev("buy", "Epump", (m + 1 + k) * 1000, 1.0, 1.0, f"t{k}")) + agg.ingest(_ev("buy", "Epump", (m + 10) * 1000, 100.0, 1.0, "tz")) # 100x off the trailing median: dropped + assert agg.stats["dropped_band"] == 1 and agg.row(m, "Epump").high == 1.0 + other = _ev("buy", "Epump", (m + 20) * 1000, 1.05, 9.0, "ty"); other["poolId"] = "P2" + agg.ingest(other) # a second pool with fewer legs this minute + assert agg.row(m, "Epump").pool_id == "P1" and agg.row(m, "Epump").nb == 5 + agg.ingest(_ev("buy", "Epump", (m + 1) * 1000, 1.0, 1.0, "t0", resq=500_000.0)) # reserve outside the band: ignored + assert agg.row(m, "Epump").nb == 5 + # the minute is complete once an event stamped >= 2 s after its end arrives, or 15 s after it by wall clock + assert agg.flush_bound(m + 30) == m # minute m is still open: only minutes before m are complete + agg.max_event_s = m + 62.5 + assert agg.flush_bound(m + 63) == m + 60 # an event 2.5 s past its end completes minute m + agg.max_event_s = m + 20 + assert agg.flush_bound(m + 76) == m + 60 # quiet stream: 15 s of wall clock completes it + + +def test_midnight_late_legs_and_lagging_stream(): + agg = pumpstream.Aggregator(); M = 20834 * 86400 # a UTC midnight + for k in range(5): + agg.ingest(_ev("buy", "Fpump", (M + 1 + k) * 1000, 1.0, 1.0, f"t{k}")) + agg.ingest(_ev("buy", "Fpump", (M - 1) * 1000, 1.0, 1.0, "late")) # a late event from the previous day + agg.ingest(_ev("buy", "Fpump", (M + 10) * 1000, 100.0, 1.0, "tz")) + assert agg.stats["dropped_band"] == 1 # today's references survived the late event + agg.flushed_through = datetime.fromtimestamp(M, timezone.utc) + agg.ingest(_ev("buy", "Fpump", (M + 30) * 1000, 1.0, 1.0, "t9")) # minute M is already written + assert agg.stats["dropped_late"] == 1 + agg.max_event_s = M + 55; agg.last_event_wall = M + 70 + assert agg.flush_bound(M + 75) == M # lagging but live: waits for the minute's events + agg.last_event_wall = M + 55 + assert agg.flush_bound(M + 75) == M + 60 # quiet for 20 s: the wall clock completes it + + +def test_load_skill_falls_back_to_the_newest_table_on_disk(tmp_path, monkeypatch): + """A distribution ships one table and no pipeline to make the next: the engine keeps using the newest it has rather + than dropping the skill inputs (the selector fails closed without them, the fly reads them as zero).""" + import pyarrow as pa + import pyarrow.parquet as pq + + from fly_trader.train import mature + monkeypatch.setattr(mature, "SKILL_DIR", tmp_path / "skill"); (tmp_path / "skill").mkdir() + agg = pumpstream.Aggregator() + pumpstream.load_skill(agg, datetime(2026, 9, 25, 12, tzinfo=timezone.utc).timestamp()) + assert agg.skill is None and agg.skill_day == "2026-09-25" # nothing on disk: as before + for d, w in (("2026-09-20", "OLD"), ("2026-09-23", "NEW")): + pq.write_table(pa.table({"wallet": [w], "bucket": pa.array([7], pa.int8())}), tmp_path / "skill" / f"{d}.parquet") + agg.skill = None + pumpstream.load_skill(agg, datetime(2026, 9, 25, 12, tzinfo=timezone.utc).timestamp()) + assert agg.skill == {"NEW": 7} and agg.skill_day == "2026-09-25" # the newest, two days stale + pumpstream.load_skill(agg, datetime(2026, 9, 23, 12, tzinfo=timezone.utc).timestamp()) + assert agg.skill == {"NEW": 7} and agg.skill_day == "2026-09-23" # the day's own table when it exists diff --git a/tests/test_rails.py b/tests/test_rails.py new file mode 100644 index 0000000000000000000000000000000000000000..6f159a0a7fdf59f6d6860d41228f94ea00e406b5 --- /dev/null +++ b/tests/test_rails.py @@ -0,0 +1,16 @@ +from fly_trader import config +from fly_trader.agent import rails + + +def test_kill_switch_and_circuit(db_conn): + db_conn.execute("UPDATE circuit_state SET fail_count=0, tripped=false, kill_switch=false, peak_wealth=NULL, entries_paused=false WHERE id=1") + assert not rails.check_drawdown(db_conn, 4.0, 5.0) # 20 % below the peak: trading on + assert rails.check_drawdown(db_conn, 5.0 * (1 - config.KILL_SWITCH_DRAWDOWN) - 0.01, 5.0) + assert rails.load_circuit(db_conn).kill_switch + for _ in range(config.CIRCUIT_THRESHOLD): + st = rails.record_failure(db_conn, "boom") + assert st.tripped + rails.record_success(db_conn) + assert rails.load_circuit(db_conn).fail_count == 0 + + diff --git a/tests/test_reset.py b/tests/test_reset.py new file mode 100644 index 0000000000000000000000000000000000000000..5911293011894c184349f6447f9371b34f4d3c2e --- /dev/null +++ b/tests/test_reset.py @@ -0,0 +1,112 @@ +from fly_trader import config +from fly_trader.db.connection import transaction +from fly_trader.ops.reset import reset_training_state + + +def test_reset_wipes_paper_but_keeps_live_and_market_data(db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "PG_ARCHIVE_DIR", tmp_path / "arch") + monkeypatch.setattr(config, "BRAIN_DIR", tmp_path / "brain") + with transaction() as conn: + conn.execute("INSERT INTO positions (book, mint, qty, cost_sol, status) VALUES ('paper_free','M1',1,0.1,'open'),('live','M2',1,0.1,'open')") + conn.execute("INSERT INTO beats (beat_no) VALUES (1)") + conn.execute("INSERT INTO tokens (mint, watch_status) VALUES ('M1','watch') ON CONFLICT DO NOTHING") + out = reset_training_state(reason="test") + with transaction() as conn: + assert conn.execute("SELECT count(*) AS n FROM positions WHERE book='paper_free'").fetchone()["n"] == 0 + assert conn.execute("SELECT count(*) AS n FROM positions WHERE book='live'").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM beats").fetchone()["n"] == 0 + assert conn.execute("SELECT count(*) AS n FROM tokens WHERE mint='M1'").fetchone()["n"] == 1 + conn.execute("DELETE FROM positions WHERE book='live' AND mint='M2'") + assert (tmp_path / "arch").exists() and out["rows"]["positions"] >= 1 + + +def test_selector_status_cleared_and_training_stats_reset(db_conn, tmp_path, monkeypatch): + from fly_trader.ops.reset import reset_training_stats + monkeypatch.setattr(config, "PG_ARCHIVE_DIR", tmp_path / "arch") + monkeypatch.setattr(config, "BRAIN_DIR", tmp_path / "brain") + with transaction() as conn: + conn.execute("INSERT INTO ui_settings (key, value) VALUES ('selector_status', '{}'::jsonb), ('training_status', '{}'::jsonb) ON CONFLICT (key) DO NOTHING") + conn.execute("INSERT INTO events (level, source, message) VALUES ('info','selector','walk-forward 2026-09-01'), ('info','selector','selector saved (snapshot 1)'), " + "('info','fly_selector','fly epoch 1'), ('info','selector','selector session started')") + reset_training_state(reason="test") + with transaction() as conn: + assert conn.execute("SELECT count(*) AS n FROM ui_settings WHERE key = 'selector_status'").fetchone()["n"] == 0 + out = reset_training_stats("selector", reason="test") + msgs = ("('walk-forward 2026-09-01','selector saved (snapshot 1)','fly epoch 1','selector session started')") + with transaction() as conn: + left = sorted(r["message"] for r in conn.execute("SELECT message FROM events WHERE source IN ('selector','fly_selector') AND message IN " + msgs).fetchall()) + assert left == ["fly epoch 1", "selector session started"] # another regimen's stats and session events stay + assert conn.execute("SELECT count(*) AS n FROM ui_settings WHERE key = 'training_status'").fetchone()["n"] == 0 + assert out["events"] >= 2 + reset_training_stats("fly_selector", reason="test") + with transaction() as conn: + assert [r["message"] for r in conn.execute("SELECT message FROM events WHERE source IN ('selector','fly_selector') AND message IN " + msgs).fetchall()] == ["selector session started"] + conn.execute("DELETE FROM events WHERE message = 'selector session started'") + + +def _race_fixture(conn): + import uuid + runs = {k: str(uuid.uuid4()) for k in ("selector", "fly", "pretrain")} + for kind, rid in runs.items(): + conn.execute("INSERT INTO runs (run_id, kind) VALUES (%s, %s)", (rid, kind)) + beats = {k: conn.execute("INSERT INTO beats (run_id, beat_no) VALUES (%s, 1) RETURNING id", (rid,)).fetchone()["id"] for k, rid in runs.items()} + for k, rid in runs.items(): + conn.execute("INSERT INTO decisions (run_id, beat_id, kind, mint) VALUES (%s, %s, %s, 'RACE_T')", (rid, beats[k], f"{k}_enter")) + conn.execute("INSERT INTO positions (book, mint, qty, cost_sol, status) VALUES ('paper_selector','RACE_S',1,0.1,'open'),('paper_fly','RACE_F',1,0.1,'open'),('paper_free','RACE_X',1,0.1,'open')") + for book, k in (("paper_selector", "selector"), ("paper_fly", "fly"), ("paper_free", "pretrain")): + conn.execute("INSERT INTO wealth_marks (beat_id, book, wealth) VALUES (%s, %s, 5.0)", (beats[k], book)) + conn.execute("INSERT INTO fly_scored (ts, mint, score) VALUES (now(), 'RACE_F', 0.01)") + conn.execute("INSERT INTO fly_rollbacks (reason) VALUES ('test')") + conn.execute("INSERT INTO brain_snapshots (path, kind) VALUES ('p1','fly_plastic'), ('p2','fly_selector'), ('p3','checkpoint')") + conn.execute("INSERT INTO book_state (book, halted) VALUES ('paper_fly', true) ON CONFLICT (book) DO UPDATE SET halted = true") + return runs + + +def _n(conn, sql, *a): + return conn.execute(sql, a).fetchone()["n"] + + +def test_runner_reset_keeps_the_race_books_and_the_fly_and_reset_fly_clears_only_the_fly(db_conn, tmp_path, monkeypatch): + from fly_trader.ops.reset import reset_fly + monkeypatch.setattr(config, "PG_ARCHIVE_DIR", tmp_path / "arch") + monkeypatch.setattr(config, "BRAIN_DIR", tmp_path / "brain") + (tmp_path / "brain" / "plastic").mkdir(parents=True); (tmp_path / "brain" / "plastic" / "state.pt").write_bytes(b"x") + with transaction() as conn: + runs = _race_fixture(conn) + reset_training_state(reason="test") + with transaction() as conn: + assert _n(conn, "SELECT count(*) AS n FROM positions WHERE mint IN ('RACE_S','RACE_F')") == 2 + assert _n(conn, "SELECT count(*) AS n FROM positions WHERE mint = 'RACE_X'") == 0 + assert _n(conn, "SELECT count(*) AS n FROM wealth_marks WHERE book IN ('paper_selector','paper_fly')") >= 2 + assert _n(conn, "SELECT count(*) AS n FROM wealth_marks WHERE book = 'paper_free'") == 0 + assert _n(conn, "SELECT count(*) AS n FROM decisions WHERE mint = 'RACE_T'") == 2 # the race sessions' decisions + assert _n(conn, "SELECT count(*) AS n FROM runs WHERE run_id = %s", runs["pretrain"]) == 0 + assert _n(conn, "SELECT count(*) AS n FROM fly_scored WHERE mint = 'RACE_F'") == 1 + assert _n(conn, "SELECT count(*) AS n FROM brain_snapshots WHERE path IN ('p1','p2')") == 2 + assert _n(conn, "SELECT count(*) AS n FROM brain_snapshots WHERE path = 'p3'") == 0 + assert (tmp_path / "brain" / "plastic" / "state.pt").exists() + reset_fly(reason="test") + with transaction() as conn: + assert _n(conn, "SELECT count(*) AS n FROM positions WHERE mint = 'RACE_F'") == 0 + assert _n(conn, "SELECT count(*) AS n FROM positions WHERE mint = 'RACE_S'") == 1 # the selector's book stays + assert _n(conn, "SELECT count(*) AS n FROM decisions WHERE mint = 'RACE_T'") == 1 + assert _n(conn, "SELECT count(*) AS n FROM fly_scored") == 0 and _n(conn, "SELECT count(*) AS n FROM fly_rollbacks") == 0 + assert _n(conn, "SELECT count(*) AS n FROM brain_snapshots WHERE path = 'p1'") == 0 + assert _n(conn, "SELECT count(*) AS n FROM brain_snapshots WHERE path = 'p2'") == 1 # the bootstrap stays + assert _n(conn, "SELECT count(*) AS n FROM book_state WHERE book = 'paper_fly'") == 0 + conn.execute("DELETE FROM positions WHERE mint LIKE 'RACE_%%'"); conn.execute("DELETE FROM decisions WHERE mint = 'RACE_T'") + conn.execute("DELETE FROM wealth_marks WHERE book IN ('paper_selector','paper_fly')"); conn.execute("DELETE FROM runs WHERE run_id = %s", (runs["selector"],)) + conn.execute("DELETE FROM brain_snapshots WHERE path = 'p2'") + assert not (tmp_path / "brain" / "plastic").exists() and list((tmp_path / "arch").glob("reset_fly_*/plastic/state.pt")) + + +def test_selector_runner_resets_on_start(monkeypatch): + from fly_trader.agent import minute_engine, runner + from fly_trader.ops import reset + calls = [] + monkeypatch.setattr(config, "LIVE_ENABLED", False); monkeypatch.setattr(config, "RESET_ON_START", True) + monkeypatch.setattr(reset, "reset_training_state", lambda reason="": calls.append("reset")) + monkeypatch.setattr(minute_engine, "main", lambda stop_event=None, live=False: calls.append("session")) + monkeypatch.setattr(runner, "record_event", lambda *a, **k: None) + runner._main() + assert calls == ["reset", "session"] diff --git a/tests/test_review_fixes.py b/tests/test_review_fixes.py new file mode 100644 index 0000000000000000000000000000000000000000..3e56ecccf8f59c3f2e6b221cbc6a611c976e4490 --- /dev/null +++ b/tests/test_review_fixes.py @@ -0,0 +1,81 @@ +"""Pins for the second review: shared creator history, discovery audit handling, parquet writer restore, config redaction.""" +from datetime import datetime, timedelta, timezone + +import pytest + +from fly_trader import config +from fly_trader.train.corpus_meta import creator_history + + +def test_creator_history_point_in_time(db_conn): + c = "CreatorTest1"; g0 = datetime(2026, 9, 1, 12, 0, tzinfo=timezone.utc) + with db_conn.cursor() as cur: + cur.execute("DELETE FROM corpus_meta WHERE creator = %s", (c,)); cur.execute("DELETE FROM pump_events WHERE signer = %s", (c,)) + for k in range(3): # three earlier creates + cur.execute("INSERT INTO pump_events (sig, ts, action, pool, mint, signer) VALUES (%s,%s,'create','pump',%s,%s)", (f"sigct{k}", g0 - timedelta(hours=5 - k), f"M{k}", c)) + # graduation A 3 h before B (rug, outcome known), graduation A2 30 min before B (outcome not yet known at B) + cur.execute("INSERT INTO corpus_meta (mint, creator, graduated_at, own_dd60, own_max60) VALUES ('M0', %s, %s, -0.95, 0.1)", (c, g0 - timedelta(hours=3))) + cur.execute("INSERT INTO corpus_meta (mint, creator, graduated_at, own_dd60, own_max60) VALUES ('M1', %s, %s, -0.10, 1.5)", (c, g0 - timedelta(minutes=30))) + db_conn.commit() + h = creator_history(db_conn, c, g0 - timedelta(hours=1), g0) + assert h["prior_launches"] == 3 and h["prior_grads"] == 2 + assert h["prior_known"] == 1 and h["prior_rug_share"] == 1.0 and h["prior_moon_share"] == 0.0 + assert creator_history(db_conn, None, None, g0)["prior_grads"] is None + with db_conn.cursor() as cur: + cur.execute("DELETE FROM corpus_meta WHERE creator = %s", (c,)); cur.execute("DELETE FROM pump_events WHERE signer = %s", (c,)) + db_conn.commit() + + +def test_nul_bytes_never_reach_postgres_text(): + import numpy as np + from fly_trader.ingest import pumpstream + from fly_trader.train.corpus_meta import _pg + assert _pg("a\x00b") == "ab" and _pg(np.float64("nan")) is None and _pg(np.int64(3)) == 3 + agg = pumpstream.Aggregator() + agg.ingest({"action": "create", "pool": "pump", "mint": "Npump", "timestamp": 1, "txSigner": "c", "name": "x\x00y", "symbol": "S\x00", "uri": None, "signature": "s"}) + assert agg.creates["Npump"]["name"] == "xy" and agg.creates["Npump"]["symbol"] == "S" + + +def test_stream_trader_falls_back_to_signer_like_the_archive(): + from fly_trader.ingest import pumpstream + agg = pumpstream.Aggregator(); m = 1_800_000_000 + agg.ingest({"action": "buy", "pool": "pump-amm", "mint": "Tpump", "timestamp": (m + 1) * 1000, "price": 1.0, "quoteMint": pumpstream.WSOL, "quoteInPool": 80.0, + "txSigner": "signer1", "breakdown": [{"action": "buy", "quoteAmount": 1.0}], "poolId": "P1"}) + assert agg.row(m, "Tpump").traders == {"signer1"} + + +def test_replay_parses_both_archive_formats(tmp_path): + import orjson + import zstandard + from fly_trader.ingest.replay_pull import parse_hour + old = {"txType": "buy", "pool": "pump-amm", "poolId": "P", "mint": "Mpump", "quoteMint": "So11111111111111111111111111111111111111112", "txSigner": "T", + "tokenAmount": 10.0, "solAmount": 0.5, "solInPool": 80.0, "tokensInPool": 1e6, "price": 1e-5, "timestamp": 1778414399974, "block": 1} + new = {"action": "sell", "pool": "pump-amm", "poolId": "P", "mint": "Mpump", "quoteMint": "So11111111111111111111111111111111111111112", "txSigner": "U", + "tokenAmount": 5.0, "quoteAmount": 0.2, "quoteInPool": 79.8, "tokensInPool": 1e6, "price": 1e-5, "timestamp": 1778414400974, "block": 2, + "breakdown": [{"action": "sell", "trader": "U", "tokenAmount": 5.0, "quoteAmount": 0.2}]} + mig = {"txType": "migrate", "pool": "pump-amm", "poolId": "P", "mint": "Mpump", "solAmount": 1.1, "solInPool": 85.0, "poolCreatedBy": "pump", "timestamp": 1778414401974, "signature": "s"} + path = tmp_path / "h.zst" + path.write_bytes(zstandard.ZstdCompressor().compress(b"\n".join(orjson.dumps(e) for e in (old, new, mig)))) + trades, events, n = parse_hour(path) + t = trades.to_pylist() + assert n == 3 and [(r["side"], r["sol"], r["quote_in_pool"]) for r in t] == [(1, 0.5, 80.0), (-1, 0.2, 79.8)] # the pre-2026-05-21 fields are read + e = events.to_pylist() + assert len(e) == 1 and e[0]["action"] == "migrate" and e[0]["quote_in_pool"] == 85.0 and e[0]["pool_created_by"] == "pump" + + +def test_replay_keeps_finished_downloads_across_restarts(tmp_path): + from fly_trader.ingest.replay_pull import _kept_downloads + h1 = datetime(2026, 5, 14, 19, tzinfo=timezone.utc); h2 = datetime(2026, 5, 14, 20, tzinfo=timezone.utc) + (tmp_path / "2026051419.zst").write_bytes(b"x" * 10) # finished, still planned: kept + (tmp_path / "2026051420.part").write_bytes(b"x" * 5) # partial: cannot resume, removed + (tmp_path / "2026051321.zst").write_bytes(b"x") # finished but no longer planned: removed + kept = _kept_downloads(tmp_path, {h1, h2}) + assert [(h, p.name, n) for h, p, n, _ in kept] == [(h1, "2026051419.zst", 10)] + assert sorted(p.name for p in tmp_path.iterdir()) == ["2026051419.zst"] + + +def test_summary_redacts_every_dsn_form(monkeypatch): + for dsn in ("postgresql://bot:s3cret@host:5432", "postgresql://bot:s3cret@host:5432/fly", "dbname=fly user=u password=hunter2 host=db"): + monkeypatch.setattr(config, "DATABASE_URL", dsn, raising=False) + out = str(config.summary().get("DATABASE_URL")) + assert "s3cret" not in out and "hunter2" not in out diff --git a/tests/test_signing.py b/tests/test_signing.py new file mode 100644 index 0000000000000000000000000000000000000000..4c0f1d226152aba721bbb9f1a63478f8069d013a --- /dev/null +++ b/tests/test_signing.py @@ -0,0 +1,92 @@ +"""Signing touches only our signature slot: message, header and fee payer stay byte-identical.""" +import base64 + +import pytest +from solders.hash import Hash +from solders.instruction import AccountMeta, Instruction +from solders.keypair import Keypair +from solders.message import MessageV0, to_bytes_versioned +from solders.pubkey import Pubkey +from solders.signature import Signature +from solders.system_program import TransferParams, transfer +from solders.transaction import VersionedTransaction + +from fly_trader.chain.signing import (NotASigner, n_lookup_programs, sign_transaction_b64, signer_index, + static_program_ids, transaction_id) + +SYSTEM = "11111111111111111111111111111111" + + +def _message(payer: Keypair, extra_signer: Keypair | None = None) -> MessageV0: + ixs = [transfer(TransferParams(from_pubkey=payer.pubkey(), to_pubkey=Pubkey.new_unique(), lamports=1000))] + if extra_signer is not None: + ixs.append(Instruction(Pubkey.from_string(SYSTEM), b"\x00", + [AccountMeta(extra_signer.pubkey(), True, True), AccountMeta(Pubkey.new_unique(), False, True)])) + return MessageV0.try_compile(payer.pubkey(), ixs, [], Hash.new_unique()) + + +def _unsigned_b64(msg: MessageV0, sigs: list[Signature] | None = None) -> str: + n = msg.header.num_required_signatures + tx = VersionedTransaction.populate(msg, sigs or [Signature.default()] * n) + return base64.b64encode(bytes(tx)).decode() + + +def test_sign_with_fee_payer_verifies_and_leaves_message_untouched(): + payer = Keypair() + msg = _message(payer) + signed_b64, idx = sign_transaction_b64(_unsigned_b64(msg), payer) + assert idx == 0 + tx = VersionedTransaction.from_bytes(base64.b64decode(signed_b64)) + assert tx.signatures[0].verify(payer.pubkey(), to_bytes_versioned(tx.message)) + assert tx.verify_with_results() == [True] + assert to_bytes_versioned(tx.message) == to_bytes_versioned(msg) + assert tx.message.header == msg.header + assert tx.message.account_keys[0] == payer.pubkey() + assert transaction_id(signed_b64) == str(tx.signatures[0]) + + +def test_keypair_that_is_not_a_signer_raises(): + payer, stranger = Keypair(), Keypair() + with pytest.raises(NotASigner): + sign_transaction_b64(_unsigned_b64(_message(payer)), stranger) + + +def test_present_but_non_signer_account_raises(): + payer, dest = Keypair(), Keypair() + ix = transfer(TransferParams(from_pubkey=payer.pubkey(), to_pubkey=dest.pubkey(), lamports=1)) + msg = MessageV0.try_compile(payer.pubkey(), [ix], [], Hash.new_unique()) + assert dest.pubkey() in list(msg.account_keys) + with pytest.raises(NotASigner, match="only the first 1"): + signer_index(_unsigned_b64(msg), dest.pubkey()) + + +def test_existing_other_signature_is_preserved(): + payer, other = Keypair(), Keypair() + msg = _message(payer, other) + assert msg.header.num_required_signatures == 2 + other_idx = list(msg.account_keys).index(other.pubkey()) + other_sig = other.sign_message(to_bytes_versioned(msg)) + sigs = [Signature.default()] * 2 + sigs[other_idx] = other_sig + signed_b64, idx = sign_transaction_b64(_unsigned_b64(msg, sigs), payer) + tx = VersionedTransaction.from_bytes(base64.b64decode(signed_b64)) + assert idx == 0 and other_idx == 1 + assert tx.signatures[other_idx] == other_sig + assert tx.verify_with_results() == [True, True] + + +def test_second_signer_slot_can_be_signed_alone(): + payer, other = Keypair(), Keypair() + msg = _message(payer, other) + signed_b64, idx = sign_transaction_b64(_unsigned_b64(msg), other) + tx = VersionedTransaction.from_bytes(base64.b64decode(signed_b64)) + assert idx == 1 + assert tx.signatures[0] == Signature.default() # fee payer slot untouched + assert tx.signatures[1].verify(other.pubkey(), to_bytes_versioned(msg)) + + +def test_static_program_ids(): + payer = Keypair() + b64 = _unsigned_b64(_message(payer)) + assert static_program_ids(b64) == [SYSTEM] + assert n_lookup_programs(b64) == 0 diff --git a/tests/test_sizing.py b/tests/test_sizing.py new file mode 100644 index 0000000000000000000000000000000000000000..31f89650d4990696dd58c11e411b8c07b4d3b6c1 --- /dev/null +++ b/tests/test_sizing.py @@ -0,0 +1,55 @@ +"""Position sizing: growth-optimal fractions, score bands, caps and the bankroll replay.""" +import numpy as np +import pytest + +from fly_trader import config +from fly_trader.agent import sizing + + +def test_kelly_counts_the_tail_and_needs_an_edge(): + assert sizing.kelly_fraction(np.array([-0.01, 0.005, -0.02])) == 0.0 # no edge + wins = np.full(90, 0.10); rugs = np.full(10, -1.0) + assert sizing.kelly_fraction(np.r_[wins, rugs]) == 0.0 # +10 % x 90 cannot pay for 10 total losses (mean -1 %) + f = sizing.kelly_fraction(np.r_[np.full(95, 0.10), np.full(5, -1.0)]) # mean +4.5 %: a small bet + assert 0.0 < f < 0.5 + f_even = sizing.kelly_fraction(np.r_[np.full(50, 0.2), np.full(50, -0.1)]) # classic: p/L - q/W = 0.5/0.1 - 0.5/0.2 = 2.5 -> capped by the grid + assert f_even == pytest.approx(0.99) + + +def test_bands_rank_by_margin_and_higher_certainty_bets_more(monkeypatch): + rng = np.random.default_rng(0); m = rng.uniform(0, 0.1, 4000) + r = np.where(rng.random(4000) < 0.86 + 0.8 * m, 0.08, -0.50) # +8 % wins vs -50 % crashes; the win rate rises with the margin + table = sizing.build_table(m, r) + assert len(table) == sizing.N_BANDS and table[0]["lo"] == 0.0 and [b["lo"] for b in table] == sorted(b["lo"] for b in table) + assert table[-1]["kelly"] > table[0]["kelly"] + monkeypatch.setattr(config, "GAS_RESERVE_SOL", 0.3); monkeypatch.setattr(config, "KELLY_FRACTION", 0.25); monkeypatch.setattr(config, "MAX_POSITION_FRACTION", 1.0) + monkeypatch.setattr(config, "MAX_POOL_SHARE", 1.0); monkeypatch.setattr(config, "MIN_POSITION_SOL", 0.0) + lo, _ = sizing.size_position(0.5 + 0.001, 0.5, table, 10.3, 10.3, 1000.0) + hi, _ = sizing.size_position(0.5 + 0.099, 0.5, table, 10.3, 10.3, 1000.0) + assert hi > lo >= 0 and hi > 0 # the weakest band may have no edge at all + + +def test_caps_minimum_and_fallback(monkeypatch): + table = [{"lo": 0.0, "n": 100, "mean": 0.05, "win": 0.7, "kelly": 0.8}] + monkeypatch.setattr(config, "GAS_RESERVE_SOL", 0.3); monkeypatch.setattr(config, "KELLY_FRACTION", 0.25); monkeypatch.setattr(config, "MAX_POSITION_FRACTION", 0.10) + monkeypatch.setattr(config, "MAX_POOL_SHARE", 0.02); monkeypatch.setattr(config, "MIN_POSITION_SOL", 0.02); monkeypatch.setattr(config, "MAX_POSITION_SOL", 0.1) + s, why = sizing.size_position(0.9, 0.8, table, 5.3, 5.3, 1000.0) + assert s == pytest.approx(0.5) and "bankroll cap" in why # 0.25*0.8*5 = 1.0 -> 10 % of 5 = 0.5 + s, why = sizing.size_position(0.9, 0.8, table, 5.3, 5.3, 10.0) + assert s == pytest.approx(0.2) and "pool depth" in why # 2 % of a 10 SOL pool + s, why = sizing.size_position(0.9, 0.8, table, 5.3, 0.31, 1000.0) + assert s == 0.0 and "minimum" in why # only 0.01 SOL free above the reserve + assert sizing.size_position(0.7, 0.8, table, 5.3, 5.3, 1000.0)[0] == 0.0 # below the line: no band + assert sizing.size_position(0.9, 0.8, [], 5.3, 5.3, 1000.0)[0] == pytest.approx(0.1) # no table: the fixed size + + +def test_bankroll_replay_enforces_cash_and_compounds(monkeypatch): + monkeypatch.setattr(config, "GAS_RESERVE_SOL", 0.3); monkeypatch.setattr(config, "KELLY_FRACTION", 0.25); monkeypatch.setattr(config, "MAX_POSITION_FRACTION", 0.10) + monkeypatch.setattr(config, "MIN_POSITION_SOL", 0.02); monkeypatch.setattr(config, "MAX_POSITION_SOL", 0.1) + ts = np.arange(200) * 3600.0; m = np.full(200, 0.05); r = np.full(200, 0.05) # every trade +5 %, one at a time + table = [{"lo": 0.0, "n": 200, "mean": 0.05, "win": 1.0, "kelly": 0.99}] + sized = sizing.simulate_bankroll(ts, 1800.0, m, r, table, start_sol=5.0); fixed = sizing.simulate_bankroll(ts, 1800.0, m, r, None, start_sol=5.0) + assert sized["final_sol"] > fixed["final_sol"] > 5.0 and sized["max_drawdown"] == 0.0 and sized["trades"] == 200 + monkeypatch.setattr(config, "MAX_POOL_SHARE", 0.02) + shallow = sizing.simulate_bankroll(ts, 1800.0, m, r, table, start_sol=5.0, res_quote=np.full(200, 1.0)) # 1 SOL pools: 0.02 SOL cap + assert shallow["final_sol"] < sized["final_sol"] and shallow["trades"] == 200 # the pool-depth cap binds, as live diff --git a/tests/test_strategies.py b/tests/test_strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..f020925ff30c64b7460cefde15a5c5d6be24a91a --- /dev/null +++ b/tests/test_strategies.py @@ -0,0 +1,174 @@ +"""The strategy stack: every fit records the best setting it reached even when none clears the bars; the objective (win rate, profit factor, trade count; weekly totals recorded, not required) orders settings with +admissible ones first; a filter must beat random removal; on a planted edge the stack finds an admissible ev strategy, +records every component with a reason, and the live decision reproduces the fitted rule.""" +import threading +from datetime import datetime, timedelta, timezone + +import numpy as np +import pytest + +from fly_trader.train import strategies as S +from fly_trader.train.decisions import X_COLS, DecisionSet + + +@pytest.fixture(autouse=True) +def _cache_in_tmp(tmp_path, monkeypatch): + """The walk-forward cache belongs to the corpus: no test may write into it.""" + monkeypatch.setattr(S, "CACHE_DIR", tmp_path / "stack_cache") + + +def _ds(days=42, per_day=600, seed=0): + rng = np.random.default_rng(seed); n = days * per_day + X = np.zeros((n, len(X_COLS)), np.float32); c = {k: i for i, k in enumerate(X_COLS)} + X[:, c["age_known"]] = 0.0 + sig = rng.normal(size=n).astype(np.float32); X[:, c["imb_5m"]] = sig + X[:, c["ret_1m"]] = rng.normal(0, 0.02, n); X[:, c["mkt_vol_1h"]] = rng.normal(5, 1, n) + d0 = datetime(2026, 7, 1, tzinfo=timezone.utc); k = np.arange(n) + ts = np.array([(d0 + timedelta(days=int(i // per_day), minutes=int(i % per_day) * 2)).timestamp() for i in k]) + day = np.array([(d0 + timedelta(days=int(i // per_day))).date() for i in k], dtype=object) + good = sig > 1.2 + fwd = np.where(good, np.where(rng.random(n) < 0.8, 0.05, -0.04), rng.normal(-0.01, 0.03, n)).astype(np.float32) + fh = {h: fwd.copy() for h in S.HOLDS_MIN} + return DecisionSet(X=X, y=(fwd > 0).astype(np.int8), fwd=fwd, fwd_pess=fwd, day=day, ts=ts, mint=np.array([f"m{i % 150}" for i in k]), + cols=list(X_COLS), horizon_s=7200.0, fwd_h=fh) + + +def test_score_sign_test_and_ordering(monkeypatch): + monkeypatch.setattr(S, "_min_trades", lambda: 5) + ds = _ds(days=21, per_day=50) + r = np.where(np.arange(len(ds.y)) % 10 < 7, 0.02, -0.01).astype(np.float32) + s = S.score_trades(ds, np.arange(len(ds.y)), r, ds.day[0]) + assert s.win == pytest.approx(0.7) and s.pf == pytest.approx((0.7 * 0.02) / (0.3 * 0.01)) and s.weeks == 3 and s.weeks_pos == 3 + assert not s.sign_ok and s.base_ok # the weekly statistic is recorded, no longer a bar + a = S.Score(n=200, win=0.7, pf=2.0, total=5.0, weeks=8, weeks_pos=8, sign_ok=True) + b = S.Score(n=200, win=0.5, pf=2.0, total=50.0, weeks=8, weeks_pos=8, sign_ok=True) + assert a.admissible and not b.admissible and b.base_ok + assert S.better(a, b) and not S.better(b, a) # admissible beats a larger fallback total + assert S.passes(b, fallback=True) and not S.passes(b, fallback=False) + + +def test_filter_must_beat_random_removal(): + ds = _ds(days=21, per_day=40) + before = np.ones(len(ds.y), bool); after = before & (ds.fwd_pess > 0) # removes exactly the losers + ok, info = S.beats_random_removal(ds, before, after, 7200.0, ds.fwd_pess, ds.day[0]) + assert ok and info["filtered_total"] > info["random_total"] + + +def test_stack_finds_the_planted_edge_and_decides_like_it(monkeypatch): + monkeypatch.setattr(S, "STRATEGIES", {"ev": {**S.STRATEGIES["ev"], "holds": (30, 120)}}) # two holds: the scan runs, the test stays quick + monkeypatch.setattr(S, "GROUPS", {}) + import fly_trader.train.strategies as mod + monkeypatch.setattr(mod, "LEGACY_COLS", ["imb_5m", "ret_1m", "age_known", "log_age_h"]) + ds = _ds(days=100, per_day=400) # enough weeks per half for the walk-forward + st = S.fit_stack(ds) + names = [c["name"] for c in st.components] + assert "strategy: ev" in names and "dump veto" in names and any(n.startswith("meta-label") for n in names) + assert not any("gates" in n for n in names) # removed 2026-09-20: they did not survive the holdout + ev = next(c for c in st.components if c["name"] == "strategy: ev") + assert ev["passed"] and ev["selection"]["win"] >= S.WIN_MIN and ev["evaluation"]["win"] >= S.WIN_MIN + assert st.deployable and not st.fallback + models = S.final_models(ds, st) + X = ds.X[:200]; d = S.decide(models, X, ds.cols, float(ds.ts[0])) + good = X[:, ds.cols.index("imb_5m")] > 1.2 + assert (d["strategy"][good] == "ev").mean() > 0.8 and (d["strategy"][~good] == None).mean() > 0.8 # noqa: E711 + assert set(np.unique(d["reason"])) <= {"trade", "no strategy fires", "win filter (meta-label)"} | {r for r in d["reason"] if "veto" in r or "gated" in r} + + +def test_a_fit_that_clears_nothing_records_what_it_reached(monkeypatch): + """A dropped component must say how close it came: the most profitable setting tried, whether or not it passed.""" + monkeypatch.setattr(S, "STRATEGIES", {"ev": {**S.STRATEGIES["ev"], "holds": (120,)}}) + monkeypatch.setattr(S, "PF_MIN", 99.0) # nothing can clear the profit factor + ds = _ds(days=60, per_day=200) # past the walk-forward warmup, so settings really are scored + uni = np.ones(len(ds.y), bool); sel = (ds.day >= ds.days[25]) & (ds.day < ds.days[45]) # inside the scored region, past the warmup + f = S.fit_strategy(ds, "ev", ["imb_5m", "ret_1m", "age_known", "log_age_h"], uni, sel, ds.day[0]) + assert f.selection == {} and f.trials > 0 and "no setting met" in f.reason + assert f.best_seen and f.best_seen["n"] > 0 and f.best_seen["win"] is not None # the numbers survive the failure + assert f.best_seen["total"] != 0.0 # a setting that traded, not the empty pick + + +def test_empty_inputs_are_skipped_and_groups_are_judged_at_one_hold(monkeypatch): + """Walk-forwards are the expensive part: an input group with no signal costs none, and the group comparison runs at a + single hold, while each strategy still fits its hold from the whole grid.""" + monkeypatch.setattr(S, "STRATEGIES", {"ev": {**S.STRATEGIES["ev"], "holds": (30, 120)}}) + monkeypatch.setattr(S, "GROUPS", {"empty": ["age_known"], "market": ["mkt_vol_1h"]}) # age_known never varies here + import fly_trader.train.strategies as mod + monkeypatch.setattr(mod, "LEGACY_COLS", ["imb_5m", "ret_1m", "log_age_h"]) + monkeypatch.setattr(S, "HOLDOUT_DAYS", 0) # this test is about the group trials, not the holdout: keep its full day budget + calls = []; real = S.fit_strategy + monkeypatch.setattr(S, "fit_strategy", lambda *a, **k: (calls.append((a[1], k.get("holds"))), real(*a, **k))[1]) + st = S.fit_stack(_ds(days=100, per_day=150)) + empty = next(c for c in st.components if c["name"] == "inputs: empty") + assert not empty["passed"] and "empty in this corpus" in empty["reason"] and empty["trials"] == 0 + n_inputs = sum(1 for c in st.components if c["name"].startswith("inputs:") and c["trials"]) + assert all(h == (S.EV_HOLD,) for _, h in calls[:1 + n_inputs]) # the baseline and every group trial: one hold + assert any(h is None for _, h in calls[1 + n_inputs:]) # the strategies themselves scan the whole grid + + +def test_walk_forward_scores_are_cached_and_never_partial(): + """A killed fit must resume from disk, and a fit that was interrupted must leave nothing behind to resume from.""" + ds = _ds(days=20, per_day=40); calls = [] + + def compute(): + calls.append(1); return np.arange(len(ds.y), dtype=np.float32) + v1 = S._cached(ds, ["a"], "ev|hold120", None, compute) + v2 = S._cached(ds, ["a"], "ev|hold120", None, compute) + assert len(calls) == 1 and np.array_equal(v1, v2) # the second fit reuses the file + ev = threading.Event(); ev.set() + S._cached(ds, ["a"], "ev|hold30", ev, compute) + assert len(calls) == 2 and not S._cache_path(ds, ["a"], "ev|hold30").exists() # interrupted: nothing written + assert S._cache_path(ds, ["a"], "ev|hold120") != S._cache_path(ds, ["b"], "ev|hold120") # the inputs are part of the key + + +def test_random_removal_scores_each_book_with_its_own_holds(): + """The filtered book carries no hold for a row the filter removed. Passing those zeros for the unfiltered book made + taken_idx re-enter the same minute forever (three training runs died this way) and scored the dropped trades as 0.""" + ds = _ds(days=30, per_day=60) + before = np.zeros(len(ds.y), bool); before[:400] = True + after = before.copy(); after[200:400] = False + hold_after = np.where(after, 1800.0, 0.0); ret_after = np.where(after, ds.fwd_pess, np.nan) + hold_before = np.where(before, 1800.0, 0.0); ret_before = np.where(before, ds.fwd_pess, np.nan) + with pytest.raises(ValueError, match="hold of zero"): + S.beats_random_removal(ds, before, after, hold_after, ret_after, ds.day[0]) + ok, info = S.beats_random_removal(ds, before, after, hold_after, ret_after, ds.day[0], + hold_before=hold_before, returns_before=ret_before) + assert isinstance(ok, bool) and info["removed"] > 0 + wrong = S.beats_random_removal(ds, before, after, hold_after, ret_after, ds.day[0], # what the old call effectively did: + hold_before=hold_before, returns_before=np.nan_to_num(ret_after))[1] # removed trades scored as zero + assert info["random_total"] != pytest.approx(wrong["random_total"], abs=1e-9) + assert info["filtered_total"] == pytest.approx(float(np.nansum(ret_after[S.taken_idx(ds.ts, ds.mint, hold_after, np.flatnonzero(after))])), abs=1e-6) + + +def test_cache_keys_separate_fits_whose_inputs_share_column_names(): + """The wallet-skill candidates all use the same column names but different values: without a tag they would reuse + each other's walk-forward scores and every candidate would look identical.""" + ds = _ds(days=20, per_day=40); cols = ["imb_5m", "ret_1m"] + a = S._cache_path(ds, cols, "ev|hold120|h20-L7") + b = S._cache_path(ds, cols, "ev|hold120|h30-L7") + assert a != b and S._cache_path(ds, cols, "ev|hold120|h20-L7") == a + + +def test_the_holdout_is_never_fitted_on_and_is_scored_once(monkeypatch): + """The last HOLDOUT_DAYS days must not reach any fit; the finished book is scored on them exactly once, for the + record. Anything that fitted against them would make the number worthless.""" + monkeypatch.setattr(S, "STRATEGIES", {"ev": {**S.STRATEGIES["ev"], "holds": (30, 120)}}) + monkeypatch.setattr(S, "GROUPS", {}) + import fly_trader.train.strategies as mod + monkeypatch.setattr(mod, "LEGACY_COLS", ["imb_5m", "ret_1m", "age_known", "log_age_h"]) + monkeypatch.setattr(S, "HOLDOUT_DAYS", 10) + ds = _ds(days=100, per_day=150) + seen = [] + real = S.score_pick + monkeypatch.setattr(S, "score_pick", lambda d_, pick, *a, **k: (seen.append(set(d_.day[pick])), real(d_, pick, *a, **k))[1]) + st = S.fit_stack(ds) + hold = set(ds.days[-10:]) + assert not [i for i, days in enumerate(seen) if days & hold], "no fit may score the holdout" + assert st.holdout_days == list(ds.days[-10:]) + fitted = [] + real_fr = S.fit_regressor + monkeypatch.setattr(S, "fit_regressor", lambda X, y, seed: (fitted.append(len(X)), real_fr(X, y, seed))[1]) + S.final_models(ds, st) # deployment: every day + S.final_models(ds, st, exclude_days=st.holdout_days) # measurement: the holdout left out + assert fitted[-1] < fitted[0], "excluding the holdout must reduce the training rows" + monkeypatch.undo() + h = S.score_holdout(ds, S.final_models(ds, st, exclude_days=st.holdout_days), st.holdout_days) + assert h["n"] > 0 and h["days"] == [str(ds.days[-10]), str(ds.days[-1])] and h.get("random_mean") is not None diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py new file mode 100644 index 0000000000000000000000000000000000000000..8917fc9f550967fa3d2c12256a53751244b7d097 --- /dev/null +++ b/tests/test_supervisor.py @@ -0,0 +1,53 @@ +import threading, time +from fly_trader.ops import supervisor as S + + +def test_supervisor_thread_lifecycle(monkeypatch, db_conn): + ticks = {"n": 0} + + def fake(stop_event=None): + while not stop_event.is_set(): + ticks["n"] += 1 + time.sleep(0.05) + + monkeypatch.setattr(S, "_entry", lambda name: fake) + monkeypatch.setattr(S.Supervisor, "external", lambda self: {}) + sup = S.Supervisor() + assert sup.start("discover", started_by="test") + time.sleep(0.3) + assert sup.alive("discover") and ticks["n"] > 2 + assert not sup.start("discover") # idempotent while alive + assert sup.stop("discover", grace_s=5) + assert not sup.alive("discover") + st = sup.status()["discover"] + assert st["stopped_at"] is not None and st["error"] is None + + +def test_supervisor_records_crash(monkeypatch, db_conn): + def boom(stop_event=None): + raise ValueError("kaput") + monkeypatch.setattr(S, "_entry", lambda name: boom) + monkeypatch.setattr(S.Supervisor, "external", lambda self: {}) + sup = S.Supervisor() + sup.start("runner", started_by="test") + sup.threads["runner"].join(5) + assert "kaput" in sup.status()["runner"]["error"] + + +def test_a_previous_consoles_row_with_our_own_pid_is_stale(monkeypatch, db_conn): + """In a container the console is always pid 1, so the row the last boot left carries this boot's pid. A new console + has registered nothing yet: such a row is stale, and without closing it every restart refused to register.""" + import os + from fly_trader.db.connection import transaction + with transaction() as conn: + conn.execute("UPDATE processes SET stopped_at = now() WHERE stopped_at IS NULL AND cmd[1] = 'thread'") + rid = conn.execute("INSERT INTO processes (name, pid, cmd, log_path, started_by) VALUES ('discover', %s, %s, 'logs/discover.log', 'test') RETURNING id", + (os.getpid(), ["thread", "discover"])).fetchone()["id"] + monkeypatch.setattr(S, "_entry", lambda name: (lambda stop_event=None: stop_event.wait(5))) + monkeypatch.setattr(S.Supervisor, "external", lambda self: {}) + sup = S.Supervisor() + with transaction() as conn: + row = conn.execute("SELECT stopped_at, exit_code FROM processes WHERE id = %s", (rid,)).fetchone() + assert row["stopped_at"] is not None and row["exit_code"] == -1 # closed at construction, not at start + assert sup.start("discover", started_by="test") # and the worker registers again + sup.stop("discover", grace_s=5) diff --git a/tests/test_wallet_skill.py b/tests/test_wallet_skill.py new file mode 100644 index 0000000000000000000000000000000000000000..353a0bb1cc7ad1bdccf27e0cd071a66f45ff5bbb --- /dev/null +++ b/tests/test_wallet_skill.py @@ -0,0 +1,67 @@ +"""Wallet skill: markouts per wallet and hold are the net return of holding (buys) or the move stepped aside from +(sells); the table for day D only uses wallet days up to D − GAP_DAYS (no look-ahead); scores are cut into deciles.""" +from datetime import date, datetime, timedelta, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from fly_trader import config +from fly_trader.train import mature, wallet_skill + +D = date(2026, 9, 5) + + +def _setup(tmp_path, monkeypatch): + monkeypatch.setattr(config, "REPLAY_DIR", tmp_path / "replay"); monkeypatch.setattr(mature, "MATURE_DIR", tmp_path / "mature") + monkeypatch.setattr(mature, "MATURE_FEAT_DIR", tmp_path / "feat") + t0 = datetime(D.year, D.month, D.day, 12, 0, tzinfo=timezone.utc) + (tmp_path / "replay" / D.isoformat()).mkdir(parents=True) + pq.write_table(pa.table({"ts": pa.array([t0 + timedelta(seconds=10), t0 + timedelta(seconds=20)], pa.timestamp("ms", tz="UTC")), "pool": ["pump-amm"] * 2, + "mint": ["Spump"] * 2, "trader": ["BUYER", "SELLER"], "side": pa.array([1, -1], pa.int8()), "sol": [2.0, 1.0], "price": [1.0, 1.0]}), + tmp_path / "replay" / D.isoformat() / "12_trades.parquet") + for d in (D, D + timedelta(days=1)): + mins = [datetime(d.year, d.month, d.day, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=k) for k in range(0, 300)] + close = [1.0 if k < 10 else 1.2 for k in range(300)] + mature.write_part(pa.table({"mint": ["Spump"] * 300, "ts": pa.array(mins, pa.timestamp("us", tz="UTC")), "close": close}), tmp_path / "mature" / f"{d}.parquet", mature.AGG_VERSION) + mature.write_part(pa.table({"mint": ["Spump"] * 300, "ts": pa.array(mins, pa.timestamp("us", tz="UTC")), "exit_cost_0p1": [0.01] * 300}), + tmp_path / "feat" / d.isoformat() / "part.parquet", 3) + + +def test_markouts_and_no_lookahead(tmp_path, monkeypatch): + _setup(tmp_path, monkeypatch) + assert wallet_skill.build_wallet_day(D, out_dir=tmp_path / "wd") > 0 + t = pq.read_table(tmp_path / "wd" / f"{D}.parquet").to_pandas().set_index(["wallet", "h"]) + assert t.loc[("BUYER", 30), "sol_m"] == pytest.approx(2.0 * (1.2 * 0.99 * 0.99 - 1)) # held past the +20 % move, costs both sides + assert t.loc[("BUYER", 10), "sol_m"] == pytest.approx(2.0 * (1.2 * 0.99 * 0.99 - 1)) + assert t.loc[("SELLER", 30), "sol_m"] == pytest.approx(-1.0 * 0.2) # sold before a +20 % move + assert wallet_skill.window_days(D + timedelta(days=wallet_skill.GAP_DAYS), 3) == [D - timedelta(days=2), D - timedelta(days=1), D] + assert all(x <= D for x in wallet_skill.window_days(D + timedelta(days=wallet_skill.GAP_DAYS), 30)) + assert wallet_skill.skill_table(D + timedelta(days=wallet_skill.GAP_DAYS - 1), 30, 1, day_dir=tmp_path / "wd") is None # day D is not yet usable + tab = wallet_skill.skill_table(D + timedelta(days=wallet_skill.GAP_DAYS), 30, 1, day_dir=tmp_path / "wd").to_pandas().set_index("wallet") + assert tab.loc["BUYER", "bucket"] > tab.loc["SELLER", "bucket"] + + +def test_decile_buckets_cover_0_to_9(): + import numpy as np, pandas as pd + rng = np.random.default_rng(0); n = 1000 + wd = pd.DataFrame({"wallet": [f"w{i}" for i in range(n)], "h": 60, "n": 1, "sol": rng.uniform(0.1, 5, n), "sol_m": rng.normal(0, 1, n)}) + import tempfile, pathlib + with tempfile.TemporaryDirectory() as td: + pq.write_table(pa.Table.from_pandas(wd, preserve_index=False), pathlib.Path(td) / f"{D}.parquet") + tab = wallet_skill.skill_table(D + timedelta(days=wallet_skill.GAP_DAYS), 60, 1, day_dir=pathlib.Path(td)).to_pandas() + counts = tab["bucket"].value_counts() + assert sorted(counts.index) == list(range(10)) and counts.min() == counts.max() == n // 10 + + +def test_daily_writes_each_table_once_its_window_exists(tmp_path, monkeypatch): + _setup(tmp_path, monkeypatch) + monkeypatch.setattr(wallet_skill, "DAY_DIR", tmp_path / "wd"); monkeypatch.setattr(mature, "SKILL_DIR", tmp_path / "skill") + monkeypatch.setattr(wallet_skill, "CONFIG_FILE", tmp_path / "skill" / "config.json") + assert wallet_skill.daily() == 1 and not list((tmp_path / "skill").glob("*.parquet")) # no (h, L) frozen: the wallet day only + wallet_skill.freeze(30, 1) + f = tmp_path / "skill" / f"{D + timedelta(days=wallet_skill.GAP_DAYS)}.parquet" + assert wallet_skill.daily() == 1 and [p.name for p in (tmp_path / "skill").glob("*.parquet")] == [f.name] # only the day whose window exists + assert wallet_skill.table_version(f) == "h30-L1" and wallet_skill.daily() == 0 + wallet_skill.freeze(10, 1) + assert wallet_skill.daily() == 1 and wallet_skill.table_version(f) == "h10-L1" # a new (h, L) rewrites it diff --git a/tools/make_seed.py b/tools/make_seed.py new file mode 100644 index 0000000000000000000000000000000000000000..4fe4d75f53fe62d5afb1872dcc8d6813e9df57eb --- /dev/null +++ b/tools/make_seed.py @@ -0,0 +1,100 @@ +"""Package what a fresh install needs to trade: the deployed selector and fly, the connectome they run on, and the +database rows that admit them -- written for the container's layout (``/app/data``). + +Run on the machine that trained them (a master checkout with its database), after a passed replay: + + .venv/bin/python tools/make_seed.py --out dist + +Outputs, under ``--out``: + models/policies/.pt, models/selectors/.joblib, models/connectome/{current.txt, , scale.json} + seed/seed.sql -- brain_snapshots rows (ids kept, so the pin still points at its selector), ui_settings: the replay + verdict the fly session requires, the selector pin, and autostart for the two trading workers. + Idempotent: every statement is ON CONFLICT DO NOTHING, so a running install is never overwritten. + +The distribution branch copies these into its tracked ``models/`` and ``seed/`` -- that is the whole "refresh" step. +""" +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # runnable as a script from any directory + +from fly_trader import config # noqa: E402 +from fly_trader.agent.selector_session import pinned_snapshot +from fly_trader.db.connection import transaction +from fly_trader.train import fly_selector, selector + +CONTAINER_DATA = "/app/data" +AUTOSTART = {"pumpstream": True, "runner": True, "train": False, "replay": False, "discover": False} + + +def _lit(v) -> str: + """A SQL literal: strings quoted, JSON for dicts, NULL for None.""" + if v is None: + return "NULL" + if isinstance(v, dict): + v = json.dumps(v, default=str) + return "'" + str(v).replace("'", "''") + "'" + + +def main(out: Path) -> None: + with transaction() as conn: + fly = fly_selector.latest_deployable(conn) + if not fly: + raise SystemExit("no deployable fly: bootstrap one first (fly-trader train-fly-selector) and run the replay") + pin = pinned_snapshot() + sel_id = pin if pin is not None else (selector.latest_current(conn) or {}).get("id") + if sel_id is None: + raise SystemExit("no deployed selector: pin one (ui_settings.pinned_selector_snapshot) or train one that qualifies") + rows = {r["id"]: dict(r) for r in conn.execute("SELECT * FROM brain_snapshots WHERE id = ANY(%s)", ([int(fly["id"]), int(sel_id)],)).fetchall()} + verdict = conn.execute("SELECT value FROM ui_settings WHERE key = 'fly_replay'").fetchone() + if not verdict: + raise SystemExit("no replay verdict stored: the fly session refuses to trade without one (fly-trader fly-replay)") + v = verdict["value"] if isinstance(verdict["value"], dict) else json.loads(verdict["value"]) + if not v.get("passed") or v.get("data") != fly_selector.FLY_VERSION: + raise SystemExit("the stored replay verdict did not pass on the current definitions: run fly-trader fly-replay again") + + models = out / "models"; (models / "policies").mkdir(parents=True, exist_ok=True); (models / "selectors").mkdir(exist_ok=True); (models / "connectome").mkdir(exist_ok=True) + stmts = ["-- generated by tools/make_seed.py: the rows a fresh install needs to trade. Safe to re-run: nothing here overwrites."] + for sid, r in sorted(rows.items()): + src = Path(r["path"]); sub = "policies" if r["kind"] == "fly_selector" else "selectors" + shutil.copy2(src, models / sub / src.name) + r["path"] = f"{CONTAINER_DATA}/brain/{sub}/{src.name}" + cols = ("id", "ts", "run_id", "beat_id", "path", "sha256", "kind", "promoted_at", "promoted_by", "note") + stmts.append(f"INSERT INTO brain_snapshots ({', '.join(cols)}) VALUES ({', '.join(_lit(r[c]) for c in cols)}) ON CONFLICT (id) DO NOTHING;") + stmts.append("SELECT setval('brain_snapshots_id_seq', GREATEST((SELECT max(id) FROM brain_snapshots), 1));") + settings = {"fly_replay": v, "autostart": AUTOSTART, + "pinned_selector_snapshot": {"id": int(sel_id), "why": "shipped with the distribution: the selector this fly was taught by", "pinned_at": str(fly["meta"].get("S", ""))[:10]}} + for k, val in settings.items(): + stmts.append(f"INSERT INTO ui_settings (key, value) VALUES ({_lit(k)}, {_lit(val)}::jsonb) ON CONFLICT (key) DO NOTHING;") + (out / "seed").mkdir(exist_ok=True) + (out / "seed" / "seed.sql").write_text("\n".join(stmts) + "\n") + + cdir = config.BRAIN_DIR / "connectome"; current = (cdir / "current.txt").read_text().strip() + for name in ("current.txt", "scale.json", current): + shutil.copy2(cdir / name, models / "connectome" / name) + # The fly's inputs include wallet skill: config.json names the fitted (h, L) that FLY_VERSION carries, and the day's + # table maps wallets to skill deciles. The newest table ships (~300 MB: too big for GitHub, so the distribution keeps + # it on Hugging Face and the container fetches it once); the engine falls back to it while it goes stale. + from fly_trader.train.mature import SKILL_DIR + tables = sorted(SKILL_DIR.glob("????-??-??.parquet")) + if not (SKILL_DIR / "config.json").exists() or not tables: + raise SystemExit("no wallet-skill config or tables: the fly's inputs need them (fly-trader fit-wallet-skill)") + sk = models / "wallet_skill"; sk.mkdir(exist_ok=True) + for name in ("config.json", "fit.json"): + if (SKILL_DIR / name).exists(): + shutil.copy2(SKILL_DIR / name, sk / name) + shutil.copy2(tables[-1], sk / tables[-1].name); (sk / "TABLE").write_text(tables[-1].name + "\n") + print(f"seed written to {out}/: fly #{fly['id']} ({Path(rows[int(fly['id'])]['path']).name}), selector #{sel_id}, connectome {current}, skill table {tables[-1].name}") + print(f" fly taught by: {fly['meta'].get('teacher')}") + print(f" replay verdict: {v.get('reason')}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--out", type=Path, default=Path("dist"), help="output directory (default: dist/)") + main(ap.parse_args().out)