| """Replication of arXiv 2608.26174 -- sentiment + on-chain + market, daily BTC. |
| |
| That paper is the strongest sentiment-inclusive result I could find that also |
| evaluates honestly: strict expanding walk-forward, a stated baseline set, and |
| transaction costs in its profit function. It reports ROC-AUC 0.6316 for its |
| "ImprovedTCN", against TCN 0.5692, ImprovedTCN_GRU 0.5314, XGBoost 0.5011, |
| LSTM 0.4586 and Random Forest 0.3627. |
| |
| The reason its AUC beats anything achievable on plain hourly direction is its |
| target: not "will price go up" but "will price rise more than 5% within the next |
| 7 days" -- a large-move question, with a positive class near a quarter of days. |
| Magnitude is forecastable in a way that sign is not, which is the same reason |
| the volatility-regime model here works while the direction models do not. |
| |
| Data, and where it differs from the paper: |
| |
| market Binance BTCUSDT hourly, resampled to daily (as paper) |
| sentiment Crypto Fear & Greed Index, alternative.me (as paper) |
| on-chain CoinMetrics community: MVRV, hashrate, active |
| addresses, tx count, supply, market cap. Realized |
| cap is derived as marketcap / MVRV and NUPL as |
| 1 - 1/MVRV, both exact identities. (as paper) |
| MISSING Google Trends. The paper uses it; pytrends is rate |
| limited and unreproducible, so it is omitted and |
| the gap is reported rather than papered over. |
| |
| Protocol follows the paper exactly: TimeSeriesSplit(n_splits=5, test_size=100), |
| expanding training window, no look-ahead. Every rolling statistic is causal. |
| |
| The ablation is the part the paper does not run and the thesis needs: the same |
| models on market-only, market+on-chain, and market+on-chain+sentiment, so the |
| contribution of sentiment is measured rather than assumed. |
| |
| Usage: |
| python scripts/run_sentiment_replication.py |
| python scripts/run_sentiment_replication.py --threshold 0.05 --horizon 7 |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| import time |
| import warnings |
| from pathlib import Path |
|
|
| import numpy as np |
| import pandas as pd |
| import requests |
| import torch |
| import torch.nn as nn |
| from sklearn.ensemble import RandomForestClassifier |
| from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score |
| from sklearn.model_selection import TimeSeriesSplit |
| from sklearn.preprocessing import StandardScaler |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT)) |
| warnings.filterwarnings("ignore") |
|
|
| from lightgbm import LGBMClassifier |
| from xgboost import XGBClassifier |
| from scripts.run_direction_sota_experiment import load_candles |
|
|
| CACHE = ROOT / "outputs" / "sentiment_replication" |
| CM_METRICS = ["CapMVRVCur", "HashRate", "AdrActCnt", "TxCnt", "SplyCur", "CapMrktCurUSD"] |
| SEED = 42 |
|
|
|
|
| |
| |
| |
| def fear_greed() -> pd.DataFrame: |
| """Daily Crypto Fear & Greed Index, 2018-02-01 to now.""" |
| cache = CACHE / "fear_greed.csv" |
| if cache.exists(): |
| return pd.read_csv(cache, parse_dates=["date"]) |
| payload = requests.get("https://api.alternative.me/fng/", |
| params={"limit": 0, "format": "json"}, timeout=60).json()["data"] |
| df = pd.DataFrame({ |
| "date": pd.to_datetime([int(r["timestamp"]) for r in payload], unit="s", utc=True).normalize(), |
| "fear_greed": [float(r["value"]) for r in payload], |
| }).sort_values("date").drop_duplicates("date").reset_index(drop=True) |
| cache.parent.mkdir(parents=True, exist_ok=True) |
| df.to_csv(cache, index=False) |
| return df |
|
|
|
|
| def coinmetrics() -> pd.DataFrame: |
| """Free daily on-chain metrics, paged through the community endpoint.""" |
| cache = CACHE / "onchain.csv" |
| if cache.exists(): |
| return pd.read_csv(cache, parse_dates=["date"]) |
| rows: list = [] |
| token = None |
| while True: |
| params = {"assets": "btc", "metrics": ",".join(CM_METRICS), "frequency": "1d", |
| "start_time": "2018-01-01", "page_size": 10000} |
| if token: |
| params["next_page_token"] = token |
| body = requests.get("https://community-api.coinmetrics.io/v4/timeseries/asset-metrics", |
| params=params, timeout=60).json() |
| rows += body.get("data", []) |
| token = body.get("next_page_token") |
| if not token: |
| break |
| time.sleep(0.2) |
| df = pd.DataFrame(rows) |
| df["date"] = pd.to_datetime(df["time"], utc=True).dt.normalize() |
| for col in CM_METRICS: |
| df[col] = pd.to_numeric(df.get(col), errors="coerce") |
| df = df[["date"] + CM_METRICS].sort_values("date").drop_duplicates("date").reset_index(drop=True) |
| cache.parent.mkdir(parents=True, exist_ok=True) |
| df.to_csv(cache, index=False) |
| return df |
|
|
|
|
| def daily_market() -> pd.DataFrame: |
| candles = load_candles("2018-01-01", ROOT / "outputs" / "volatility" / "btc_1h.csv") |
| agg = {"open": "first", "high": "max", "low": "min", "close": "last", |
| "volume": "sum", "quote_volume": "sum", "num_trades": "sum"} |
| have = {k: v for k, v in agg.items() if k in candles.columns} |
| daily = candles.set_index("timestamp").resample("1D").agg(have).dropna().reset_index() |
| daily["date"] = daily["timestamp"].dt.normalize() |
| return daily.drop(columns=["timestamp"]) |
|
|
|
|
| |
| |
| |
| def build(threshold: float, horizon: int) -> tuple[pd.DataFrame, dict]: |
| df = (daily_market() |
| .merge(coinmetrics(), on="date", how="left") |
| .merge(fear_greed(), on="date", how="left") |
| .sort_values("date").reset_index(drop=True)) |
| |
| |
| df[CM_METRICS + ["fear_greed"]] = df[CM_METRICS + ["fear_greed"]].ffill() |
|
|
| eps = 1e-12 |
| x = df.copy() |
| x["log_return"] = np.log(x["close"].clip(lower=eps)).diff() |
| for w in (7, 14, 30): |
| x[f"ret_{w}d"] = np.log(x["close"].clip(lower=eps)).diff(w) |
| x[f"vol_{w}d"] = x["log_return"].rolling(w).std() |
| x[f"close_sma{w}_log"] = np.log(x["close"].clip(lower=eps) |
| / x["close"].rolling(w).mean().clip(lower=eps)) |
| x["range_pct"] = (x["high"] - x["low"]) / x["close"].clip(lower=eps) |
| x["volume_logchg"] = np.log1p(x["volume"].clip(lower=0)).diff() |
| x["trades_logchg"] = np.log1p(x["num_trades"].clip(lower=0)).diff() |
| delta = x["close"].diff() |
| up = delta.clip(lower=0).rolling(14).mean() |
| down = (-delta.clip(upper=0)).rolling(14).mean() |
| x["rsi_14"] = 100 - 100 / (1 + up / down.replace(0, np.nan)) |
|
|
| |
| x["mvrv"] = x["CapMVRVCur"] |
| x["nupl"] = 1.0 - 1.0 / x["CapMVRVCur"].replace(0, np.nan) |
| x["realised_cap"] = x["CapMrktCurUSD"] / x["CapMVRVCur"].replace(0, np.nan) |
| x["mvrv_chg_7d"] = x["mvrv"].diff(7) |
| x["hashrate_logchg_7d"] = np.log1p(x["HashRate"].clip(lower=0)).diff(7) |
| x["active_addr_logchg_7d"] = np.log1p(x["AdrActCnt"].clip(lower=0)).diff(7) |
| x["txcnt_logchg_7d"] = np.log1p(x["TxCnt"].clip(lower=0)).diff(7) |
| x["mcap_real_log"] = np.log(x["CapMrktCurUSD"].clip(lower=eps) |
| / x["realised_cap"].clip(lower=eps)) |
|
|
| |
| x["fg"] = x["fear_greed"] |
| x["fg_chg_1d"] = x["fg"].diff() |
| x["fg_chg_7d"] = x["fg"].diff(7) |
| x["fg_sma7"] = x["fg"].rolling(7).mean() |
| x["fg_extreme_fear"] = (x["fg"] <= 25).astype(float) |
| x["fg_extreme_greed"] = (x["fg"] >= 75).astype(float) |
| roll = x["fg"].rolling(90) |
| x["fg_z90"] = (x["fg"] - roll.mean()) / roll.std().replace(0, np.nan) |
|
|
| |
| |
| fwd = x["close"].shift(-horizon) / x["close"] - 1.0 |
| x["target"] = (fwd > threshold).astype(float) |
| x.loc[fwd.isna(), "target"] = np.nan |
|
|
| groups = { |
| "market": ["log_return", "range_pct", "volume_logchg", "trades_logchg", "rsi_14"] |
| + [f"{p}_{w}d" for w in (7, 14, 30) for p in ("ret", "vol")] |
| + [f"close_sma{w}_log" for w in (7, 14, 30)], |
| "onchain": ["mvrv", "nupl", "mvrv_chg_7d", "hashrate_logchg_7d", |
| "active_addr_logchg_7d", "txcnt_logchg_7d", "mcap_real_log"], |
| "sentiment": ["fg", "fg_chg_1d", "fg_chg_7d", "fg_sma7", "fg_z90", |
| "fg_extreme_fear", "fg_extreme_greed"], |
| } |
| return x, groups |
|
|
|
|
| |
| |
| |
| class TCN(nn.Module): |
| """Dilated causal convolutions over a window of days, as in the paper.""" |
|
|
| def __init__(self, n_features: int, channels: int = 32, levels: int = 3, dropout: float = 0.2): |
| super().__init__() |
| layers: list[nn.Module] = [] |
| in_ch = n_features |
| for level in range(levels): |
| dilation = 2 ** level |
| layers += [ |
| nn.Conv1d(in_ch, channels, kernel_size=3, padding=2 * dilation, dilation=dilation), |
| nn.ReLU(), nn.BatchNorm1d(channels), nn.Dropout(dropout)] |
| in_ch = channels |
| self.body = nn.Sequential(*layers) |
| self.head = nn.Linear(channels, 1) |
| self.levels = levels |
|
|
| def forward(self, x): |
| h = self.body(x.transpose(1, 2)) |
| h = h[:, :, : x.size(1)] |
| return self.head(h[:, :, -1]).squeeze(-1) |
|
|
|
|
| def windows(values: np.ndarray, length: int) -> np.ndarray: |
| """Sliding windows ending at each row, left-padded by edge repetition.""" |
| pad = np.repeat(values[:1], length - 1, axis=0) |
| padded = np.vstack([pad, values]) |
| return np.stack([padded[i: i + length] for i in range(len(values))]) |
|
|
|
|
| def fit_tcn(X_tr, y_tr, X_te, seq_len: int = 14, epochs: int = 60) -> np.ndarray: |
| torch.manual_seed(SEED) |
| model = TCN(X_tr.shape[1]) |
| opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4) |
| pos = max(y_tr.sum(), 1) |
| loss_fn = nn.BCEWithLogitsLoss( |
| pos_weight=torch.tensor([(len(y_tr) - pos) / pos], dtype=torch.float32)) |
| Xw = torch.tensor(windows(X_tr, seq_len), dtype=torch.float32) |
| yt = torch.tensor(y_tr, dtype=torch.float32) |
| model.train() |
| for _ in range(epochs): |
| opt.zero_grad() |
| loss = loss_fn(model(Xw), yt) |
| loss.backward() |
| opt.step() |
| model.eval() |
| with torch.no_grad(): |
| return torch.sigmoid(model(torch.tensor(windows(X_te, seq_len), |
| dtype=torch.float32))).numpy().ravel() |
|
|
|
|
| |
| def models(): |
| return { |
| "LightGBM": LGBMClassifier(n_estimators=300, learning_rate=0.03, num_leaves=15, |
| min_child_samples=30, subsample=0.8, subsample_freq=1, |
| colsample_bytree=0.7, reg_lambda=1.0, |
| random_state=SEED, verbose=-1), |
| "XGBoost": XGBClassifier(n_estimators=300, learning_rate=0.03, max_depth=4, |
| subsample=0.8, colsample_bytree=0.7, reg_lambda=1.0, |
| random_state=SEED, eval_metric="logloss"), |
| "RandomForest": RandomForestClassifier(n_estimators=400, min_samples_leaf=10, |
| random_state=SEED, n_jobs=-1), |
| "TCN": "tcn", |
| } |
|
|
|
|
| def run(frame: pd.DataFrame, feats: list[str], folds: int, test_size: int, |
| paper_threshold: float) -> list[dict]: |
| ok = frame[feats + ["target"]].notna().all(axis=1) & np.isfinite( |
| frame[feats].to_numpy(dtype=float)).all(axis=1) |
| data = frame[ok].reset_index(drop=True) |
| X = data[feats].to_numpy(dtype=float) |
| y = data["target"].to_numpy().astype(int) |
|
|
| out = [] |
| for fold, (tr, te) in enumerate(TimeSeriesSplit(n_splits=folds, test_size=test_size) |
| .split(X), start=1): |
| scaler = StandardScaler().fit(X[tr]) |
| X_tr, X_te = scaler.transform(X[tr]), scaler.transform(X[te]) |
| if len(set(y[tr])) < 2 or len(set(y[te])) < 2: |
| continue |
| for name, model in models().items(): |
| p = (fit_tcn(X_tr, y[tr], X_te) if model == "tcn" |
| else model.fit(X_tr, y[tr]).predict_proba(X_te)[:, 1]) |
| for tag, thr in (("0.5", 0.5), ("paper", paper_threshold)): |
| pred = (p >= thr).astype(int) |
| out.append(dict( |
| fold=fold, model=name, thr_tag=tag, n_test=len(te), |
| start=str(data["date"].iloc[te[0]])[:10], end=str(data["date"].iloc[te[-1]])[:10], |
| pos_rate=float(y[te].mean()), |
| auc=roc_auc_score(y[te], p), |
| precision=precision_score(y[te], pred, zero_division=0), |
| recall=recall_score(y[te], pred, zero_division=0), |
| f1=f1_score(y[te], pred, zero_division=0), |
| pred_rate=float(pred.mean()))) |
| return out |
|
|
|
|
| PAPER = {"ImprovedTCN": 0.6316, "TCN": 0.5692, "ImprovedTCN_GRU": 0.5314, |
| "XGBoost": 0.5011, "LSTM": 0.4586, "RandomForest": 0.3627} |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--threshold", type=float, default=0.05, help="forward return threshold") |
| ap.add_argument("--horizon", type=int, default=7, help="forward days") |
| ap.add_argument("--folds", type=int, default=5) |
| ap.add_argument("--test-size", type=int, default=100) |
| ap.add_argument("--paper-threshold", type=float, default=0.11) |
| args = ap.parse_args() |
|
|
| frame, groups = build(args.threshold, args.horizon) |
| sets = { |
| "market only": groups["market"], |
| "market + onchain": groups["market"] + groups["onchain"], |
| "market + onchain + SENTIMENT": groups["market"] + groups["onchain"] + groups["sentiment"], |
| } |
| valid = frame["target"].notna() |
| print(f"daily rows {int(valid.sum()):,} " |
| f"{str(frame.loc[valid, 'date'].min())[:10]} to {str(frame.loc[valid, 'date'].max())[:10]}") |
| print(f"target: forward {args.horizon}d return > {args.threshold:.0%} " |
| f"positive rate {frame.loc[valid, 'target'].mean():.1%} (paper: 26%)") |
| print(f"fear & greed coverage: {frame['fg'].notna().mean():.1%} of rows") |
| print(f"protocol: TimeSeriesSplit(n_splits={args.folds}, test_size={args.test_size}) " |
| f"= {args.folds * args.test_size} test days\n") |
|
|
| rows = [] |
| for label, feats in sets.items(): |
| print(f"running [{label}] {len(feats)} features ...") |
| for r in run(frame, feats, args.folds, args.test_size, args.paper_threshold): |
| r["feature_set"] = label |
| rows.append(r) |
|
|
| df = pd.DataFrame(rows) |
| print("\n" + "=" * 100) |
| print(f"ROC-AUC by feature set and model (mean of {args.folds} folds)") |
| print(" paper's reported AUCs: " + ", ".join(f"{k} {v:.4f}" for k, v in PAPER.items())) |
| print("=" * 100) |
| auc = (df[df.thr_tag == "0.5"].groupby(["feature_set", "model"])["auc"] |
| .agg(["mean", "std", "count"]).reset_index()) |
| print(f"{'feature set':<32}{'model':<14}{'AUC':>8}{'sd':>8}{'vs paper TCN':>14}") |
| for _, r in auc.iterrows(): |
| ref = PAPER.get(r["model"], PAPER["ImprovedTCN"]) |
| print(f"{r['feature_set']:<32}{r['model']:<14}{r['mean']:>8.4f}{r['std']:>8.4f}" |
| f"{r['mean'] - ref:>+14.4f}") |
|
|
| print("\n" + "-" * 100) |
| print("SENTIMENT ABLATION -- does adding Fear & Greed change the AUC?") |
| print("-" * 100) |
| base = auc[auc.feature_set == "market + onchain"].set_index("model")["mean"] |
| full = auc[auc.feature_set == "market + onchain + SENTIMENT"].set_index("model")["mean"] |
| for model in full.index: |
| if model in base.index: |
| d = full[model] - base[model] |
| print(f" {model:<14} {base[model]:.4f} -> {full[model]:.4f} {d:+.4f}" |
| f" {'helps' if d > 0.01 else 'hurts' if d < -0.01 else 'no effect'}") |
|
|
| print("\n" + "-" * 100) |
| print(f"At the paper's operating threshold ({args.paper_threshold}) -- " |
| "paper reports precision 19.3%, recall 100.0%, F1 0.3234") |
| print("-" * 100) |
| op = (df[(df.thr_tag == "paper") & (df.feature_set == "market + onchain + SENTIMENT")] |
| .groupby("model")[["precision", "recall", "f1", "pred_rate", "pos_rate"]].mean()) |
| print(f"{'model':<14}{'precision':>11}{'recall':>9}{'F1':>8}{'says yes':>10}{'base rate':>11}") |
| for model, r in op.iterrows(): |
| print(f"{model:<14}{r['precision'] * 100:>10.1f}%{r['recall'] * 100:>8.1f}%" |
| f"{r['f1']:>8.4f}{r['pred_rate'] * 100:>9.1f}%{r['pos_rate'] * 100:>10.1f}%") |
|
|
| out = ROOT / "outputs" / "sentiment_replication.json" |
| out.write_text(json.dumps(rows, indent=2, default=str), encoding="utf-8") |
| print(f"\nwritten: {out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
