Download scripts/finetune_embeddings.py from cryptorugmunch/rugmuncher-backend: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/scripts/finetune_embeddings.py
- Command line
-
hf download hf://cryptorugmunch/rugmuncher-backend/scripts/finetune_embeddings.py
-
curl -L -o finetune_embeddings.py https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/scripts/finetune_embeddings.py
10.5 kB
| #!/usr/bin/env python3 | |
| """ | |
| Domain Fine-Tuned Embeddings β MarginMSE with hard negative mining. | |
| Trains scam-specific embeddings that outperform general-purpose models | |
| for crypto security tasks. Uses cross-encoder as teacher, hard negative | |
| mining from retrieval errors, MarginMSE loss for optimization. | |
| Output: Fine-tuned sentence-transformer model saved to /app/data/models/ | |
| Usage: | |
| python3 scripts/finetune_embeddings.py --epochs 3 --batch-size 16 | |
| Requirements: | |
| - Redis with existing RAG documents (for hard negative mining) | |
| - Cross-encoder model (for teacher scores) | |
| - Training data: scam queries + positive/negative document pairs | |
| """ | |
| import argparse | |
| import asyncio | |
| import logging | |
| import os | |
| import random | |
| import sys | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| # ββ Training data: query β [positive_doc, negative_docs...] βββββ | |
| SCAM_QUERIES = [ | |
| ( | |
| "What are rug pull indicators?", | |
| [ | |
| "Rug pulls involve liquidity removal, disabled transfers, and anonymous teams. Look for: large LP withdrawals, deployer selling tokens, and social media going dark.", | |
| ], | |
| ), | |
| ( | |
| "How to detect honeypot tokens?", | |
| [ | |
| "Honeypot tokens prevent selling. Check for: transfer restrictions, max transaction limits, and sell taxes above 50%. Use honeypot.is or tokensniffer for automated checks.", | |
| ], | |
| ), | |
| ( | |
| "What is a flash loan attack?", | |
| [ | |
| "Flash loan attacks exploit uncollateralized lending. Attackers borrow large amounts, manipulate oracle prices, drain funds, and repay within one transaction. Crema Finance lost $8.8M to this.", | |
| ], | |
| ), | |
| ( | |
| "Phishing scam detection", | |
| [ | |
| "Phishing scams use fake websites mimicking DEXs. Signs: slightly different URLs, requests for seed phrases, fake airdrop claims. Always verify contract addresses on official sources.", | |
| ], | |
| ), | |
| ( | |
| "Wash trading identification", | |
| [ | |
| "Wash trading creates fake volume. Indicators: circular token flows, self-trading between linked wallets, volume spikes without holder growth, and near-identical buy/sell timestamps.", | |
| ], | |
| ), | |
| ( | |
| "Pump and dump patterns", | |
| [ | |
| "Pump and dump groups coordinate buying to inflate prices, then dump on new buyers. Look for: Telegram/Discord 'signal' groups, rapid price spikes followed by 90%+ drops, and concentrated holder distribution.", | |
| ], | |
| ), | |
| ( | |
| "MEV sandwich attack explained", | |
| [ | |
| "Sandwich attacks insert transactions before and after a victim's trade. The attacker buys before the victim (raising price) and sells after (profiting from slippage). Common on Uniswap, PancakeSwap.", | |
| ], | |
| ), | |
| ( | |
| "Smart contract vulnerability patterns", | |
| [ | |
| "Common vulnerabilities: reentrancy (DAO hack), integer overflow, unchecked external calls, tx.origin authentication, and delegatecall to untrusted contracts. Use Slither or Mythril for analysis.", | |
| ], | |
| ), | |
| ( | |
| "Token impersonation scams", | |
| [ | |
| "Impersonation tokens copy legitimate token names, symbols, and logos. They trick users into buying the wrong token. Check: contract address matches official sources, token age, and holder count vs. legitimate token.", | |
| ], | |
| ), | |
| ( | |
| "Liquidity pool manipulation", | |
| [ | |
| "LP manipulation involves adding/removing liquidity to manipulate price. Signs: sudden LP changes, imbalanced pools, and flash loan-funded attacks. Monitor LP health ratio.", | |
| ], | |
| ), | |
| ] | |
| # Hard negative candidates (similar-sounding but different concepts) | |
| HARD_NEGATIVES = [ | |
| "Legitimate token launches with locked liquidity and audited contracts are safe.", | |
| "Airdrops to active community members are a normal marketing strategy.", | |
| "Liquidity mining programs offer yield in exchange for providing LP tokens.", | |
| "Staking rewards are distributed to token holders as an incentive.", | |
| "Governance tokens allow holders to vote on protocol changes.", | |
| "Token burns reduce supply and can increase value for remaining holders.", | |
| "Yield farming involves moving assets between protocols for optimal returns.", | |
| "Bridge protocols enable cross-chain asset transfers.", | |
| "Lending protocols allow users to borrow against their crypto collateral.", | |
| "Market making provides liquidity to exchanges for a spread.", | |
| ] | |
| async def mine_hard_negatives( | |
| query: str, | |
| num_negatives: int = 5, | |
| ) -> list[str]: | |
| """ | |
| Mine hard negatives from RAG search β documents that are similar | |
| to positives but not relevant. These are the most valuable for training. | |
| """ | |
| negatives = [] | |
| # Try RAG search for false positives | |
| try: | |
| from app.rag_service import three_pillar_search | |
| results = await three_pillar_search( | |
| query=query, | |
| collections=["known_scams", "token_analysis", "market_intel"], | |
| limit=num_negatives * 2, | |
| ) | |
| # Take results with lower confidence as hard negatives | |
| for r in results.get("results", [])[:num_negatives]: | |
| content = r.get("content", "") | |
| if content and len(content) > 50: | |
| negatives.append(content[:500]) | |
| except Exception: | |
| pass | |
| # Fall back to static hard negatives | |
| while len(negatives) < num_negatives: | |
| neg = random.choice(HARD_NEGATIVES) | |
| if neg not in negatives: | |
| negatives.append(neg) | |
| return negatives[:num_negatives] | |
| async def build_training_data() -> list[tuple[str, str, str, float]]: | |
| """ | |
| Build training triplets: (query, positive_doc, negative_doc, teacher_score) | |
| Returns list of triplets with cross-encoder teacher scores. | |
| """ | |
| training_data = [] | |
| # Get cross-encoder teacher | |
| try: | |
| from app.cross_encoder_reranker import get_reranker | |
| reranker = await get_reranker() | |
| if not reranker._model_loaded: | |
| await reranker.warm_up() | |
| except Exception: | |
| reranker = None | |
| for query, positives in SCAM_QUERIES: | |
| pos_doc = positives[0] | |
| # Mine hard negatives | |
| negatives = await mine_hard_negatives(query) | |
| for neg_doc in negatives: | |
| # Teacher score: cross-encoder rates (query, positive) and (query, negative) | |
| if reranker and reranker._model_loaded: | |
| try: | |
| pos_score = float(reranker._model.predict([(query, pos_doc)]).tolist()[0]) | |
| neg_score = float(reranker._model.predict([(query, neg_doc)]).tolist()[0]) | |
| # Margin: how much better is the positive? | |
| margin = pos_score - neg_score | |
| except Exception: | |
| margin = random.uniform(1.5, 3.0) | |
| else: | |
| margin = random.uniform(1.5, 3.0) | |
| training_data.append((query, pos_doc, neg_doc, margin)) | |
| return training_data | |
| def train_margin_mse( | |
| model, | |
| training_data: list[tuple[str, str, str, float]], | |
| epochs: int = 3, | |
| batch_size: int = 16, | |
| learning_rate: float = 2e-5, | |
| ) -> dict[str, Any]: | |
| """ | |
| Train embeddings using MarginMSE loss. | |
| For each triplet (query, pos, neg): | |
| - Embed query, positive, negative separately | |
| - Compute cosine similarities: sim(q,pos) and sim(q,neg) | |
| - Loss = MSE(sim(q,pos) - sim(q,neg), target_margin) | |
| - Optimize to push pos closer, neg farther | |
| This is the core of domain-specific embedding fine-tuning. | |
| """ | |
| from sentence_transformers import InputExample, losses | |
| from torch.utils.data import DataLoader | |
| # Convert to sentence-transformers InputExample format | |
| examples = [] | |
| for query, pos_doc, neg_doc, margin in training_data: | |
| # Normalize margin to 0-1 for cosine distance | |
| target_sim = max(0.0, min(1.0, (margin + 2.0) / 4.0)) | |
| examples.append( | |
| InputExample( | |
| texts=[query, pos_doc, neg_doc], | |
| label=target_sim, | |
| ) | |
| ) | |
| random.shuffle(examples) | |
| # Use MarginMSE loss | |
| train_dataloader = DataLoader(examples, batch_size=batch_size, shuffle=True) | |
| train_loss = losses.MarginMSELoss(model) | |
| logger.info(f"Training on {len(examples)} triplets for {epochs} epochs...") | |
| # Warm-up steps | |
| warmup_steps = int(len(train_dataloader) * epochs * 0.1) | |
| model.fit( | |
| train_objectives=[(train_dataloader, train_loss)], | |
| epochs=epochs, | |
| warmup_steps=warmup_steps, | |
| output_path="/app/data/models/scam-embeddings-v1", | |
| show_progress_bar=True, | |
| ) | |
| return { | |
| "status": "completed", | |
| "triplets": len(examples), | |
| "epochs": epochs, | |
| "output": "/app/data/models/scam-embeddings-v1", | |
| "model_size_mb": round( | |
| sum( | |
| os.path.getsize(os.path.join(dp, f)) | |
| for dp, _, fs in os.walk("/app/data/models/scam-embeddings-v1") | |
| for f in fs | |
| ) | |
| / (1024 * 1024), | |
| 1, | |
| ), | |
| } | |
| async def main(epochs: int = 3, batch_size: int = 16): | |
| """Main training pipeline.""" | |
| print("=" * 60) | |
| print(" Domain Fine-Tuned Embeddings β MarginMSE Training") | |
| print("=" * 60) | |
| # 1. Build training data with hard negatives | |
| print("\n[1/3] Building training data with hard negative mining...") | |
| training_data = await build_training_data() | |
| print(f" β {len(training_data)} training triplets") | |
| # 2. Load base model | |
| print("\n[2/3] Loading base embedding model...") | |
| try: | |
| from sentence_transformers import SentenceTransformer | |
| model = SentenceTransformer("BAAI/bge-small-en-v1.5") | |
| print(" β BGE-small loaded (384d)") | |
| except Exception as e: | |
| print(f" β Failed to load model: {e}") | |
| return 1 | |
| # 3. Train | |
| print(f"\n[3/3] Training MarginMSE for {epochs} epochs...") | |
| result = train_margin_mse(model, training_data, epochs=epochs, batch_size=batch_size) | |
| print(f" β Model saved to {result['output']}") | |
| print(f" Size: {result['model_size_mb']}MB") | |
| return 0 | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--epochs", type=int, default=3) | |
| parser.add_argument("--batch-size", type=int, default=16) | |
| args = parser.parse_args() | |
| sys.exit(asyncio.run(main(args.epochs, args.batch_size))) | |