Download app/databus/mega_scraper.py from cryptorugmunch/rugmuncher-backend: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/app/databus/mega_scraper.py
- Command line
-
hf download hf://cryptorugmunch/rugmuncher-backend/app/databus/mega_scraper.py
-
curl -L -o mega_scraper.py https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/app/databus/mega_scraper.py
11.3 kB
| """ | |
| RMI MEGA SCRAPER v3 β Substack, Mirror.xyz, Medium, Blog Scrapers | |
| Grabs EVERYTHING crypto. Biggest free news DB on the internet. | |
| """ | |
| import hashlib | |
| import json | |
| import logging | |
| import time | |
| from xml.etree import ElementTree as ET | |
| import httpx | |
| logger = logging.getLogger("rmi.scraper") | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # SUBSTACK β 50+ top crypto newsletters (free RSS) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| SUBSTACK_FEEDS = [ | |
| ("substack-bankless", "https://substack.com/@bankless/feed"), | |
| ("substack-milkroad", "https://substack.com/@milkroad/feed"), | |
| ("substack-messaricrypto", "https://substack.com/@messari/feed"), | |
| ("substack-defiweekly", "https://substack.com/@defiweekly/feed"), | |
| ("substack-thedefiedge", "https://substack.com/@thedefiedge/feed"), | |
| ("substack-cryptopragmatist", "https://substack.com/@cryptopragmatist/feed"), | |
| ("substack-blockworks", "https://substack.com/@blockworks/feed"), | |
| ("substack-coindesk", "https://substack.com/@coindesk/feed"), | |
| ("substack-reflexivity", "https://substack.com/@reflexivityresearch/feed"), | |
| ("substack-delphi", "https://substack.com/@delphidigital/feed"), | |
| ("substack-0xresearch", "https://substack.com/@0xresearch/feed"), | |
| ("substack-decentralised", "https://substack.com/@decentralisedco/feed"), | |
| ("substack-cryptoventure", "https://substack.com/@cryptoventure/feed"), | |
| ("substack-onchaintimes", "https://substack.com/@onchaintimes/feed"), | |
| ("substack-web3isgoinggreat", "https://substack.com/@web3isgreat/feed"), | |
| ("substack-dirtroads", "https://substack.com/@dirtroads/feed"), | |
| ("substack-frontiertech", "https://substack.com/@frontiertech/feed"), | |
| ("substack-chainalysis", "https://substack.com/@chainalysis/feed"), | |
| ("substack-elliptic", "https://substack.com/@elliptic/feed"), | |
| ("substack-trmlabs", "https://substack.com/@trmlabs/feed"), | |
| ("substack-thetie", "https://substack.com/@thetie/feed"), | |
| ("substack-glassnode", "https://substack.com/@glassnode/feed"), | |
| ("substack-nansen", "https://substack.com/@nansen/feed"), | |
| ("substack-arkham", "https://substack.com/@arkhamintel/feed"), | |
| ("substack-paradigm", "https://substack.com/@paradigm/feed"), | |
| ("substack-a16zcrypto", "https://substack.com/@a16zcrypto/feed"), | |
| ("substack-dragonfly", "https://substack.com/@dragonfly/feed"), | |
| ("substack-pantera", "https://substack.com/@panteracapital/feed"), | |
| ("substack-multicoin", "https://substack.com/@multicoincap/feed"), | |
| ("substack-variant", "https://substack.com/@variantfund/feed"), | |
| ("substack-electriccapital", "https://substack.com/@electriccapital/feed"), | |
| ("substack-coinfund", "https://substack.com/@coinfund/feed"), | |
| ("substack-framework", "https://substack.com/@frameworkventures/feed"), | |
| ("substack-1confirmation", "https://substack.com/@1confirmation/feed"), | |
| ("substack-placeholder", "https://substack.com/@placeholder/feed"), | |
| ("substack-union-square", "https://substack.com/@usv/feed"), | |
| ] | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # MIRROR.XYZ β Decentralized crypto publishing | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| MIRROR_FEEDS = [ | |
| ("mirror-weekly", "https://mirror.xyz/0x/feed"), | |
| # Individual writers (their RSS) | |
| ("mirror-zoranjoc", "https://zoranjoc.mirror.xyz/feed"), | |
| ("mirror-liam", "https://liam.mirror.xyz/feed"), | |
| ("mirror-dcbuilder", "https://dcbuilder.mirror.xyz/feed"), | |
| ("mirror-pacman", "https://pacman.mirror.xyz/feed"), | |
| ("mirror-nick", "https://nick.mirror.xyz/feed"), | |
| ] | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # MEDIUM β Crypto publications | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| MEDIUM_FEEDS = [ | |
| ("medium-coinfund", "https://blog.coinfund.io/feed"), | |
| ("medium-a16z", "https://a16zcrypto.com/feed/"), | |
| ("medium-paradigm", "https://medium.com/feed/paradigm"), | |
| ("medium-dragonfly", "https://medium.com/feed/dragonfly-research"), | |
| ("medium-multicoin", "https://multicoin.capital/feed/"), | |
| ("medium-electric", "https://medium.com/feed/electric-capital"), | |
| ("medium-coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"), | |
| ] | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # CRYPTO RESEARCH / TECH BLOGS | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| RESEARCH_FEEDS = [ | |
| ("arxiv-crypto", "https://rss.arxiv.org/rss/cs.CR"), | |
| ("github-trending-crypto", "https://github.com/trending?l=solidity&since=daily"), | |
| ("cryptopanic-news", "https://cryptopanic.com/news/rss/"), | |
| ("coingecko-research", "https://www.coingecko.com/en/research/rss"), | |
| ("thelafeed", "https://www.thela.news/feed"), | |
| ("cryptonewsbtc", "https://www.newsbtc.com/feed/"), | |
| ("cryptodailyuk", "https://cryptodaily.co.uk/feed/"), | |
| ("cryptoglobe", "https://www.cryptoglobe.com/feed/"), | |
| ("coinculture", "https://coinculture.com/au/feed/"), | |
| ("cryptopress", "https://cryptopress.com/feed/"), | |
| ("cryptopurview", "https://cryptopurview.com/feed/"), | |
| ("coinchapter", "https://coinchapter.com/feed/"), | |
| ("cryptowisser", "https://www.cryptowisser.com/feed/"), | |
| ("blockster", "https://blockster.com/feed/"), | |
| ("crypto-news-feed", "https://www.crypto-news-feed.com/feed/"), | |
| ("coininsider", "https://www.coininsider.com/feed/"), | |
| ("dailycoin", "https://dailycoin.com/feed/"), | |
| ("thecryptoupdates", "https://thecryptoupdates.com/feed/"), | |
| ("cryptoflies", "https://cryptoflies.com/feed/"), | |
| ("coincentral", "https://coincentral.com/feed/"), | |
| ] | |
| def fetch_generic_feeds(feed_list, prefix="", label="generic"): | |
| """Fetch any list of RSS feeds. Returns articles.""" | |
| results = [] | |
| for source, url in feed_list: | |
| try: | |
| resp = httpx.get(url, timeout=15, headers={"User-Agent": "RMI/4.0 MegaScraper"}) | |
| if resp.status_code != 200: | |
| continue | |
| root = ET.fromstring(resp.content) | |
| items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry") | |
| for item in items[:20]: | |
| title = (item.findtext("title", "") or "").strip() | |
| content = ( | |
| item.findtext("description", "") | |
| or item.findtext("content", "") | |
| or item.findtext("{http://www.w3.org/2005/Atom}summary", "") | |
| or "" | |
| ).strip() | |
| if title and len(title) > 5: | |
| doc_id = ( | |
| f"{prefix}{source}:" | |
| + hashlib.sha256((source + title).encode()).hexdigest()[:16] | |
| ) | |
| results.append( | |
| { | |
| "id": doc_id, | |
| "title": title, | |
| "content": content[:3000], | |
| "url": url, | |
| "source": source, | |
| "sentiment": 0.0, | |
| "tickers": [], | |
| "published": "", | |
| "ingested_at": time.time(), | |
| } | |
| ) | |
| logger.info(f" {prefix}{source}: {len(results)} articles") | |
| except Exception as e: | |
| logger.debug(f" {prefix}{source}: {str(e)[:60]}") | |
| return results | |
| def store_all(): | |
| """Fetch all sources and store in Redis + Postgres.""" | |
| all_articles = [] | |
| all_articles.extend(fetch_generic_feeds(SUBSTACK_FEEDS, "[Substack] ", "substack")) | |
| all_articles.extend(fetch_generic_feeds(MIRROR_FEEDS, "[Mirror] ", "mirror")) | |
| all_articles.extend(fetch_generic_feeds(MEDIUM_FEEDS, "[Medium] ", "medium")) | |
| all_articles.extend(fetch_generic_feeds(RESEARCH_FEEDS, "[Research] ", "research")) | |
| try: | |
| from dotenv import load_dotenv | |
| load_dotenv("/app/.env", override=True) | |
| import os | |
| import psycopg2 | |
| import redis | |
| r = redis.Redis( | |
| host="rmi-redis", port=6379, password=os.getenv("REDIS_PASSWORD"), decode_responses=True | |
| ) | |
| # Dedup and store | |
| count = 0 | |
| for a in all_articles: | |
| if not r.exists(f"rmi:news:article:{a['id']}"): | |
| r.zadd("rmi:news:substack:index", {a["id"]: a["ingested_at"]}) | |
| r.set(f"rmi:news:article:{a['id']}", json.dumps(a)) | |
| count += 1 | |
| # Update stats | |
| existing = json.loads(r.get("rmi:news:stats") or "{}") | |
| existing["substack_articles"] = count | |
| existing["total_sources"] = ( | |
| existing.get("total_sources", 0) | |
| + len(SUBSTACK_FEEDS) | |
| + len(MIRROR_FEEDS) | |
| + len(MEDIUM_FEEDS) | |
| + len(RESEARCH_FEEDS) | |
| ) | |
| r.set("rmi:news:stats", json.dumps(existing)) | |
| # Also Postgres | |
| try: | |
| pg = psycopg2.connect( | |
| host="rmi-postgres", | |
| port=5432, | |
| user="rmi", | |
| password=os.getenv("POSTGRES_PASSWORD"), | |
| dbname="rmi", | |
| ) | |
| cur = pg.cursor() | |
| cur.execute( | |
| "CREATE TABLE IF NOT EXISTS crypto_news (id TEXT PRIMARY KEY, title TEXT, content TEXT, source TEXT, sentiment REAL, ingested_at DOUBLE PRECISION, category TEXT DEFAULT 'research')" | |
| ) | |
| for a in all_articles: | |
| cur.execute( | |
| "INSERT INTO crypto_news (id, title, content, source, sentiment, ingested_at, category) VALUES (%s,%s,%s,%s,%s,%s,'research') ON CONFLICT (id) DO NOTHING", | |
| (a["id"], a["title"], a["content"], a["source"], 0.0, a["ingested_at"]), | |
| ) | |
| pg.commit() | |
| cur.close() | |
| pg.close() | |
| except Exception as e: | |
| logger.error(f"Postgres: {e}") | |
| return { | |
| "substack": len(SUBSTACK_FEEDS), | |
| "mirror": len(MIRROR_FEEDS), | |
| "medium": len(MEDIUM_FEEDS), | |
| "research": len(RESEARCH_FEEDS), | |
| "new_articles": count, | |
| } | |
| except Exception as e: | |
| logger.error(f"Store failed: {e}") | |
| return {"error": str(e), "articles_collected": len(all_articles)} | |
| if __name__ == "__main__": | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s [mega] %(message)s") | |
| result = store_all() | |
| logger.info(json.dumps(result, indent=2)) | |