Download scripts/convert_new_papers.py from cryptorugmunch/rugmuncher-backend: direct link, hf CLI and curl.
- Browser
- Download file 4.12 kB
-
https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/scripts/convert_new_papers.py
- Command line
-
hf download hf://cryptorugmunch/rugmuncher-backend/scripts/convert_new_papers.py
-
curl -L -o convert_new_papers.py https://huggingface.co/cryptorugmunch/rugmuncher-backend/resolve/main/scripts/convert_new_papers.py
4.12 kB
| #!/usr/bin/env python3 | |
| """Convert and ingest additional arxiv papers into RAG.""" | |
| import json | |
| import os | |
| import sys | |
| import pymupdf4llm | |
| sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) | |
| PAPERS_DIR = os.path.dirname(os.path.abspath(__file__)).rstrip("/scripts") + "/data/papers" | |
| with open(os.path.join(PAPERS_DIR, "papers_metadata.json")) as f: | |
| existing = json.load(f) | |
| new_papers = { | |
| "2109.05940": { | |
| "title": "Smart Contract Vulnerability Detection: Graph Feature + Expert Pattern Fusion", | |
| "authors": [], | |
| "year": 2021, | |
| "arxiv_id": "2109.05940", | |
| "attack_types": ["vulnerability", "static-analysis"], | |
| "collection": "forensic_reports", | |
| "description": "Improving smart contract vulnerability detection using graph features and expert patterns.", | |
| }, | |
| "2006.06116": { | |
| "title": "Temporal-Amount Snapshot MultiGraph for Ethereum Transaction Tracking", | |
| "authors": [], | |
| "year": 2020, | |
| "arxiv_id": "2006.06116", | |
| "attack_types": ["transaction-graph", "tracking"], | |
| "collection": "forensic_reports", | |
| "description": "Multi-graph snapshot approach for tracking Ethereum transactions.", | |
| }, | |
| "1905.00346": { | |
| "title": "Reentrancy Vulnerability Identification in Ethereum Smart Contracts", | |
| "authors": [], | |
| "year": 2019, | |
| "arxiv_id": "1905.00346", | |
| "attack_types": ["reentrancy", "vulnerability"], | |
| "collection": "forensic_reports", | |
| "description": "Detection of reentrancy vulnerabilities in Ethereum smart contracts.", | |
| }, | |
| "2207.01247": { | |
| "title": "How to Design Tokenomics: Framework from 20+ Projects", | |
| "authors": [], | |
| "year": 2022, | |
| "arxiv_id": "2207.01247", | |
| "attack_types": ["tokenomics", "framework"], | |
| "collection": "market_intel", | |
| "description": "Tokenomics design framework from analyzing 20+ blockchain projects.", | |
| }, | |
| "2202.05146": { | |
| "title": "Empirical Evidence from Governance Token Distributions", | |
| "authors": [], | |
| "year": 2022, | |
| "arxiv_id": "2202.05146", | |
| "attack_types": ["governance", "token-distribution"], | |
| "collection": "market_intel", | |
| "description": "Empirical analysis of governance token distributions and DAO implications.", | |
| }, | |
| "2004.02968": { | |
| "title": "Ponzi Scheme Detection in Ethereum Transaction Network", | |
| "authors": [], | |
| "year": 2020, | |
| "arxiv_id": "2004.02968", | |
| "attack_types": ["ponzi", "detection"], | |
| "collection": "forensic_reports", | |
| "description": "Graph-based Ponzi scheme detection in Ethereum transactions.", | |
| }, | |
| "2104.02368": { | |
| "title": "EtherClue: Digital Investigation of Attacks on Ethereum Smart Contracts", | |
| "authors": [], | |
| "year": 2021, | |
| "arxiv_id": "2104.02368", | |
| "attack_types": ["forensics", "attack-detection"], | |
| "collection": "forensic_reports", | |
| "description": "EtherClue digital forensic investigation of smart contract attacks.", | |
| }, | |
| } | |
| converted = 0 | |
| for arxiv_id, info in new_papers.items(): | |
| pdf_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.pdf") | |
| txt_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.txt") | |
| if not os.path.exists(pdf_path): | |
| print(f"SKIP {arxiv_id}: PDF not found") | |
| continue | |
| if arxiv_id in existing: | |
| print(f"SKIP {arxiv_id}: already in metadata") | |
| continue | |
| sz = os.path.getsize(pdf_path) | |
| if sz < 5000: | |
| print(f"INVALID {arxiv_id}: PDF too small ({sz} bytes)") | |
| continue | |
| try: | |
| text = pymupdf4llm.to_markdown(pdf_path) | |
| with open(txt_path, "w") as f: | |
| f.write(text) | |
| words = len(text.split()) | |
| print(f"OK {arxiv_id}: {words} words") | |
| existing[arxiv_id] = info | |
| converted += 1 | |
| except Exception as e: | |
| print(f"ERROR {arxiv_id}: {e}") | |
| with open(os.path.join(PAPERS_DIR, "papers_metadata.json"), "w") as f: | |
| json.dump(existing, f, indent=2) | |
| print(f"Converted: {converted}, Total papers: {len(existing)}") | |