Upload 2 files

src/services/analysis.py

@@ -1,12 +1,13 @@
-import datetime
 import re
-from pathlib import Path
-from typing import List, Optional, Tuple
-
+import datetime
 import pandas as pd
+from typing import List, Optional, Tuple
+from pathlib import Path
 
+# Import our modular components
 from ..state import get_user_temp_dir
-from .utils import cleanup_after_analysis, convert_html_to_pdf, now_str
+from .utils import now_str, convert_html_to_pdf, cleanup_after_analysis
+from .futures_engine import PDFParser
 
 # --- Constants for Reporting ---
 ORIGINAL_HTML_STYLE = """
@@ -28,77 +29,8 @@ ORIGINAL_MATCHED_HEADERS = ["Ticker", "Spot MrktCap", "Spot Volume", "Spot VTMR"
 ORIGINAL_FUTURES_HEADERS = ["Ticker", "Market Cap", "Volume", "VTMR", "OISS", "Funding Rate"]
 ORIGINAL_SPOT_HEADERS = ["Ticker", "MarketCap", "Volume", "VTMR"]
 
-
-class SignalEngine:
-    """Calculates OISS and Funding signals and formats them into HTML spans."""
-
-    @staticmethod
-    def _oi_score_and_signal(oi_change: float) -> Tuple[int, str]:
-        if oi_change > 0.20:
-            return 5, "Strong"
-        if oi_change > 0.10:
-            return 4, "Bullish"
-        if oi_change > 0.00:
-            return 3, "Build-Up"
-        if oi_change > -0.10:
-            return 2, "Weakening"
-        if oi_change > -0.20:
-            return 1, "Exiting"
-        return 0, "Exiting"
-
-    @staticmethod
-    def _funding_score_and_signal(funding_val: float) -> Tuple[str, str]:
-        if funding_val >= 0.05:
-            return "Greed", "oi-strong"
-        if funding_val > 0.00:
-            return "Bullish", "oi-strong"
-        if funding_val <= -0.05:
-            return "Extreme Fear", "oi-weak"
-        if funding_val < 0.00:
-            return "Bearish", "oi-weak"
-        return "Neutral", ""
-
-    @classmethod
-    def make_oiss(cls, oi_percent_str: str) -> str:
-        if not oi_percent_str or str(oi_percent_str) in ['-', 'N/A', 'nan']:
-            return "-"
-        val = str(oi_percent_str).replace("%", "").strip()
-        try:
-            oi_change = float(val) / 100
-            score, signal = cls._oi_score_and_signal(oi_change)
-            
-            if oi_change > 0:
-                css_class = "oi-strong"
-            elif oi_change < 0:
-                css_class = "oi-weak"
-            else:
-                css_class = ""
-
-            sign = "+" if oi_change > 0 else ""
-            if css_class:
-                return f'<span class="{css_class}">{sign}{oi_change*100:.0f}%</span> {signal}'
-            return f"{sign}{oi_change*100:.0f}% {signal}"
-        except Exception:
-            return "-"
-
-    @classmethod
-    def make_funding_signal(cls, funding_str: str) -> str:
-        if not funding_str or str(funding_str) in ['-', 'N/A', 'nan']:
-            return "-"
-        try:
-            val = float(str(funding_str).replace('%', '').strip())
-            signal_word, css_class = cls._funding_score_and_signal(val)
-            
-            if css_class:
-                return f'<span class="{css_class}">{val}%</span> <span style="font-size:0.8em; color:#7f8c8d;">{signal_word}</span>'
-            return f'{val}% {signal_word}'
-        except Exception:
-            return str(funding_str)
-
-
 class FileScanner:
-    """Locates the latest Spot data and parsed Futures CSV files in the USER directory."""
-
+    """Locates the latest Spot and Futures data files in the USER directory."""
     @staticmethod
     def find_files(user_id) -> Tuple[Optional[Path], Optional[Path]]:
         spot_file: Optional[Path] = None
@@ -108,14 +40,16 @@ class FileScanner:
         if not user_dir.exists():
             return None, None
 
+        # Get today's date for filtering
         today = datetime.datetime.now().date()
+        
+        # Filter for today's files only
         today_files = []
-
         for f in user_dir.iterdir():
             if f.is_file():
                 try:
                     file_time = datetime.datetime.fromtimestamp(f.stat().st_mtime)
-                    if file_time.date() == today:
+                    if file_time.date() == today:  # Only use today's files
                         today_files.append(f)
                 except Exception:
                     continue
@@ -123,11 +57,12 @@ class FileScanner:
         if not today_files:
             return None, None
             
+        # Sort by modification time (newest first)
         files = sorted(today_files, key=lambda x: x.stat().st_mtime, reverse=True)
 
         for f in files:
             name = f.name.lower()
-            if not futures_file and f.suffix == ".csv" and "futures" in name:
+            if not futures_file and f.suffix == ".pdf" and "futures" in name:
                 futures_file = f
             elif not spot_file and f.suffix in [".csv", ".html"] and "spot" in name:
                 spot_file = f
@@ -137,14 +72,14 @@ class FileScanner:
                 
         return spot_file, futures_file
 
-
 class DataProcessor:
-    """Handles Dataframe loading, merging, signal generation, and HTML generation."""
-
+    """Handles Dataframe loading, merging, and HTML generation."""
+    
     @staticmethod
     def load_spot(path: Path) -> pd.DataFrame:
         print(f"   Parsing Spot File: {path.name}")
         try:
+            # Explicit UTF-8 for Unicode preservation
             if path.suffix == '.html':
                 df = pd.read_html(str(path), encoding='utf-8')[0]
             else:
@@ -154,7 +89,7 @@ class DataProcessor:
             col_map = {
                 'ticker': 'ticker',
                 'symbol': 'ticker', 
-                'vtmr': 'vtmr',
+                'vtmr': 'vtmr',           # <--- Ensures VTMR isn't blank
                 'spot_vtmr': 'vtmr', 
                 'flipping_multiple': 'vtmr',
                 'market_cap': 'market_cap',
@@ -165,12 +100,14 @@ class DataProcessor:
             
             df = df.rename(columns=col_map, errors='ignore')
             
+            # Normalize ticker column (Find it if it's missing)
             if 'ticker' not in df.columns:
-                for col in df.columns:
+                 for col in df.columns:
                     if 'sym' in col or 'tick' in col or 'tok' in col:
                         df = df.rename(columns={col: 'ticker'})
                         break
 
+            # Unicode-safe cleaning (Protects Chinese characters)
             if 'ticker' in df.columns:
                 df['ticker'] = df['ticker'].apply(lambda x: str(x).strip().upper())
                 
@@ -179,25 +116,7 @@ class DataProcessor:
         except Exception as e:
             print(f"   Spot File Error: {e}")
             return pd.DataFrame()
-
-    @staticmethod
-    def load_futures(path: Path) -> pd.DataFrame:
-        """Loads raw parsed Futures CSV and formats OISS and Funding signals."""
-        print(f"   Parsing Futures CSV: {path.name}")
-        try:
-            df = pd.read_csv(path, encoding='utf-8')
-            if df.empty:
-                return pd.DataFrame()
-
-            df['oiss'] = df['oi_pct'].apply(SignalEngine.make_oiss)
-            df['funding'] = df['funding'].apply(SignalEngine.make_funding_signal)
-
-            print(f"   Loaded {len(df)} futures tokens from CSV")
-            return df
-        except Exception as e:
-            print(f"   Futures CSV Error: {e}")
-            return pd.DataFrame()
-
+            
     @staticmethod
     def _generate_table_html(title: str, df: pd.DataFrame, headers: List[str], df_cols: List[str]) -> str:
         if df.empty:
@@ -208,6 +127,7 @@ class DataProcessor:
             df_display[m] = ""
         df_display = df_display[df_cols]
         df_display.columns = headers
+        # escape=False is critical for rendering ticker links
         table_html = df_display.to_html(index=False, classes='table', escape=False)
         return f'<div class="table-container"><h2>{title}</h2>{table_html}</div>'
 
@@ -229,6 +149,7 @@ class DataProcessor:
             print(f"   Futures high-quality filtering error: {e}")
             valid_futures['vtmr_display'] = valid_futures['vtmr']
 
+        # Suffix-based merge to prevent blank column mapping issues
         merged = pd.merge(spot_df, valid_futures, on='ticker', how='inner', suffixes=('_spot', '_fut'))
         if 'vtmr_fut' in merged.columns:
             merged = merged.sort_values('vtmr_fut', ascending=False)
@@ -315,23 +236,25 @@ class DataProcessor:
         """
         return html
 
-
 def crypto_analysis_v4(user_keys, user_id) -> None:
     """Main execution flow for Advanced Analysis."""
     print("   ADVANCED QUANT CRYPTO VOLUME ANALYSIS")
-    print("   Scanning for Parsed Futures CSV and Spot CSV/HTML files")
+    print("   Scanning for Futures PDF and Spot CSV/HTML files")
     print("   " + "=" * 50)
     
+    # Find Files
     spot_file, futures_file = FileScanner.find_files(user_id)
     if not spot_file or not futures_file:
         print("   Required files not found.")
-        raise FileNotFoundError("   You Need Spot Market Data and Parsed Futures CSV Data. Kindly Generate Spot Data And Parse Futures PDF First.")
+        raise FileNotFoundError("   You Need CoinAlyze Futures PDF and Spot Market Data. Kindly Generate Spot Data And Upload Futures PDF First.")
     
-    futures_df = DataProcessor.load_futures(futures_file)
+    # Parse Files
+    futures_df = PDFParser.extract(futures_file)
     spot_df = DataProcessor.load_spot(spot_file)
     
     html_content = DataProcessor.generate_html_report(futures_df, spot_df)
     if html_content:
+        # Create PDF
         pdf_path = convert_html_to_pdf(html_content, user_id)
         
         if pdf_path:

src/services/futures_engine.py

@@ -1,15 +1,13 @@
 import re
 import pandas as pd
-from pathlib import Path
 from dataclasses import dataclass
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple
 
 try:
     import pypdf
 except Exception:
     pypdf = None
 
-
 @dataclass
 class TokenData:
     ticker: str
@@ -18,12 +16,11 @@ class TokenData:
     volume: str
     vtmr: float
     funding: str = "-"
-    oi_pct: str = "-"
-
+    oiss: str = "-"
 
 class PDFParser:
     """Handles extraction of tabular data from Coinalyze PDFs using regex."""
-
+    
     FINANCIAL_PATTERN = re.compile(
         r'(\$?[+-]?[\d,\.]+[kKmMbB]?)\s+'             
         r'(\$?[+-]?[\d,\.]+[kKmMbB]?)\s+'             
@@ -37,29 +34,76 @@ class PDFParser:
         'mkt cap', 'vol 24h', 'vtmr', 'coins', 'all contracts', 'custom metrics', 'watchlists'
     }
 
+    # --- Signal Helpers (Moved inside to keep logic self-contained) ---
+
+    @staticmethod
+    def _oi_score_and_signal(oi_change: float) -> Tuple[int, str]:
+        if oi_change > 0.20: return 5, "Strong"
+        if oi_change > 0.10: return 4, "Bullish"
+        if oi_change > 0.00: return 3, "Build-Up"
+        if oi_change > -0.10: return 2, "Weakening"
+        if oi_change > -0.20: return 1, "Exiting"
+        return 0, "Exiting"
+
+    @staticmethod
+    def _funding_score_and_signal(funding_val: float) -> Tuple[str, str]:
+        if funding_val >= 0.05: return "Greed", "oi-strong"
+        if funding_val > 0.00: return "Bullish", "oi-strong"
+        if funding_val <= -0.05: return "Extreme Fear", "oi-weak"
+        if funding_val < 0.00: return "Bearish", "oi-weak"
+        return "Neutral", ""
+
     @classmethod
-    def extract(cls, path: Union[str, Path]) -> pd.DataFrame:
-        """Parses Futures PDF and returns a DataFrame containing raw extracted data."""
-        pdf_path = Path(path)
-        print(f"   Parsing Futures PDF: {pdf_path.name}")
-        
+    def make_oiss(cls, oi_percent_str: str) -> str:
+        if not oi_percent_str: return "-"
+        val = oi_percent_str.replace("%", "").strip()
+        try:
+            oi_change = float(val) / 100
+            score, signal = cls._oi_score_and_signal(oi_change)
+            
+            if oi_change > 0: css_class = "oi-strong"
+            elif oi_change < 0: css_class = "oi-weak"
+            else: css_class = ""
+
+            sign = "+" if oi_change > 0 else ""
+            if css_class:
+                return f'<span class="{css_class}">{sign}{oi_change*100:.0f}%</span> {signal}'
+            return f"{sign}{oi_change*100:.0f}% {signal}"
+        except Exception:
+            return "-"
+
+    @classmethod
+    def make_funding_signal(cls, funding_str: str) -> str:
+        if not funding_str or funding_str in ['-', 'N/A']: return "-"
+        try:
+            val = float(funding_str.replace('%', '').strip())
+            signal_word, css_class = cls._funding_score_and_signal(val)
+            
+            if css_class:
+                return f'<span class="{css_class}">{val}%</span> <span style="font-size:0.8em; color:#7f8c8d;">{signal_word}</span>'
+            return f'{val}% {signal_word}'
+        except Exception:
+            return funding_str
+
+    # --- Core Extraction Logic ---
+
+    @classmethod
+    def extract(cls, path) -> pd.DataFrame:
+        print(f"   Parsing Futures PDF: {path.name}")
         if pypdf is None:
             print("   pypdf not available - PDF parsing disabled.")
             return pd.DataFrame()
-
         data: List[TokenData] = []
         try:
-            reader = pypdf.PdfReader(pdf_path)
+            reader = pypdf.PdfReader(path)
             for page in reader.pages:
                 raw = page.extract_text() or ""
                 lines = [ln.strip() for ln in raw.split("\n") if ln.strip()]
                 page_data = cls._parse_page_smart(lines)
                 data.extend(page_data)
-
             print(f"   Extracted {len(data)} futures tokens")
             if not data:
                 return pd.DataFrame()
-
             df = pd.DataFrame([vars(t) for t in data])
             df['ticker'] = df['ticker'].apply(lambda x: re.sub(r'[^A-Z0-9]', '', str(x).upper()))
             df = df[df['ticker'].str.len() > 1]
@@ -69,17 +113,6 @@ class PDFParser:
             print(f"   PDF Error: {e}")
             return pd.DataFrame()
 
-    @classmethod
-    def parse_and_export_csv(cls, pdf_path: Union[str, Path], output_csv_path: Union[str, Path]) -> pd.DataFrame:
-        """Extracts Futures PDF data and saves raw results to a temporary CSV file."""
-        df = cls.extract(pdf_path)
-        if not df.empty:
-            out_path = Path(output_csv_path)
-            out_path.parent.mkdir(parents=True, exist_ok=True)
-            df.to_csv(out_path, index=False, encoding='utf-8')
-            print(f"   Raw Futures CSV saved to: {out_path}")
-        return df
-
     @classmethod
     def _parse_page_smart(cls, lines: List[str]) -> List[TokenData]:
         financials = []
@@ -100,7 +133,7 @@ class PDFParser:
                 try:
                     float(vtmr)
                     financials.append((mc, vol, vtmr, oi_str, fund_str))
-                except Exception:
+                except:
                     raw_text_lines.append(line)
             else:
                 if not line.isdigit() and len(line) > 1:
@@ -140,22 +173,23 @@ class PDFParser:
             name, ticker = token_pairs[k]
             mc, vol, vtmr, oi_pct, fund_pct = financials[k]
 
+            oiss_val = cls.make_oiss(oi_pct) if oi_pct and oi_pct not in ['-', 'N/A'] else "-"
+            funding_val = cls.make_funding_signal(fund_pct)
+
             tokens.append(TokenData(
                 ticker=ticker,
                 name=name,
                 market_cap=mc,
                 volume=vol,
                 vtmr=float(vtmr),
-                funding=fund_pct if fund_pct and fund_pct not in ['-', 'N/A'] else "-",
-                oi_pct=oi_pct if oi_pct and oi_pct not in ['-', 'N/A'] else "-"
+                funding=funding_val,
+                oiss=oiss_val
             ))
         return tokens
 
     @staticmethod
     def _clean_ticker_strict(text: str) -> Optional[str]:
-        if len(text) > 15:
-            return None
+        if len(text) > 15: return None
         cleaned = re.sub(r'[^A-Z0-9]', '', text.upper())
-        if 2 <= len(cleaned) <= 12:
-            return cleaned
+        if 2 <= len(cleaned) <= 12: return cleaned
         return None