Deploy updated nsedash (clean obfuscated deployment)

README.md

@@ -1,11 +0,0 @@
----
-title: Nsedash
-emoji: 🌖
-colorFrom: yellow
-colorTo: yellow
-sdk: docker
-pinned: false
-short_description: NSE Symbols information in a Dashboard
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -44,7 +44,7 @@ def _silent_loader():
     if not app_script.exists():
         app_script = Path("nsedash_app.py")
 
-    args = [sys.executable, str(app_script), "-hf"]
+    args = [sys.executable, str(app_script), "--hf"]
     try:
         os.execv(sys.executable, args)
     except Exception:

nsecorpact.py

@@ -1,284 +0,0 @@
-# This script processes corporate action data from the NSE and generates a JSON file
-# containing the necessary information for dashboard integration.
-
-import pandas as pd
-import re
-import requests
-import os
-from datetime import datetime, timedelta
-
-# --- Configuration and Setup ---
-# Directory to save processed files (final destination)
-processed_dir = './processed'
-# Temporary directory for downloads
-temp_dir = './tempfiles'
-
-# Ensure both directories exist
-os.makedirs(processed_dir, exist_ok=True)
-os.makedirs(temp_dir, exist_ok=True)
-
-# Output JSON file path
-output_file = os.path.join(processed_dir, 'nse_dashboard_corpact.json')
-
-# Base URL for downloading corporate actions data
-DOWNLOAD_URL_TEMPLATE = "https://www.nseindia.com/api/corporates-corporateActions?index=equities&from_date=01-01-2020&to_date={}&csv=true"
-
-# URL for the main corporate actions page (to get session cookies)
-NSE_CORPORATE_ACTIONS_PAGE = "https://www.nseindia.com/companies-listing/corporate-filings-actions"
-
-# --- Helper Functions for Date and Download ---
-
-def get_last_nse_working_day():
-    """
-    Determines the last NSE working day (i.e., the most recent weekday).
-    NSE is typically closed on Saturdays and Sundays.
-    """
-    today = datetime.now()
-    # Subtract days until a weekday is found (Monday=0, Tuesday=1, ..., Sunday=6)
-    # If today is Saturday (5), subtract 1 to get Friday.
-    # If today is Sunday (6), subtract 2 to get Friday.
-    # If today is Monday-Friday, use today.
-    while today.weekday() > 4: # 5 is Saturday, 6 is Sunday
-        today -= timedelta(days=1)
-    return today.strftime('%d-%m-%Y')
-
-def download_nse_corporate_actions_file():
-    """
-    Downloads the corporate actions CSV file from NSE to the tempfiles directory.
-    It returns the path to the downloaded file in the temporary directory.
-    Returns None if download fails.
-    """
-    last_working_day = get_last_nse_working_day()
-    download_url = DOWNLOAD_URL_TEMPLATE.format(last_working_day)
-    
-    # Construct the temporary file name and path
-    temp_file_name = f"CF-CA-equities-01-01-2020-to-{last_working_day}.csv"
-    temp_file_path = os.path.join(temp_dir, temp_file_name)
-
-    # Note: We always attempt to download the latest to temp, no skipping here.
-    # The check for existing file is done at the processed_dir level later.
-
-    print(f"Attempting to download corporate actions data to temporary directory.")
-    
-    # Create a session to persist cookies and headers
-    session = requests.Session()
-    
-    try:
-        # Step 1: Visit the main corporate actions page to get necessary cookies
-        print(f"Fetching cookies.")
-        session_headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-            'Accept-Language': 'en-US,en;q=0.9',
-            'Connection': 'keep-alive',
-            'Upgrade-Insecure-Requests': '1',
-        }
-        session.get(NSE_CORPORATE_ACTIONS_PAGE, headers=session_headers)
-        
-        # Step 2: Use the same session to download the CSV file
-        print(f"Downloading CSV using established session.")
-        download_headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Accept': 'text/csv', # Specifically request CSV
-            'Accept-Language': 'en-US,en;q=0.9',
-            'Connection': 'keep-alive',
-            'Referer': NSE_CORPORATE_ACTIONS_PAGE, # Refer to the main page visited earlier
-            'Sec-Fetch-Dest': 'document',
-            'Sec-Fetch-Mode': 'navigate',
-            'Sec-Fetch-Site': 'same-origin',
-            'Sec-Fetch-User': '?1',
-            'Upgrade-Insecure-Requests': '1',
-        }
-        response = session.get(download_url, headers=download_headers)
-        response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx)
-
-        # Get the raw binary content
-        raw_bytes_content = response.content
-        
-        # --- BOM Preprocessing ---
-        # Detect and remove UTF-8 BOM if present (EF BB BF)
-        if raw_bytes_content.startswith(b'\xef\xbb\xbf'):
-            # print("Detected and removed UTF-8 BOM from the file content.")
-            raw_bytes_content = raw_bytes_content[3:]
-        # --- End BOM Preprocessing ---
-
-        # Decode to string for further preprocessing (only line stripping, no .strip('"'))
-        raw_content_string = raw_bytes_content.decode('latin1', errors='ignore')
-        
-        # Preprocessing: Only strip leading/trailing whitespace from each line
-        cleaned_lines = [line.strip() for line in raw_content_string.splitlines()]
-        cleaned_content = "\n".join(cleaned_lines)
-        
-        # Save the cleaned content to the temporary file path
-        with open(temp_file_path, 'w', encoding='latin1') as f:
-            f.write(cleaned_content)
-            
-        print(f"Successfully downloaded and preprocessed to temporary file.")
-        return temp_file_path
-    except requests.exceptions.RequestException as e:
-        print(f"Error downloading file: {e}")
-        print("This often indicates an issue with the NSE API (e.g., authentication required, rate limiting, or changed endpoint).")
-        print("You may still need to manually download the file if this persists.")
-        return None
-
-# --- Existing Functions (Copied from original prompt) ---
-
-# Function to calculate adjustment factor for bonuses
-def calculate_bonus_adj_factor(purpose):
-    """Extracts and calculates the adjustment factor for bonus shares."""
-    if "Bonus" in purpose:
-        try:
-            # Ratio format example: "1:4" in "Bonus 1:4"
-            # It's safer to use regex to extract the ratio explicitly
-            match = re.search(r"Bonus (\d+:\d+)", purpose)
-            if match:
-                ratio = match.group(1)
-                bonus, base = map(int, ratio.split(":"))
-                return f"{bonus}:{base}", "Bonus"
-        except ValueError:
-            print(f"Could not parse bonus ratio from: {purpose}")
-    return None, None
-
-# Function to calculate adjustment factor for stock splits
-def calculate_split_adj_factor(purpose):
-    """Extracts and calculates the adjustment factor for stock splits."""
-    if "Face Value Split" in purpose:
-        try:
-            # Use regex to find the 'From' and 'To' values
-            # Example: "Face Value Split (Sub-Division) - From Rs 10/- Per Share To Re 1/- Per Share"
-            match = re.search(r"From Rs (\d+(?:\.\d+)?)/?-? Per Share To (?:Re|Rs) (\d+(?:\.\d+)?)/?-? Per Share", purpose)
-            if match:
-                from_value_str = match.group(1)
-                to_value_str = match.group(2)
-
-                # Convert to float to handle potential decimal values
-                from_value = float(from_value_str)
-                to_value = float(to_value_str)
-
-                # Return ratio as 'new_face_value:old_face_value'
-                # For a split from 10 to 1, the ratio is 1:10
-                return f"{int(to_value)}:{int(from_value)}", "Split"
-            else:
-                pass # No match found
-        except ValueError:
-            print(f"Could not parse split values from: {purpose}")
-    return None, None
-
-def main():
-    temp_downloaded_file_path = None # Path of the file in tempfiles
-    input_file = None                 # Final path of the file in processed_dir
-    df = None                         # DataFrame for processing
-
-    # Step 1: Attempt to download and preprocess the input file to tempfiles
-    temp_downloaded_file_path = download_nse_corporate_actions_file()
-
-    # Define the final target file path in the processed directory
-    final_processed_file_name = 'Corporate_action.parquet'
-    final_processed_file_path = os.path.join(processed_dir, final_processed_file_name)
-
-    if temp_downloaded_file_path and os.path.exists(temp_downloaded_file_path):
-        try:
-            # Read downloaded CSV
-            df_temp = pd.read_csv(temp_downloaded_file_path, encoding='latin1')
-            # Save to Parquet
-            df_temp.to_parquet(final_processed_file_path, index=False)
-            # Remove temp CSV
-            os.remove(temp_downloaded_file_path)
-            # Also clean up any legacy Corporate_action.csv in the processed dir
-            legacy_csv = os.path.join(processed_dir, 'Corporate_action.csv')
-            if os.path.exists(legacy_csv):
-                os.remove(legacy_csv)
-            input_file = final_processed_file_path
-            print("CSV downloaded file converted to Parquet and saved successfully.")
-        except Exception as e:
-            print(f"Error processing downloaded CSV file: {e}")
-            input_file = final_processed_file_path if os.path.exists(final_processed_file_path) else None
-    else:
-        print("\nAutomatic download to temporary directory failed or file not found in tempfiles.")
-        # Fallback: Check for the specifically named file in processed_dir
-        print(f"Checking for '{final_processed_file_name}' in '{processed_dir}' as a fallback...")
-        if os.path.exists(final_processed_file_path):
-            input_file = final_processed_file_path
-            print(f"Using existing file as input: '{input_file}'")
-        else:
-            # Check if there is an existing Corporate_action.csv to convert
-            fallback_csv = os.path.join(processed_dir, 'Corporate_action.csv')
-            if os.path.exists(fallback_csv):
-                try:
-                    df_temp = pd.read_csv(fallback_csv, encoding='latin1')
-                    df_temp.to_parquet(final_processed_file_path, index=False)
-                    os.remove(fallback_csv)
-                    input_file = final_processed_file_path
-                    print(f"Auto-converted existing '{fallback_csv}' to Parquet.")
-                except Exception as e:
-                    print(f"Failed to auto-convert existing CSV fallback: {e}")
-            
-            if not input_file:
-                print(f"No '{final_processed_file_name}' found in '{processed_dir}'. Cannot proceed without an input file.")
-                print(f"Please manually download the latest CSV from the following URL and place it in the '{temp_dir}' directory, then rerun the script:")
-                print(f"  {DOWNLOAD_URL_TEMPLATE.format(get_last_nse_working_day())}")
-                return # Exit if no input file is available
-
-    # Step 4: Use the determined input file as the dataframe source
-    if input_file and os.path.exists(input_file): # Ensure input_file exists before reading
-        try:
-            df = pd.read_parquet(input_file)
-            print(f"Successfully loaded data from '{input_file}'.")
-        except Exception as e:
-            print(f"Error reading input Parquet file '{input_file}': {e}")
-            return # Exit if reading the Parquet fails
-    else:
-        print("No valid input file determined or file does not exist. Exiting without processing.")
-        return
-
-    # Initialize an output DataFrame
-    output_data = []
-
-    # Only proceed if df was successfully loaded
-    if df is not None:
-        for _, row in df.iterrows():
-            symbol = row['SYMBOL']
-            purpose = str(row['PURPOSE']) # Ensure purpose is a string
-            ex_date = row['EX-DATE']
-            
-            ratio, corp_action = None, None
-
-            # Calculate adjustment factor and corp action type
-            if "Bonus" in purpose:
-                ratio, corp_action = calculate_bonus_adj_factor(purpose)
-            elif "Face Value Split" in purpose:
-                ratio, corp_action = calculate_split_adj_factor(purpose)
-            
-            # Only add data if a valid corporate action and ratio were identified
-            if ratio and corp_action:
-                try:
-                    # Convert date to YYYY-MM-DD format, handling potential errors
-                    formatted_date = pd.to_datetime(ex_date, format='%d-%b-%Y').strftime('%Y-%m-%d')
-                    output_data.append({
-                        "symbol": symbol,
-                        "type": corp_action,
-                        "date": formatted_date,
-                        "ratio": ratio
-                    })
-                except ValueError as ve:
-                    print(f"Error parsing date '{ex_date}' for symbol '{symbol}': {ve}")
-                except Exception as e:
-                    print(f"An unexpected error occurred processing row for symbol '{symbol}': {e}")
-
-
-        # Convert output data to a DataFrame
-        output_df = pd.DataFrame(output_data)
-
-        # Save to the target file in JSON format
-        output_df.to_json(output_file, orient='records', indent=4)
-
-        print(f"\nCorporate action data processed and saved to {output_file}")
-        print(f"Total records processed: {len(df)}")
-        print(f"Total valid corporate actions extracted: {len(output_df)}")
-    else:
-        print("Dataframe 'df' was not created due to previous errors. No processing performed.")
-
-
-if __name__ == "__main__":
-    main()
-

nsedash/config.py

@@ -1,221 +0,0 @@
-"""
-nsedash.config
-==============
-Central configuration: directory layout, file paths, constants, index
-constituent lists, and the Recommendation enum.
-
-All paths derive from a single base directory (default: current working
-directory) and preserve the exact directory contract of nsedash.py v4.5:
-
-    <base_dir>/
-        processed/
-            nse_processed_data.csv
-            nse_summary_data.parquet
-            nse_dashboard_etflist.json
-            nse_dashboard_mclink.json
-            nse_dashboard_contents.json
-            nse_dashboard_config.json
-            recommender/
-            watchlists/
-            PortFolioEqtSummary.csv        (optional)
-            SecurityMaster.zip             (optional)
-        downloads/
-            nse_dashboard.log
-            nse_dashboard_metrics.log
-"""
-
-from __future__ import annotations
-
-import os
-from enum import Enum
-from pathlib import Path
-
-# ---------------------------------------------------------------------------
-# Runtime mode flags (set by CLI switches or HF Space environment)
-# ---------------------------------------------------------------------------
-
-#: True when running inside a Hugging Face Space (SPACE_ID is set) or
-#: when the ``-hf`` CLI flag was passed.
-HF_MODE = bool(int(os.environ.get("HF_MODE", "0")))
-
-#: False when the ``-noadmin`` CLI flag was passed.  Disables admin-only
-#: actions (script runner, raw data tools, etc.).
-ADMIN_MODE = bool(int(os.environ.get("ADMIN_MODE", "1")))
-
-#: HuggingFace dataset repo used for two-way data persistence.
-HF_DATASET_REPO = os.environ.get("NSEDASH_HF_DATASET", "Sam-max1/nsedash_data")
-
-# ---------------------------------------------------------------------------
-# Constants
-# ---------------------------------------------------------------------------
-
-DATA_RETENTION_DAYS = 730          # 2-year historical data window
-DEFAULT_REFRESH_RATE_MIN = 5       # live data polling interval (minutes)
-DEFAULT_VOLUME_CUTOFF = 10_000     # minimum average volume for volume analysis
-WATCHLIST_SLOTS = 20               # named watchlist slots
-GRID_COLUMNS = 5                   # dashboard grid columns (UI hint)
-AUTO_QTY_BUDGET = 100_000          # ₹ budget for the auto-quantity heuristic
-ANALYSIS_LIMIT = 50                # top-N rows for every analysis screen
-HTTP_TIMEOUT_SEC = 10              # network timeout (AUDIT FIX C-1: original had none)
-FETCH_MAX_WORKERS = 8              # concurrent symbol fetches (AUDIT FIX P-1)
-
-MARKET_TZ = "Asia/Kolkata"         # NSE market timezone (AUDIT FIX H-8)
-MARKET_OPEN = (9, 15)
-MARKET_CLOSE = (15, 30)
-
-REQUEST_HEADERS = {
-    'User-Agent': (
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        'AppleWebKit/537.36 (KHTML, like Gecko) '
-        'Chrome/125.0.0.0 Safari/537.36'
-    ),
-    'Accept-Language': 'en-US,en;q=0.9',
-    'Accept-Encoding': 'gzip, deflate, br',
-    'Accept': (
-        'text/html,application/xhtml+xml,application/xml;'
-        'q=0.9,image/webp,image/apng,*/*;q=0.8,'
-        'application/signed-exchange;v=b3;q=0.7'
-    ),
-    'Connection': 'keep-alive',
-    'Upgrade-Insecure-Requests': '1',
-}
-
-CSV_COLUMNS = [
-    'SYMBOL', 'SECURITY', 'TRAN_DATE',
-    'OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE',
-    'PREV_CL_PR', 'CLOSE_PRICE', 'NET_TRDVAL',
-    'NET_TRDQTY', 'TRADES', 'HI_52_WK', 'LO_52_WK',
-    'HI_DIFF', 'LO_DIFF'
-]
-
-CSV_DTYPES = {
-    'SYMBOL': 'category',
-    'SECURITY': 'category',
-    'OPEN_PRICE': 'float32',
-    'HIGH_PRICE': 'float32',
-    'LOW_PRICE': 'float32',
-    'PREV_CL_PR': 'float32',
-    'CLOSE_PRICE': 'float32',
-    'NET_TRDVAL': 'float32',
-    'NET_TRDQTY': 'float32',
-    'TRADES': 'float32',
-    'HI_52_WK': 'float32',
-    'LO_52_WK': 'float32',
-    'HI_DIFF': 'float32',
-    'LO_DIFF': 'float32',
-}
-
-
-class Recommendation(str, Enum):
-    """LSTM model recommendation values."""
-    BUY = "BUY"
-    SELL = "SELL"
-    HOLD = "HOLD"
-    NA = "N/A"
-
-
-# ---------------------------------------------------------------------------
-# Index Constituent Lists (retained verbatim from v4.5)
-# ---------------------------------------------------------------------------
-
-SENSEX_STOCKS = [
-    "ASIANPAINT", "AXISBANK", "BAJAJFINSV", "BAJFINANCE", "BHARTIARTL",
-    "BPCL", "DRREDDY", "HDFC", "HDFCBANK", "HEROMOTOCO",
-    "HINDUNILVR", "HCLTECH", "ICICIBANK", "INDUSINDBK", "INFY",
-    "ITC", "KOTAKBANK", "LT", "M&M", "MARUTI",
-    "NESTLEIND", "NTPC", "ONGC", "POWERGRID", "RELIANCE",
-    "SBIN", "SUNPHARMA", "TCS", "TATAMOTORS", "ULTRACEMCO"
-]
-
-NIFTY_STOCKS = [
-    "ADANIENT", "ADANIPORTS", "APOLLOHOSP", "ASIANPAINT", "AXISBANK",
-    "BAJAJ-AUTO", "BAJFINANCE", "BAJAJFINSV", "BEL", "BHARTIARTL",
-    "BPCL", "BRITANNIA", "CIPLA", "COALINDIA", "DRREDDY",
-    "EICHERMOT", "GRASIM", "HCLTECH", "HDFCLIFE", "HDFCBANK",
-    "HEROMOTOCO", "HINDALCO", "HINDUNILVR", "ICICIBANK", "INDUSINDBK",
-    "INFY", "ITC", "JIOFIN", "JSWSTEEL", "KOTAKBANK",
-    "LT", "M&M", "MARUTI", "NESTLEIND", "NTPC",
-    "ONGC", "POWERGRID", "RELIANCE", "SBIN", "SBILIFE",
-    "SHRIRAMFIN", "SUNPHARMA", "TATACONSUM", "TATAMOTORS", "TATASTEEL",
-    "TCS", "TECHM", "TITAN", "TRENT", "ULTRACEMCO", "WIPRO"
-]
-
-UTILITY_SCRIPTS = [
-    ("Download Daily Data", "nsedown.py"),
-    ("Process Corporate Actions", "nsecorpact.py"),
-    ("Process Daily Data", "nseprocessor.py"),
-    ("Process Max / Min", "nsesymhist.py"),
-    ("Download Live Data", "nsemaxlive.py"),
-]
-
-
-
-class Paths:
-    """All filesystem paths for a given base directory."""
-
-    def __init__(self, base_dir: str | os.PathLike | None = None, session_token: str | None = None):
-        self.base_dir = Path(base_dir or os.environ.get("NSEDASH_BASE_DIR") or Path.cwd())
-        self.session_token = session_token
-        self.data_dir = self.base_dir / 'processed'
-
-        # Isolate session files
-        if session_token and session_token != "admin":
-            self.watchlist_dir = self.data_dir / 'watchlists' / f"session_{session_token}"
-            self.state_file = self.data_dir / f"nse_dashboard_contents_{session_token}.json"
-            self.config_file = self.data_dir / f"nse_dashboard_config_{session_token}.json"
-            self.portfolio_file = self.data_dir / f"PortFolioEqtSummary_{session_token}.parquet"
-        else:
-            self.watchlist_dir = self.data_dir / 'watchlists'
-            self.state_file = self.data_dir / 'nse_dashboard_contents.json'
-            self.config_file = self.data_dir / 'nse_dashboard_config.json'
-            self.portfolio_file = self.data_dir / 'PortFolioEqtSummary.parquet'
-
-        self.log_dir = self.base_dir / 'downloads'
-        self.recommender_dir = self.data_dir / 'recommender'
-        self.aianalysis_dir = self.data_dir / 'aianalysis'
-
-        self.input_file = self.data_dir / 'nse_processed_data.parquet'
-        self.extract_file = self.data_dir / 'nse_summary_data.parquet'
-        self.etf_list_file = self.data_dir / 'nse_dashboard_etflist.json'
-        self.mc_link_file = self.data_dir / 'nse_dashboard_mclink.json'
-        self.log_file = self.log_dir / 'nse_dashboard.log'
-        self.metrics_file = self.log_dir / 'nse_dashboard_metrics.log'
-        self.security_master_zip = self.data_dir / 'SecurityMaster.zip'
-
-        wl = self.watchlist_dir
-        # Live analysis watchlists
-        self.max_vol_watchlist_file = wl / 'nse_dashboard_maxvol50.json'
-        self.max_price_watchlist_file = wl / 'nse_dashboard_maxprice50.json'
-        self.max_fall_watchlist_file = wl / 'nse_dashboard_maxfall50.json'
-        self.max_score_watchlist_file = wl / 'nse_dashboard_maxscore50.json'
-        self.max_return_files = {
-            "1W": wl / 'nse_dashboard_maxreturn1w50.json',
-            "1M": wl / 'nse_dashboard_maxreturn1m50.json',
-            "3M": wl / 'nse_dashboard_maxreturn3m50.json',
-        }
-        # Historical max/min price analysis watchlists
-        self.hist_price_files = {
-            ("1D", "Max"): wl / 'nse_dashboard_hist_maxprice1d50.json',
-            ("1D", "Min"): wl / 'nse_dashboard_hist_minprice1d50.json',
-            ("1W", "Max"): wl / 'nse_dashboard_hist_maxprice1w50.json',
-            ("1W", "Min"): wl / 'nse_dashboard_hist_minprice1w50.json',
-            ("1M", "Max"): wl / 'nse_dashboard_hist_maxprice1m50.json',
-            ("1M", "Min"): wl / 'nse_dashboard_hist_minprice1m50.json',
-            ("3M", "Max"): wl / 'nse_dashboard_hist_maxprice3m50.json',
-            ("3M", "Min"): wl / 'nse_dashboard_hist_minprice3m50.json',
-        }
-        # Historical volume analysis watchlists
-        self.hist_vol_files = {
-            "1D": wl / 'nse_dashboard_hist_maxvol1d50.json',
-            "1W": wl / 'nse_dashboard_hist_maxvol1w50.json',
-            "1M": wl / 'nse_dashboard_hist_maxvol1m50.json',
-            "3M": wl / 'nse_dashboard_hist_maxvol3m50.json',
-        }
-
-    def watchlist_slot(self, n: int) -> Path:
-        """Path for numbered watchlist slot n (1..WATCHLIST_SLOTS)."""
-        return self.watchlist_dir / f"nse_dashboard_watchlist{n}.json"
-
-    def ensure_dirs(self) -> None:
-        for d in (self.data_dir, self.data_dir / 'watchlists', self.watchlist_dir, self.log_dir, self.recommender_dir, self.aianalysis_dir):
-            d.mkdir(parents=True, exist_ok=True)

nsedash/core/dashboard_service.py

@@ -1,887 +0,0 @@
-"""
-nsedash.core.dashboard_service
-==============================
-Server-side state controller — the headless successor to v4.5's
-``StockDashboard``. All Tkinter dialogs become structured return payloads
-(``{"ok":..., "message":..., ...}``) the frontend renders as toasts/modals;
-all card widgets become JSON card payloads rendered by the browser.
-
-State parity with v4.5:
-    cells (ordered stock cards with Quantity/Price/current_cmp/selection),
-    stocks universe keyed "SYMBOL - SECURITY", selected_date,
-    current_watchlist_name, watchlist_modified, real_time_active,
-    live_data_source(+preference), live_refresh_rate,
-    volume_cutoff_threshold, selected_symbols_for_watchlist.
-"""
-
-from __future__ import annotations
-
-import logging
-import threading
-from datetime import datetime, timedelta
-from pathlib import Path
-from zoneinfo import ZoneInfo
-
-import pandas as pd
-
-from ..config import (AUTO_QTY_BUDGET, DEFAULT_REFRESH_RATE_MIN,
-                      DEFAULT_VOLUME_CUTOFF, GRID_COLUMNS, MARKET_CLOSE,
-                      MARKET_OPEN, MARKET_TZ, NIFTY_STOCKS, SENSEX_STOCKS,
-                      WATCHLIST_SLOTS, Paths)
-from .portfolio import compute_portfolio_summary
-from .stock_data import StockData
-from .watchlists import WatchlistStore
-
-CHART_WINDOW = 90  # points displayed, parity with v4.5 update_graph
-DMA_WINDOWS = (9, 20, 50, 200)
-
-
-class Cell:
-    """One stock card's server-side state."""
-
-    __slots__ = ("symbol", "quantity", "price", "current_cmp", "prev_close",
-                 "selected", "live", "recommendation")
-
-    def __init__(self, symbol: str, quantity: float = 0.0, price: float = 0.0):
-        self.symbol = symbol
-        self.quantity = float(quantity)
-        self.price = float(price)
-        self.current_cmp = 0.0
-        self.prev_close = 0.0
-        self.selected = False
-        self.live: dict | None = None
-        self.recommendation: dict | None = None
-
-    def get_data(self) -> dict:
-        """Parity with StockCell.get_data()."""
-        return {"SYMBOL": self.symbol, "Quantity": self.quantity,
-                "Price": self.price}
-
-
-class DashboardService:
-
-    def __init__(self, paths: Paths, hf_sync=None):
-        self.paths = paths
-        self._lock = threading.RLock()
-        self.stock_data = StockData(paths)
-        self.watchlists = WatchlistStore(paths, hf_sync=hf_sync)
-
-        self.cells: list[Cell] = []
-        self.stocks: dict[str, pd.Series] = {}
-        self.COLUMNS = GRID_COLUMNS
-        self.real_time_active = False
-        self.last_refresh = None
-        self.live_data_source = "N/A"
-        self.live_data_source_preference: str | None = None
-        self.live_refresh_rate = DEFAULT_REFRESH_RATE_MIN
-        self.watchlist_modified = False
-        self.volume_cutoff_threshold = DEFAULT_VOLUME_CUTOFF
-        self.selected_date = self.stock_data.last_date
-        self.watchlist_names = self.watchlists.load_watchlist_names()
-
-        cfg = self.watchlists.load_config()
-        self.current_watchlist_name = cfg.get(
-            'current_watchlist_name', "Default List"
-        )
-        self._mc_links_cache: dict | None = None
-        self.load_universe()
-        self.load_dashboard_state_by_name(self.current_watchlist_name)
-
-    # ==================================================================
-    # Universe / date data
-    # ==================================================================
-
-    def load_universe(self) -> None:
-        """Build the stocks dict for the selected date (v4.5 load_data)."""
-        data = self.stock_data.get_data_for_date(self.selected_date)
-        with self._lock:
-            self.stocks = {
-                f"{row['SYMBOL']} - {row['SECURITY']}": row
-                for _, row in data.iterrows()
-            }
-            # AUDIT FIX P-8: symbol → key index removes the O(cells×stocks)
-            # linear prefix scans v4.5 performed throughout.
-            self._symbol_key_index = {
-                str(row['SYMBOL']): key for key, row in self.stocks.items()
-            }
-            self._mc_links_cache = None
-
-    def key_for_symbol(self, symbol: str) -> str | None:
-        return self._symbol_key_index.get(symbol)
-
-    def mc_links(self) -> dict:
-        if self._mc_links_cache is None:
-            self._mc_links_cache = self.stock_data.load_mc_link_data()
-        return self._mc_links_cache
-
-    # ==================================================================
-    # Card payloads
-    # ==================================================================
-
-    @staticmethod
-    def _chart_payload(price_history, volume_history) -> dict | None:
-        """
-        90-day chart series with 9/20/50/200 DMA computed over the full
-        history (parity with StockCell.update_graph). Sent to the browser
-        for client-side rendering.
-        """
-        if not price_history or len(price_history) < 2:
-            return None
-        dates = [x[0] for x in price_history]
-        prices = [float(x[1]) for x in price_history]
-        volumes = [float(x[1]) for x in (volume_history or [])]
-        prices_series = pd.Series(prices)
-        start = max(0, len(dates) - CHART_WINDOW)
-
-        def fmt_date(d):
-            if hasattr(d, 'strftime'):
-                return d.strftime('%Y-%m-%d')
-            return str(d)[:10]
-
-        payload = {
-            "dates": [fmt_date(d) for d in dates[start:]],
-            "prices": prices[start:],
-            "volumes": volumes[start:] if volumes else [],
-        }
-        for w in DMA_WINDOWS:
-            if len(prices_series) >= w:
-                dma = prices_series.rolling(window=w).mean()
-                payload[f"dma_{w}"] = [
-                    None if pd.isna(v) else round(float(v), 2)
-                    for v in dma.iloc[start:]
-                ]
-            else:
-                payload[f"dma_{w}"] = None
-        return payload
-
-    def _cell_payload(self, cell: Cell, include_chart: bool = True) -> dict:
-        """Full card payload (historic + live + MC + P&L inputs)."""
-        key = self.key_for_symbol(cell.symbol)
-        row = self.stocks.get(key) if key else None
-        mc = self.mc_links().get(cell.symbol, {})
-        payload = {
-            "symbol": cell.symbol,
-            "quantity": cell.quantity,
-            "price": cell.price,
-            "current_cmp": cell.current_cmp,
-            "selected": cell.selected,
-            "live": cell.live,
-            "recommendation": cell.recommendation,
-            "mc": {
-                "mcap": mc.get('MC_MCAP', 0.0),
-                "fv": mc.get('MC_FV', 0.0),
-                "score": mc.get('MC_SCORE', 0.0),
-                "ath": mc.get('MC_ATH', 0.0),
-                "atl": mc.get('MC_ATL', 0.0),
-                "link": mc.get('MC_LINK'),
-            },
-        }
-        if row is not None:
-            prev_price = float(row['PREV_CL_PR'])
-            curr_price = float(row['CLOSE_PRICE'])
-            try:
-                daily_change = (
-                    (curr_price / prev_price - 1) * 100 if prev_price != 0 else 0.0
-                )
-            except (ValueError, TypeError):
-                daily_change = 0.0
-            vc = row.get('VOLUME_CHANGES') or []
-            padded = ([0.0] * (5 - len(vc)) + list(vc)) if len(vc) < 5 else list(vc)[:5]
-            payload.update({
-                "security": str(row['SECURITY']),
-                "tran_date": row['TRAN_DATE'].strftime('%d %b %Y'),
-                "prev_close": prev_price,
-                "close_price": curr_price,
-                "hi_52wk": float(row['HI_52_WK']),
-                "hi_diff": float(row['HI_DIFF']),
-                "lo_52wk": float(row['LO_52_WK']),
-                "lo_diff": float(row['LO_DIFF']),
-                "changes": {
-                    "1D": daily_change,
-                    "1W": float(row.get('WEEK_CHANGE', 0) or 0),
-                    "1M": float(row.get('MONTH_CHANGE', 0) or 0),
-                    "3M": float(row.get('THREE_MONTH_CHANGE', 0) or 0),
-                },
-                # Display order parity: reversed padded list, labelled D1..D5
-                "volume_changes": padded[::-1],
-            })
-            if include_chart:
-                payload["chart"] = self._chart_payload(
-                    row.get('PRICE_HISTORY'), row.get('VOLUME_HISTORY')
-                )
-        return payload
-
-    def board_payload(self, include_charts: bool = True) -> dict:
-        """Complete dashboard snapshot for the frontend."""
-        with self._lock:
-            return {
-                "banner": {
-                    "mode": "live" if self.real_time_active else "historic",
-                    "source": self.live_data_source,
-                    "watchlist": self.current_watchlist_name,
-                    "refresh_rate": self.live_refresh_rate,
-                },
-                "selected_date": str(self.selected_date)[:10],
-                "last_data_date": str(self.stock_data.last_date)[:10],
-                "watchlist_modified": self.watchlist_modified,
-                "watchlist_names": self.watchlist_names,
-                "volume_cutoff": self.volume_cutoff_threshold,
-                "columns": self.COLUMNS,
-                "cells": [
-                    self._cell_payload(c, include_charts) for c in self.cells
-                ],
-                "portfolio": self.portfolio_summary(),
-                "search_keys": sorted(self.stocks.keys()),
-                "trading_hour": self.is_trading_hour(),
-                "indices": self.get_index_quotes(),
-            }
-
-    # ==================================================================
-    # Cell management (add/remove/delete-all/inputs/selection)
-    # ==================================================================
-
-    def add_stock(self, selected: str | None = None, quantity: float = 0.0,
-                  price: float = 0.0, save_state: bool = True) -> Cell | None:
-        with self._lock:
-            if not selected or selected not in self.stocks:
-                logging.warning(
-                    f"Skipping addition: stock '{selected}' not found in data."
-                )
-                return None
-            row = self.stocks[selected]
-            symbol = str(row['SYMBOL'])
-            # Auto-quantity heuristic (v4.5 M-8): only on addition.
-            if float(quantity) == 0.0 and float(price) == 0.0:
-                curr_price = float(row.get('CLOSE_PRICE', 0) or 0)
-                if curr_price > 0:
-                    price = curr_price
-                    quantity = int(AUTO_QTY_BUDGET / curr_price)
-            cell = Cell(symbol, quantity, price)
-            cell.current_cmp = float(row.get('CLOSE_PRICE', 0) or 0)
-            cell.prev_close = float(row.get('PREV_CL_PR', 0) or 0)
-            self.cells.append(cell)
-            if save_state:
-                self.save_dashboard_state()
-            return cell
-
-    def _find_cell(self, symbol: str) -> Cell | None:
-        return next((c for c in self.cells if c.symbol == symbol), None)
-
-    def remove_stock(self, symbol: str) -> bool:
-        with self._lock:
-            cell = self._find_cell(symbol)
-            if not cell:
-                return False
-            self.cells.remove(cell)
-            self.save_dashboard_state()
-            return True
-
-    def delete_all_stocks(self, mark_modified: bool = True) -> None:
-        with self._lock:
-            self.cells.clear()
-            if mark_modified:
-                self.watchlist_modified = True
-            self.save_dashboard_state()
-
-    def update_cell_inputs(self, symbol: str, quantity: float,
-                           price: float) -> dict:
-        """Quantity/Price entry change (v4.5 _on_entry_change)."""
-        with self._lock:
-            cell = self._find_cell(symbol)
-            if not cell:
-                return {"ok": False,
-                        "message": f"Stock {symbol} not on the board."}
-            try:
-                cell.quantity = float(quantity)
-                cell.price = float(price)
-            except (TypeError, ValueError):
-                cell.quantity = 0.0
-                cell.price = 0.0
-                return {"ok": False,
-                        "message": "Please enter valid numbers for "
-                                   "Quantity and Price."}
-            self.watchlist_modified = True
-            return {"ok": True, "portfolio": self.portfolio_summary()}
-
-    def toggle_selection(self, symbol: str, selected: bool) -> None:
-        with self._lock:
-            cell = self._find_cell(symbol)
-            if cell:
-                cell.selected = bool(selected)
-
-    @property
-    def selected_symbols_for_watchlist(self) -> list[str]:
-        return [c.symbol for c in self.cells if c.selected]
-
-    # ==================================================================
-    # Live data
-    # ==================================================================
-
-    def is_trading_hour(self) -> bool:
-        """True within NSE market hours (IST). AUDIT FIX H-8: v4.5 used the
-        server's local clock; this uses Asia/Kolkata explicitly."""
-        now = datetime.now(ZoneInfo(MARKET_TZ))
-        if not (0 <= now.weekday() <= 4):
-            return False
-        market_open = now.replace(hour=MARKET_OPEN[0], minute=MARKET_OPEN[1],
-                                  second=0, microsecond=0)
-        market_close = now.replace(hour=MARKET_CLOSE[0], minute=MARKET_CLOSE[1],
-                                   second=0, microsecond=0)
-        return market_open <= now <= market_close
-
-    def apply_live_quotes(self, quotes: list[dict], source: str) -> int:
-        """Apply fetched quotes to matching cells; returns update count."""
-        updated = 0
-        with self._lock:
-            self.live_data_source = source
-            by_symbol = {q["SYMBOL"]: q for q in quotes}
-            for cell in self.cells:
-                q = by_symbol.get(cell.symbol)
-                if not q:
-                    continue
-                curr = float(q['CLOSE_PRICE'])
-                prev = float(q['PREV_CL_PR'])
-                curr_vol = float(q.get('CURR_VOL', 0) or 0)
-                prev_vol = float(q.get('PREV_VOL', 0) or 0)
-                cell.current_cmp = curr
-                cell.live = {
-                    "price": curr,
-                    "pct_change": (curr / prev - 1) * 100 if prev else 0,
-                    "day_high": float(q.get('DAY_HIGH', 0) or 0),
-                    "day_low": float(q.get('DAY_LOW', 0) or 0),
-                    "vol_change": (
-                        (curr_vol / prev_vol - 1) * 100 if prev_vol else 0.0
-                    ),
-                    "updated_at": datetime.now(ZoneInfo(MARKET_TZ)).strftime('%d %b %Y | %H:%M:%S IST'),
-                }
-                updated += 1
-            self.last_refresh = datetime.now(ZoneInfo(MARKET_TZ))
-        return updated
-
-    def get_index_quotes(self) -> dict[str, dict]:
-        """Fetch live index quotes for NIFTY and SENSEX with 15s cache."""
-        import time
-        now = time.time()
-        if hasattr(self, '_index_cache') and self._index_cache and (now - getattr(self, '_index_cache_time', 0) < 15):
-            return self._index_cache
-        try:
-            fetcher = LiveFetcher(session=self.session)
-            quotes = fetcher.fetch_index_quotes()
-            if quotes:
-                self._index_cache = quotes
-                self._index_cache_time = now
-                return quotes
-        except Exception as e:
-            logging.warning(f"Error fetching index quotes: {e}")
-        return getattr(self, '_index_cache', {})
-
-    def load_live_data(self) -> dict:
-        """One live refresh cycle for all displayed stocks, with post-market caching."""
-        symbols = [c.symbol for c in self.cells]
-        if not symbols:
-            return {"ok": True, "updated": 0, "source": self.live_data_source}
-        
-        now = datetime.now(ZoneInfo(MARKET_TZ))
-        cache_file = self.paths.data_dir / 'nse_live_quotes_cache.json'
-        is_live = self.is_trading_hour()
-
-        # If market is closed, check cache first
-        if not is_live:
-            # Helper to calculate last session date
-            def get_last_session_date(dt: datetime) -> datetime.date:
-                curr = dt
-                if curr.weekday() < 5:
-                    if curr.hour < 15 or (curr.hour == 15 and curr.minute < 30):
-                        curr -= timedelta(days=1)
-                while curr.weekday() >= 5:
-                    curr -= timedelta(days=1)
-                return curr.date()
-
-            last_session_now = get_last_session_date(now)
-
-            if cache_file.exists():
-                try:
-                    import json
-                    with open(cache_file, 'r') as f:
-                        cache_data = json.load(f)
-                    cache_time = datetime.fromisoformat(cache_data["timestamp"]).astimezone(ZoneInfo(MARKET_TZ))
-                    last_session_cache = get_last_session_date(cache_time)
-
-                    # Cache is valid if it's from the same last trading session and was pulled at/after 15:30
-                    is_same_session = last_session_now == last_session_cache
-                    is_post_close = cache_time.hour > 15 or (cache_time.hour == 15 and cache_time.minute >= 30)
-
-                    if is_same_session and is_post_close:
-                        logging.info("Loading live data from post-market cache.")
-                        quotes = cache_data["quotes"]
-                        # Filter quotes to only keep those currently on the board
-                        board_symbols = set(symbols)
-                        filtered_quotes = [q for q in quotes if q["SYMBOL"] in board_symbols]
-                        updated = self.apply_live_quotes(filtered_quotes, cache_data["source"])
-                        return {
-                            "ok": True,
-                            "updated": updated,
-                            "source": cache_data["source"],
-                            "cached": True,
-                            "portfolio": self.portfolio_summary(),
-                            "message": "Loaded live data from post-market cache."
-                        }
-                except Exception as e:
-                    logging.warning(f"Failed to read live quote cache: {e}")
-
-            logging.info("No valid post-market cache found. Pulling live data once after market hours.")
-
-        # If market is open, or cache is missing/invalid when closed:
-        sess_id = self.paths.state_file.stem.split('_')[-1] if '_' in self.paths.state_file.stem else 'admin'
-        logging.info("[LIVE] Starting live data pull cycle — %d symbol(s) (Source: %s, Session: %s)",
-                     len(symbols), self.live_data_source_preference or 'Google', sess_id)
-        quotes, source = self.stock_data.fetch_real_time_data(
-            symbols, preferred_source=self.live_data_source_preference or "Google"
-        )
-        if not quotes:
-            logging.error("Failed to fetch live data from any source.")
-            return {"ok": False, "updated": 0, "source": "None",
-                    "message": "Failed to fetch live data from any source."}
-
-        # Save to cache
-        try:
-            import json
-            cache_payload = {
-                "timestamp": now.isoformat(),
-                "source": source,
-                "quotes": quotes
-            }
-            with open(cache_file, 'w') as f:
-                json.dump(cache_payload, f, indent=4)
-            logging.info("Successfully updated live quote cache.")
-        except Exception as e:
-            logging.warning(f"Failed to write live quote cache: {e}")
-
-        updated = self.apply_live_quotes(quotes, source)
-        return {"ok": True, "updated": updated, "source": source,
-                "portfolio": self.portfolio_summary()}
-
-    def start_live(self, source: str, refresh_rate: int | None) -> dict:
-        """Enable live mode (v4.5 toggle_live_data + ask_live_data_source)."""
-        with self._lock:
-            if source not in ("Google", "Yahoo"):
-                self.real_time_active = False
-                return {"ok": False,
-                        "message": "Live data not started as source "
-                                   "selection was cancelled."}
-            try:
-                rate = int(refresh_rate) if refresh_rate else DEFAULT_REFRESH_RATE_MIN
-            except (TypeError, ValueError):
-                rate = DEFAULT_REFRESH_RATE_MIN
-            self.live_refresh_rate = rate if rate > 0 else DEFAULT_REFRESH_RATE_MIN
-            self.live_data_source_preference = source
-            self.real_time_active = True
-            self.last_refresh = datetime.now()
-            sess_id = self.paths.state_file.stem.split('_')[-1] if '_' in self.paths.state_file.stem else 'admin'
-            print(f"[LIVE] Activating live data pull: source={source}, rate={self.live_refresh_rate}m (Session: {sess_id})", flush=True)
-        return {"ok": True, "refresh_rate": self.live_refresh_rate}
-
-    def stop_live(self) -> dict:
-        """Disable live mode and restore historic data (v4.5 behaviour)."""
-        with self._lock:
-            self.real_time_active = False
-            self.last_refresh = None
-            for cell in self.cells:
-                cell.live = None
-        logging.info("Live data updates stopped. Displaying historic data.")
-        self.load_historic_data()
-        return {"ok": True}
-
-    def load_historic_data(self) -> dict:
-        """Refresh all displayed cells from historic data for selected_date."""
-        try:
-            data = self.stock_data.get_data_for_date(self.selected_date)
-        except Exception as e:
-            logging.error(f"Failed to refresh historic data: {e}")
-            return {"ok": False, "message": str(e)}
-        with self._lock:
-            historic = {str(r['SYMBOL']): r for _, r in data.iterrows()}
-            removed = []
-            for cell in list(self.cells):
-                row = historic.get(cell.symbol)
-                if row is not None:
-                    cell.current_cmp = float(row['CLOSE_PRICE'])
-                    cell.prev_close = float(row['PREV_CL_PR'])
-                else:
-                    logging.warning(
-                        f"Historic data not found for {cell.symbol}. Removing cell."
-                    )
-                    self.cells.remove(cell)
-                    removed.append(cell.symbol)
-            if removed:
-                self.save_dashboard_state()
-        return {"ok": True, "removed": removed}
-
-    # ==================================================================
-    # Portfolio summary
-    # ==================================================================
-
-    def portfolio_summary(self) -> dict:
-        cells = []
-        for c in self.cells:
-            key = self.key_for_symbol(c.symbol)
-            prev_close = 0.0
-            if key and 'PREV_CL_PR' in self.stocks[key]:
-                prev_close = float(self.stocks[key]['PREV_CL_PR'])
-            elif c.current_cmp != 0.0:
-                prev_close = c.current_cmp
-            cells.append({
-                "Quantity": c.quantity, "Price": c.price,
-                "current_cmp": c.current_cmp, "prev_close": prev_close,
-            })
-        return compute_portfolio_summary(cells)
-
-    # ==================================================================
-    # Watchlists
-    # ==================================================================
-
-    def save_dashboard_state(self) -> None:
-        self.watchlists.save_dashboard_state([c.get_data() for c in self.cells])
-
-    def _load_cells_from_entries(self, entries: list[dict],
-                                 live_data_map: dict | None = None) -> int:
-        """Rebuild the board from watchlist entries; returns count loaded."""
-        self.delete_all_stocks(mark_modified=False)
-        loaded = 0
-        for stock_info in entries:
-            symbol = stock_info.get('SYMBOL')
-            key = self.key_for_symbol(symbol)
-            if key:
-                cell = self.add_stock(
-                    key,
-                    quantity=stock_info.get('Quantity', 0.0),
-                    price=stock_info.get('Price', 0.0),
-                    save_state=False,
-                )
-                if cell and live_data_map and symbol in live_data_map:
-                    self.apply_live_quotes(
-                        [live_data_map[symbol]], self.live_data_source
-                    )
-                loaded += 1 if cell else 0
-        return loaded
-
-    def load_dashboard_state_by_name(self, watchlist_name: str) -> dict:
-        """Restore the dashboard from a named watchlist (v4.5 parity)."""
-        try:
-            file_path = None
-            if watchlist_name == "Default List":
-                file_path = self.paths.state_file
-            else:
-                for i, name in enumerate(self.watchlist_names):
-                    if name == watchlist_name:
-                        file_path = self.paths.watchlist_slot(i + 1)
-                        break
-            if not file_path or not file_path.exists():
-                logging.warning(
-                    f"Watchlist file not found for '{watchlist_name}'."
-                )
-                self.delete_all_stocks(mark_modified=False)
-                return {"ok": False, "message": "Watchlist file not found."}
-            _name, entries = self.watchlists.load_symbols_from_json(file_path)
-            self._load_cells_from_entries(entries)
-            with self._lock:
-                self.watchlist_modified = False
-                self.current_watchlist_name = watchlist_name
-            return {"ok": True, "loaded": len(self.cells)}
-        except Exception as e:
-            logging.error(
-                f"Failed to load dashboard state for '{watchlist_name}': {e}"
-            )
-            self.delete_all_stocks(mark_modified=False)
-            self.current_watchlist_name = "Default List"
-            return {"ok": False, "message": f"Failed to load dashboard: {e}"}
-
-    def load_watchlist(self, watchlist_num: int) -> dict:
-        """Load a numbered watchlist slot (v4.5 load_watchlist)."""
-        try:
-            file_path = self.paths.watchlist_slot(watchlist_num)
-            name, entries = self.watchlists.load_symbols_from_json(file_path)
-            if not entries:
-                self.delete_all_stocks(mark_modified=False)
-                self.current_watchlist_name = "Default List"
-                return {"ok": False,
-                        "message": f"Watchlist {watchlist_num} is empty."}
-            if not name:
-                name = f"Watchlist {watchlist_num}"
-            self._load_cells_from_entries(entries)
-            with self._lock:
-                self.current_watchlist_name = name
-                self.watchlist_modified = False
-            self.save_dashboard_state()
-            return {"ok": True, "name": name, "loaded": len(self.cells)}
-        except Exception as e:
-            self.current_watchlist_name = "Default List"
-            return {"ok": False, "message": f"Failed to load watchlist: {e}"}
-
-    def load_watchlist_from_file(self, file_path: Path) -> dict:
-        """Load a watchlist from an arbitrary JSON file (v4.5 parity,
-        including the live-quote prefetch when live mode is active)."""
-        try:
-            target_path = Path(file_path)
-            if not target_path.exists():
-                global_path = self.paths.data_dir / 'watchlists' / target_path.name
-                if global_path.exists():
-                    target_path = global_path
-
-            name, entries = self.watchlists.load_symbols_from_json(target_path)
-            if not entries:
-                return {"ok": False,
-                        "message": f"The watchlist '{target_path.stem}' is empty or file not found."}
-            if not name:
-                name = target_path.stem
-            live_map: dict = {}
-            if self.real_time_active:
-                symbols = [e['SYMBOL'] for e in entries]
-                quotes, source = self.stock_data.fetch_real_time_data(
-                    symbols,
-                    preferred_source=self.live_data_source_preference or "Google",
-                )
-                live_map = {q['SYMBOL']: q for q in quotes}
-                self.live_data_source = source
-            self._load_cells_from_entries(entries, live_map)
-            with self._lock:
-                self.current_watchlist_name = name
-            self.save_dashboard_state()
-            return {"ok": True, "name": name, "loaded": len(self.cells),
-                    "message": f"Successfully loaded {len(self.cells)} "
-                               f"stocks from '{name}'."}
-        except Exception as e:
-            logging.error(f"Error loading custom watchlist: {e}")
-            return {"ok": False,
-                    "message": f"Failed to load custom watchlist: {e}"}
-
-    def save_dashboard(self) -> dict:
-        """Save the board to the currently loaded named slot."""
-        if not self.cells:
-            return {"ok": False, "message": "No stocks to save."}
-        try:
-            idx = self.watchlist_names.index(self.current_watchlist_name)
-        except ValueError:
-            return {"ok": False, "needs_save_as": True,
-                    "message": "Current view is not a saved watchlist. "
-                               "Use 'Save as Watchlist'."}
-        try:
-            self.watchlists.save_slot(
-                idx + 1, self.current_watchlist_name,
-                [c.get_data() for c in self.cells],
-            )
-            self.watchlist_modified = False
-            self.watchlists.save_config(self.current_watchlist_name)
-            return {"ok": True,
-                    "message": f"Dashboard saved to: "
-                               f"{self.current_watchlist_name}"}
-        except Exception as e:
-            return {"ok": False, "message": f"Failed to save dashboard: {e}"}
-
-    def save_as_watchlist(self, watchlist_num: int, name: str) -> dict:
-        """Save the board to a specific slot under a new name."""
-        if not self.cells:
-            return {"ok": False, "message": "No stocks to save"}
-        if not (1 <= int(watchlist_num) <= WATCHLIST_SLOTS):
-            return {"ok": False,
-                    "message": f"Slot must be 1..{WATCHLIST_SLOTS}."}
-        name = (name or "").strip() or f"Watchlist {watchlist_num}"
-        try:
-            self.watchlists.save_slot(
-                int(watchlist_num), name, [c.get_data() for c in self.cells]
-            )
-            self.current_watchlist_name = name
-            self.watchlist_names = self.watchlists.load_watchlist_names()
-            self.watchlist_modified = False
-            self.watchlists.save_config(self.current_watchlist_name)
-            return {"ok": True, "message": f"Saved as {name}"}
-        except Exception as e:
-            return {"ok": False, "message": f"Failed to save watchlist: {e}"}
-
-    def add_selected_to_watchlist(self, watchlist_num: int) -> dict:
-        """Merge checkbox-selected symbols into a slot (v4.5 parity,
-        including clearing the selections afterwards)."""
-        selected = self.selected_symbols_for_watchlist
-        if not selected:
-            return {"ok": False,
-                    "message": "Please select at least one stock using "
-                               "the checkboxes."}
-        try:
-            name, added = self.watchlists.add_symbols_to_slot(
-                int(watchlist_num), selected
-            )
-        except Exception as e:
-            return {"ok": False,
-                    "message": f"Failed to add stocks to watchlist: {e}"}
-        with self._lock:
-            for c in self.cells:
-                c.selected = False
-        if added > 0:
-            self.watchlist_modified = True
-            self.watchlist_names = self.watchlists.load_watchlist_names()
-            return {"ok": True,
-                    "message": f"Added {added} stock(s) to '{name}'."}
-        return {"ok": True,
-                "message": "All selected stocks are already in the "
-                           "chosen watchlist."}
-
-    # ==================================================================
-    # Index / analysis board loads
-    # ==================================================================
-
-    def load_index_stocks(self, index_name: str) -> dict:
-        stock_list = {"SENSEX": SENSEX_STOCKS, "NIFTY": NIFTY_STOCKS}.get(index_name)
-        if stock_list is None:
-            return {"ok": False, "message": f"Unknown index: {index_name}"}
-        self.delete_all_stocks(mark_modified=False)
-        for symbol in stock_list:
-            key = self.key_for_symbol(symbol)
-            if key:
-                self.add_stock(key, save_state=False)
-        self.current_watchlist_name = f"{index_name} Index"
-        self.save_dashboard_state()
-        return {"ok": True, "loaded": len(self.cells)}
-
-    def display_analysis_result(self, symbols: list[str],
-                                json_name: str, cached: bool) -> dict:
-        """Common tail of every analysis flow: rebuild board from symbols."""
-        if not symbols:
-            return {"ok": False, "message": "No stock data available for filter."}
-        self.delete_all_stocks(mark_modified=False)
-        for symbol in symbols:
-            key = self.key_for_symbol(symbol)
-            if key:
-                self.add_stock(key, save_state=False)
-        self.save_dashboard_state()
-        self.current_watchlist_name = json_name.replace(" Portfolio", "")
-        message = (
-            f"Loaded {len(self.cells)} stocks from saved file."
-            if cached else
-            f"Displayed {len(self.cells)} stocks after calculation."
-        )
-        return {"ok": True, "message": message, "loaded": len(self.cells)}
-
-    def load_portfolio_holdings(self, holdings: list[dict],
-                                unmapped: list[str], broker_name: str = "ICICIDirect") -> dict:
-        """Board rebuild after portfolio import, automatically saving to Watchlist 2."""
-        if not holdings:
-            return {"ok": False, "message": "No stocks mapped."}
-        with self._lock:
-            self.delete_all_stocks(mark_modified=False)
-            loaded = 0
-            for h in holdings:
-                key = self.key_for_symbol(h['SYMBOL'])
-                if key:
-                    self.add_stock(key, quantity=h['Quantity'],
-                                   price=h['Price'], save_state=False)
-                    loaded += 1
-            self.save_dashboard_state()
-            
-            # Auto-save to watchlist slot 2 and rename it
-            watchlist_name = f"{broker_name} Portfolio"
-            self.watchlists.save_slot(
-                2, watchlist_name, [c.get_data() for c in self.cells]
-            )
-            self.current_watchlist_name = watchlist_name
-            self.watchlist_names = self.watchlists.load_watchlist_names()
-            self.watchlist_modified = False
-            self.watchlists.persist_config() if hasattr(self.watchlists, 'persist_config') else self.watchlists.save_config(self.current_watchlist_name)
-            
-        msg = f"Successfully loaded {loaded} stocks from {broker_name}."
-        if unmapped:
-            msg += f" Could not map: {', '.join(str(u) for u in unmapped)}"
-        return {"ok": True, "loaded": loaded, "unmapped": unmapped,
-                "message": msg}
-
-    # ==================================================================
-    # Date change / refresh / export / config
-    # ==================================================================
-
-    def change_date(self, new_date_str: str) -> dict:
-        """v4.5 on_date_change: reload universe and update board cells."""
-        try:
-            new_date = datetime.strptime(new_date_str, '%Y-%m-%d').date()
-        except ValueError:
-            return {"ok": False, "message": "Invalid date format (YYYY-MM-DD)."}
-        if new_date > datetime.now().date():
-            return {"ok": False, "message": "Cannot select future dates."}
-        old_date = self.selected_date
-        self.selected_date = new_date
-        try:
-            self.load_universe()
-        except ValueError:
-            self.selected_date = old_date
-            return {"ok": False,
-                    "message": f"No stock data available for {new_date}."}
-        except Exception as e:
-            self.selected_date = old_date
-            return {"ok": False,
-                    "message": f"Failed to load data for {new_date}: {e}"}
-        with self._lock:
-            removed = []
-            for cell in list(self.cells):
-                key = self.key_for_symbol(cell.symbol)
-                if key:
-                    row = self.stocks[key]
-                    cell.current_cmp = float(row['CLOSE_PRICE'])
-                    cell.prev_close = float(row['PREV_CL_PR'])
-                else:
-                    self.cells.remove(cell)
-                    removed.append(cell.symbol)
-            if removed:
-                self.save_dashboard_state()
-        return {"ok": True,
-                "message": f"Dashboard updated for {self.selected_date}.",
-                "removed": removed}
-
-    def refresh_dashboard(self) -> dict:
-        """v4.5 refresh_dashboard: reload CSV + rebuild universe + re-add."""
-        stopped_live = False
-        if self.real_time_active:
-            self.stop_live()
-            stopped_live = True
-        try:
-            self.stock_data.load_data()
-        except Exception as e:
-            return {"ok": False,
-                    "message": f"Failed to reload data: {e}\n"
-                               "Dashboard might not be fully updated."}
-        current = {c.symbol: c.get_data() for c in self.cells}
-        try:
-            self.load_universe()
-            self.delete_all_stocks(mark_modified=False)
-            reloaded = 0
-            for symbol, old in current.items():
-                key = self.key_for_symbol(symbol)
-                if key:
-                    self.add_stock(key, quantity=old['Quantity'],
-                                   price=old['Price'], save_state=False)
-                    reloaded += 1
-            self.save_dashboard_state()
-            msg = f"Data reloaded. {reloaded} stocks re-displayed."
-            if stopped_live:
-                msg = "Live data updates have been paused for dashboard " \
-                      "refresh. " + msg
-            return {"ok": True, "message": msg, "reloaded": reloaded}
-        except Exception as e:
-            return {"ok": False,
-                    "message": f"An error occurred while updating the "
-                               f"dashboard: {e}"}
-
-    def export_to_csv(self) -> dict:
-        """Export cell data to a timestamped CSV in downloads/."""
-        try:
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            filename = self.paths.log_dir / f"stock_data_{timestamp}.csv"
-            pd.DataFrame([c.get_data() for c in self.cells]).to_csv(
-                filename, index=False
-            )
-            return {"ok": True, "file": str(filename),
-                    "message": f"Data exported to: {filename}"}
-        except Exception as e:
-            logging.error(f"Failed to export data to CSV: {e}")
-            return {"ok": False,
-                    "message": "Failed to export data. Check logs."}
-
-    def persist_config(self) -> None:
-        self.watchlists.save_config(self.current_watchlist_name)

nsedash/core/hf_sync.py

@@ -1,338 +0,0 @@
-"""
-nsedash.core.hf_sync
-====================
-Two-way synchronisation between the local ``processed/`` directory and
-the Hugging Face dataset ``Sam-max1/nsedash_data``.
-
-Active ONLY when ``SPACE_ID`` is set in the environment (i.e. inside a
-Hugging Face Space).  In local/dev mode this module is a complete no-op
-so that local runs are unaffected.
-
-Key design decisions
----------------------
-* **Startup pull** — ``pull_on_startup()`` is called *before* Flask starts
-  and *before* ``DashboardService`` is created so that processed data is
-  fresh on every Space restart.  It is synchronous (blocks startup) to
-  guarantee correctness.
-* **Async push** — every local file write triggers ``push_file_async()``
-  which queues the upload on a background daemon thread.  The main request
-  path is never blocked.
-* **3-second debounce** — a timer is reset on each push call for the same
-  path so that rapid sequential writes (e.g. saving all watchlist slots at
-  once) are coalesced into a single upload.
-* **Session cleanup** — ``delete_session_files(token)`` is called when a
-  session expires so per-user files are removed from the dataset, keeping
-  the repo clean.
-* **Read-only parquet guard** — the large ``nse_processed_data.parquet``
-  is pulled on startup but never pushed at runtime (it is only updated via
-  ``push_to_hf.py``).
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import threading
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    pass
-
-log = logging.getLogger(__name__)
-
-# Files that are pulled on startup but NEVER pushed at runtime (too large /
-# managed by push_to_hf.py instead).
-_READ_ONLY_FILES = frozenset({
-    "nse_processed_data.parquet",
-    "nse_summary_data.parquet",
-})
-
-# Remote prefix inside the HF dataset repo for all processed files.
-_REMOTE_PREFIX = "processed"
-
-
-class HFDataSync:
-    """
-    Two-way sync between ``processed/`` and ``Sam-max1/nsedash_data``.
-
-    Parameters
-    ----------
-    data_dir:
-        Local path to ``<base_dir>/processed/``.
-    repo_id:
-        HuggingFace dataset repo ID (default: ``Sam-max1/nsedash_data``).
-    token:
-        HF API token.  If *None*, tries ``HF_TOKEN`` env var.
-    enabled:
-        Forced override — set to ``False`` to disable even in HF mode.
-    """
-
-    def __init__(
-        self,
-        data_dir: Path,
-        repo_id: str = "Sam-max1/nsedash_data",
-        token: str | None = None,
-        enabled: bool | None = None,
-    ) -> None:
-        self.data_dir = Path(data_dir)
-        self.repo_id = repo_id
-
-        # Resolve token
-        self._token = token or os.environ.get("HF_TOKEN") or os.environ.get("HF_PRIVATE_TOKEN")
-
-        # Enable only when running inside a HF Space or HF mode is set (unless forced)
-        if enabled is None:
-            self.enabled = "SPACE_ID" in os.environ or os.environ.get("HF_MODE") == "1"
-        else:
-            self.enabled = enabled
-
-        self._api = None          # lazy HfApi instance
-        self._lock = threading.Lock()
-
-        # Debounce: path → (Timer, repo_path)
-        self._pending: dict[str, threading.Timer] = {}
-        self._debounce_sec = 3.0
-
-        if self.enabled:
-            if not self._token:
-                log.warning(
-                    "[HFSync] HF_TOKEN not set — HF dataset sync is disabled."
-                )
-                self.enabled = False
-            else:
-                log.info(
-                    "[HFSync] Enabled — dataset: %s, data_dir: %s",
-                    self.repo_id, self.data_dir,
-                )
-        else:
-            log.info("[HFSync] Disabled (not in HF Space mode).")
-
-    # ------------------------------------------------------------------
-    # Internal helpers
-    # ------------------------------------------------------------------
-
-    def _get_api(self):
-        """Lazy-load HfApi (avoids import cost when sync is disabled)."""
-        if self._api is None:
-            from huggingface_hub import HfApi
-            self._api = HfApi(token=self._token)
-        return self._api
-
-    def _ensure_repo(self) -> None:
-        """Create the dataset repo if it does not yet exist."""
-        try:
-            api = self._get_api()
-            api.create_repo(
-                repo_id=self.repo_id,
-                repo_type="dataset",
-                private=False,
-                exist_ok=True,
-            )
-            log.info("[HFSync] Dataset repo '%s' is ready.", self.repo_id)
-        except Exception as exc:
-            log.error("[HFSync] Could not ensure dataset repo exists: %s", exc)
-
-    def _remote_path(self, local_path: Path) -> str:
-        """Compute the in-repo path for a local file under data_dir."""
-        try:
-            rel = local_path.relative_to(self.data_dir)
-        except ValueError:
-            rel = Path(local_path.name)
-        return f"{_REMOTE_PREFIX}/{rel.as_posix()}"
-
-    # ------------------------------------------------------------------
-    # Startup pull
-    # ------------------------------------------------------------------
-
-    def pull_on_startup(self) -> None:
-        """
-        Download all files from the HF dataset into ``processed/``.
-
-        Runs synchronously during startup.  Missing remote files are
-        silently skipped (first-run scenario).  The local parquet is
-        never overwritten if it is larger than the remote copy (safety
-        guard for the large data file).
-        """
-        if not self.enabled:
-            return
-
-        log.info("[HFSync] Starting startup pull from %s …", self.repo_id)
-        try:
-            self._ensure_repo()
-            api = self._get_api()
-            remote_files = list(api.list_repo_files(
-                repo_id=self.repo_id, repo_type="dataset"
-            ))
-        except Exception as exc:
-            log.error("[HFSync] Could not list remote files: %s", exc)
-            return
-
-        pulled, skipped, failed = 0, 0, 0
-        for remote_path in remote_files:
-            # Skip HF metadata files
-            if remote_path.startswith("."):
-                continue
-            if not remote_path.startswith(f"{_REMOTE_PREFIX}/"):
-                skipped += 1
-                continue
-
-            rel = remote_path[len(_REMOTE_PREFIX) + 1:]    # strip "processed/"
-            local_path = self.data_dir / rel
-            
-            # Ensure local file is removed before pulling to guarantee overwriting
-            if local_path.exists() and local_path.is_file():
-                try:
-                    local_path.unlink()
-                except Exception as e:
-                    log.warning("[HFSync] Could not remove local file %s before pull: %s", local_path, e)
-
-            local_path.parent.mkdir(parents=True, exist_ok=True)
-
-            try:
-                from huggingface_hub import hf_hub_download
-                tmp = hf_hub_download(
-                    repo_id=self.repo_id,
-                    repo_type="dataset",
-                    filename=remote_path,
-                    token=self._token,
-                    local_dir=str(self.data_dir.parent),  # downloads into parent/processed/rel
-                    local_dir_use_symlinks=False,
-                )
-                # hf_hub_download places file at local_dir/<remote_path>
-                # which resolves to data_dir.parent/processed/rel = data_dir/rel ✓
-                pulled += 1
-                log.info("[HFSync] Pulled: %s", rel)
-            except Exception as exc:
-                log.warning("[HFSync] Could not pull '%s': %s", remote_path, exc)
-                failed += 1
-
-        log.info(
-            "[HFSync] Startup pull complete — pulled=%d, skipped=%d, failed=%d",
-            pulled, skipped, failed,
-        )
-
-    # ------------------------------------------------------------------
-    # Async push (write path)
-    # ------------------------------------------------------------------
-
-    def push_file_async(self, local_path: Path | str,
-                        repo_path: str | None = None) -> None:
-        """
-        Schedule an upload of *local_path* to the HF dataset.
-
-        Calls within ``debounce_sec`` for the same path are coalesced —
-        only the last write is uploaded.  This is a fire-and-forget
-        background thread; the caller is never blocked.
-        """
-        if not self.enabled:
-            return
-
-        local_path = Path(local_path)
-        if not local_path.exists():
-            return
-
-        # Never push read-only / large files at runtime
-        if local_path.name in _READ_ONLY_FILES:
-            return
-
-        repo_path = repo_path or self._remote_path(local_path)
-        key = str(local_path)
-
-        with self._lock:
-            # Cancel any pending timer for this path
-            existing = self._pending.pop(key, None)
-            if existing:
-                existing.cancel()
-
-            def _upload():
-                with self._lock:
-                    self._pending.pop(key, None)
-                try:
-                    api = self._get_api()
-                    api.upload_file(
-                        path_or_fileobj=str(local_path),
-                        path_in_repo=repo_path,
-                        repo_id=self.repo_id,
-                        repo_type="dataset",
-                        commit_message=f"auto: update {local_path.name}",
-                    )
-                    log.info("[HFSync] Pushed: %s → %s", local_path.name, repo_path)
-                except Exception as exc:
-                    log.warning(
-                        "[HFSync] Push failed for '%s': %s", local_path.name, exc
-                    )
-
-            timer = threading.Timer(self._debounce_sec, _upload)
-            timer.daemon = True
-            timer.start()
-            self._pending[key] = timer
-
-    # ------------------------------------------------------------------
-    # Session cleanup (delete per-user files from HF dataset)
-    # ------------------------------------------------------------------
-
-    def delete_session_files(self, token: str) -> None:
-        """
-        Remove per-user files for *token* from the HF dataset.
-
-        Called when a session expires or the user disconnects, to keep
-        the dataset clean.  Runs in a background thread.
-        """
-        if not self.enabled or not token or token in ("admin", "default"):
-            return
-
-        def _delete():
-            try:
-                api = self._get_api()
-                remote_files = list(api.list_repo_files(
-                    repo_id=self.repo_id, repo_type="dataset"
-                ))
-                suffix = f"_{token}"
-                targets = [
-                    f for f in remote_files
-                    if token in f and not f.startswith(".")
-                ]
-                for path in targets:
-                    try:
-                        api.delete_file(
-                            path_in_repo=path,
-                            repo_id=self.repo_id,
-                            repo_type="dataset",
-                            commit_message=f"cleanup: session {token[:8]} expired",
-                        )
-                        log.info("[HFSync] Deleted session file: %s", path)
-                    except Exception as exc:
-                        log.warning(
-                            "[HFSync] Could not delete '%s': %s", path, exc
-                        )
-            except Exception as exc:
-                log.error(
-                    "[HFSync] Session cleanup failed for token %s: %s",
-                    token[:8], exc,
-                )
-
-        threading.Thread(target=_delete, daemon=True, name=f"hfsync-cleanup-{token[:8]}").start()
-
-
-# ---------------------------------------------------------------------------
-# Module-level no-op singleton (used when sync is disabled)
-# ---------------------------------------------------------------------------
-
-class _NoOpSync:
-    """Drop-in replacement when HF sync is disabled — all methods are no-ops."""
-    enabled = False
-
-    def pull_on_startup(self) -> None: pass
-    def push_file_async(self, *_a, **_kw) -> None: pass
-    def delete_session_files(self, *_a, **_kw) -> None: pass
-
-
-def make_sync(data_dir: Path, repo_id: str, token: str | None = None) -> HFDataSync | _NoOpSync:
-    """
-    Factory — returns a live ``HFDataSync`` when in HF mode, else a no-op.
-    """
-    is_hf = "SPACE_ID" in os.environ or os.environ.get("HF_MODE") == "1"
-    if not is_hf:
-        return _NoOpSync()
-    return HFDataSync(data_dir=data_dir, repo_id=repo_id, token=token)

nsedash/core/live_fetch.py

@@ -1,361 +0,0 @@
-"""
-nsedash.core.live_fetch
-=======================
-Real-time price/volume scraping from Google Finance and Yahoo Finance,
-with per-symbol source failover — functional parity with v4.5's
-``fetch_real_time_data_gf`` / ``fetch_real_time_data_yf`` /
-``fetch_real_time_data``.
-
-Audit fixes applied here:
-  C-1  Every HTTP call now has an explicit timeout (original had none: a
-       single hung socket froze the refresh cycle indefinitely).
-  P-1  Symbols are fetched concurrently via ThreadPoolExecutor instead of
-       strictly sequentially (original: ~2 blocking requests per symbol,
-       serially — 50 symbols ≈ 100 serial round trips).
-  P-2  A shared ``requests.Session`` reuses TCP/TLS connections.
-  A-1  ``fetch_real_time_volume`` returned ``([], [])`` on error but
-       ``(float, float)`` on success; the type anomaly is fixed —
-       always returns ``(float, float)``.
-  P-3  The Google path no longer performs a second full Yahoo page fetch
-       just for volume when the Yahoo page was already fetched during
-       failover; volume scraping is still done for Google-sourced quotes
-       (behaviour retained) but through the shared session.
-"""
-
-from __future__ import annotations
-
-import logging
-from concurrent.futures import ThreadPoolExecutor
-from datetime import datetime
-from typing import Callable, Iterable
-
-import json
-import requests
-from bs4 import BeautifulSoup
-
-from ..config import FETCH_MAX_WORKERS, HTTP_TIMEOUT_SEC, REQUEST_HEADERS
-
-PrevVolumeLookup = Callable[[str], float]
-
-
-import threading
-
-_global_fetch_lock = threading.Lock()
-
-
-class LiveFetcher:
-    """
-    Scrapes live quotes. ``prev_volume_lookup`` is injected by the data
-    layer so this module stays free of DataFrame coupling.
-    """
-
-    def __init__(self, prev_volume_lookup: PrevVolumeLookup | None = None,
-                 session: requests.Session | None = None):
-        self.prev_volume_lookup = prev_volume_lookup or (lambda _s: 0.0)
-        self.session = session or requests.Session()
-        self.session.headers.update(REQUEST_HEADERS)
-
-    # ------------------------------------------------------------------
-    # Volume (Yahoo)
-    # ------------------------------------------------------------------
-
-    def fetch_real_time_volume(self, symbol: str) -> tuple[float, float]:
-        """Return (current_volume, previous_volume) for a symbol."""
-        current_volume, previous_volume = 0.0, 0.0
-        yahoo_symbol = f'{symbol}.NS'
-        try:
-            response = self.session.get(
-                f'https://finance.yahoo.com/quote/{yahoo_symbol}',
-                timeout=HTTP_TIMEOUT_SEC,
-            )
-            response.raise_for_status()
-            soup = BeautifulSoup(response.text, 'html.parser')
-            best_res = None
-            for s in soup.find_all('script'):
-                if s.string and 'quoteResponse' in s.string:
-                    try:
-                        data = json.loads(s.string)
-                        body = json.loads(data['body'])
-                        for r_item in body['quoteResponse']['result']:
-                            if r_item.get('symbol') == yahoo_symbol:
-                                if best_res is None or (r_item.get('regularMarketDayHigh') is not None and best_res.get('regularMarketDayHigh') is None):
-                                    best_res = r_item
-                    except Exception:
-                        pass
-
-            if best_res:
-                vol_val = best_res.get('regularMarketVolume')
-                if isinstance(vol_val, dict):
-                    current_volume = float(vol_val.get('raw', 0.0))
-                elif vol_val is not None:
-                    current_volume = float(vol_val)
-            else:
-                # Fallback to the old HTML parser
-                vol_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketVolume'})
-                if vol_tag and vol_tag.text:
-                    try:
-                        current_volume = float(vol_tag.text.replace(',', ''))
-                    except ValueError:
-                        pass
-        except requests.exceptions.RequestException as req_err:
-            logging.error(f"HTTP Request error for {symbol} from Yahoo Finance: {req_err}")
-            return 0.0, 0.0
-        except Exception as e:
-            logging.error(f"Error fetching volume for {symbol} from Yahoo Finance: {e}")
-            return 0.0, 0.0
-        try:
-            previous_volume = float(self.prev_volume_lookup(symbol))
-        except Exception as e:
-            logging.error(f"Error fetching previous volume for {symbol} from historic data: {e}")
-        return current_volume, previous_volume
-
-    # ------------------------------------------------------------------
-    # Google Finance
-    # ------------------------------------------------------------------
-
-    def fetch_symbol_gf(self, symbol: str) -> dict | None:
-        """Scrape one symbol from Google Finance. None on failure."""
-        try:
-            response = self.session.get(
-                f'https://www.google.com/finance/quote/{symbol}:NSE',
-                headers={'User-Agent': 'python-requests/2.32.5'},
-                timeout=HTTP_TIMEOUT_SEC,
-            )
-            response.raise_for_status()
-            soup = BeautifulSoup(response.text, 'html.parser')
-
-            price_element = soup.find('div', {'class': 'YMlKec fxKbKc'})
-            current_price = (
-                float(price_element.text[1:].replace(",", ""))
-                if price_element else 0.0
-            )
-            if current_price <= 0.0:
-                raise ValueError(f"Invalid or zero price scraped for {symbol}")
-            previous_price_element = soup.find('div', {'class': 'P6K39c'})
-            previous_close_price = (
-                float(previous_price_element.text[1:].replace(",", ""))
-                if previous_price_element else 0.0
-            )
-            all_p6k39c = soup.find_all('div', {'class': 'P6K39c'})
-            day_high = day_low = 0.0
-            if len(all_p6k39c) > 1:
-                range_text = all_p6k39c[1].text.replace('₹', '').replace(',', '')
-                if ' - ' in range_text:
-                    low_str, high_str = range_text.split(' - ')
-                    day_low, day_high = float(low_str), float(high_str)
-            if previous_close_price == 0.0:
-                previous_close_price = current_price
-            current_volume, previous_volume = self.fetch_real_time_volume(symbol)
-            return {
-                "SYMBOL": symbol,
-                "CLOSE_PRICE": current_price,
-                "PREV_CL_PR": previous_close_price,
-                "DAY_HIGH": day_high,
-                "DAY_LOW": day_low,
-                "CURR_VOL": current_volume,
-                "PREV_VOL": previous_volume,
-            }
-        except requests.exceptions.RequestException as req_err:
-            logging.error(f"HTTP Request error for {symbol} from Google Finance: {req_err}")
-        except AttributeError:
-            logging.error(
-                f"Parsing error for {symbol} from Google Finance. "
-                "HTML structure might have changed."
-            )
-        except ValueError:
-            logging.error(
-                f"Data conversion error for {symbol} from Google Finance. "
-                "Could not convert price to float."
-            )
-        except Exception as e:
-            logging.error(f"Error fetching data for {symbol} from Google Finance: {e}")
-        return None
-
-    # ------------------------------------------------------------------
-    # Yahoo Finance
-    # ------------------------------------------------------------------
-
-    def fetch_symbol_yf(self, symbol: str) -> dict | None:
-        """Scrape one symbol from Yahoo Finance. None on failure."""
-        yahoo_symbol = f'{symbol}.NS'
-        try:
-            response = self.session.get(
-                f'https://finance.yahoo.com/quote/{yahoo_symbol}',
-                timeout=HTTP_TIMEOUT_SEC,
-            )
-            response.raise_for_status()
-            soup = BeautifulSoup(response.text, 'html.parser')
-
-            best_res = None
-            for s in soup.find_all('script'):
-                if s.string and 'quoteResponse' in s.string:
-                    try:
-                        data = json.loads(s.string)
-                        body = json.loads(data['body'])
-                        for r_item in body['quoteResponse']['result']:
-                            if r_item.get('symbol') == yahoo_symbol:
-                                if best_res is None or (r_item.get('regularMarketDayHigh') is not None and best_res.get('regularMarketDayHigh') is None):
-                                    best_res = r_item
-                    except Exception:
-                        pass
-
-            if not best_res:
-                # Fallback to the old HTML tag parser (used by tests and old site structure)
-                try:
-                    price_tag = soup.find(attrs={"data-testid": "qsp-price"})
-                    if not price_tag:
-                        price_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketPrice'})
-                    
-                    if not price_tag:
-                        raise ValueError(f"No Svelte Sourced Quote data or HTML tags found for {symbol}")
-                    
-                    current_price = float(price_tag.text.replace(',', ''))
-                    
-                    prev_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketPreviousClose'})
-                    previous_close_price = float(prev_tag.text.replace(',', '')) if prev_tag else current_price
-                    
-                    day_high, day_low = 0.0, 0.0
-                    range_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketDayRange'})
-                    if range_tag and range_tag.text:
-                        parts = [p.strip() for p in range_tag.text.split('-')]
-                        if len(parts) == 2:
-                            day_low = float(parts[0].replace(',', ''))
-                            day_high = float(parts[1].replace(',', ''))
-                    
-                    vol_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketVolume'})
-                    current_volume = float(vol_tag.text.replace(',', '')) if vol_tag else 0.0
-                except Exception as fallback_err:
-                    raise ValueError(f"No Svelte Sourced Quote data found for {symbol} (HTML fallback failed: {fallback_err})")
-            else:
-                def get_val(field):
-                    val = best_res.get(field)
-                    if isinstance(val, dict):
-                        return float(val.get('raw', 0.0))
-                    return float(val) if val is not None else 0.0
-
-                current_price = get_val('regularMarketPrice')
-                if current_price <= 0.0:
-                    raise ValueError(f"Invalid or zero price scraped for {symbol}")
-                previous_close_price = get_val('regularMarketPreviousClose')
-                day_high = get_val('regularMarketDayHigh')
-                day_low = get_val('regularMarketDayLow')
-                current_volume = get_val('regularMarketVolume')
-            previous_volume = 0.0
-            try:
-                previous_volume = float(self.prev_volume_lookup(symbol))
-            except Exception as e:
-                logging.error(f"Error fetching previous volume for {symbol}: {e}")
-            if previous_close_price == 0.0:
-                previous_close_price = current_price
-            return {
-                "SYMBOL": symbol,
-                "CLOSE_PRICE": current_price,
-                "PREV_CL_PR": previous_close_price,
-                "DAY_HIGH": day_high,
-                "DAY_LOW": day_low,
-                "CURR_VOL": current_volume,
-                "PREV_VOL": previous_volume,
-            }
-        except requests.exceptions.RequestException as req_err:
-            logging.error(f"HTTP Request error for {symbol} from Yahoo Finance: {req_err}")
-        except Exception as e:
-            logging.error(f"Error fetching data for {symbol} from Yahoo Finance: {e}")
-        return None
-
-    # Batch wrappers preserved for API parity with v4.5 ------------------
-
-    def fetch_real_time_data_gf(self, symbols: Iterable[str]) -> list[dict]:
-        return [d for d in (self.fetch_symbol_gf(s) for s in symbols) if d]
-
-    def fetch_real_time_data_yf(self, symbols: Iterable[str]) -> list[dict]:
-        return [d for d in (self.fetch_symbol_yf(s) for s in symbols) if d]
-
-    # ------------------------------------------------------------------
-    # Failover orchestration
-    # ------------------------------------------------------------------
-
-    def _fetch_one_with_failover(self, symbol: str,
-                                 preferred_source: str) -> tuple[dict | None, str]:
-        """Fetch one symbol trying preferred source first, then the other."""
-        order = (
-            [("Google", self.fetch_symbol_gf), ("Yahoo", self.fetch_symbol_yf)]
-            if preferred_source == "Google"
-            else [("Yahoo", self.fetch_symbol_yf), ("Google", self.fetch_symbol_gf)]
-        )
-        primary_name = order[0][0]
-        for i, (name, fn) in enumerate(order):
-            data = fn(symbol)
-            if data:
-                return data, name
-            if i == 0:
-                logging.warning(
-                    f"{primary_name} Finance failed for {symbol}. "
-                    f"Falling back to {order[1][0]}."
-                )
-        logging.error(f"Both sources failed for {symbol}.")
-        return None, "None"
-
-    def fetch_real_time_data(self, symbols: list[str],
-                             preferred_source: str = "Google"
-                             ) -> tuple[list[dict], str]:
-        """
-        Fetch real-time data for all symbols with per-symbol failover.
-        Returns (list of quote dicts, source_name). Source name reflects
-        the last successful source, matching v4.5 semantics.
-        """
-        preferred_source = preferred_source or "Google"
-        data_source = "None"
-        real_time_data: list[dict] = []
-        failed_symbols: list[str] = []
-        symbols = list(symbols)
-        if not symbols:
-            return [], data_source
-
-        with _global_fetch_lock:
-            with ThreadPoolExecutor(max_workers=min(FETCH_MAX_WORKERS, len(symbols))) as pool:
-                results = list(pool.map(
-                    lambda s: (s, self._fetch_one_with_failover(s, preferred_source)),
-                    symbols,
-                ))
-            for symbol, (data, source) in results:
-                if data:
-                    real_time_data.append(data)
-                    data_source = source
-                else:
-                    failed_symbols.append(symbol)
-            if failed_symbols:
-                logging.error(f"Failed symbols: {', '.join(failed_symbols)}")
-            return real_time_data, data_source
-
-    def fetch_index_quotes(self) -> dict[str, dict]:
-        """
-        Scrape live market index quotes for NIFTY 50 (^NSEI) and SENSEX (^BSESN).
-        Returns dict keyed by index name ('NIFTY', 'SENSEX').
-        """
-        indices = {
-            'NIFTY': '^NSEI',
-            'SENSEX': '^BSESN',
-        }
-        res = {}
-        for name, sym in indices.items():
-            try:
-                url = f"https://query1.finance.yahoo.com/v8/finance/chart/{sym}?interval=1m"
-                r = self.session.get(url, timeout=HTTP_TIMEOUT_SEC)
-                r.raise_for_status()
-                data = r.json()
-                meta = data['chart']['result'][0]['meta']
-                price = float(meta.get('regularMarketPrice', 0.0))
-                prev = float(meta.get('chartPreviousClose') or meta.get('previousClose') or price)
-                chg = price - prev
-                pct = (chg / prev * 100.0) if prev > 0 else 0.0
-                res[name] = {
-                    "name": name,
-                    "price": round(price, 2),
-                    "change": round(chg, 2),
-                    "pct_change": round(pct, 2),
-                    "formatted_price": f"{price:,.2f}",
-                    "formatted_change": f"{chg:+,.2f} ({pct:+.2f}%)"
-                }
-            except Exception as e:
-                logging.warning(f"Could not fetch live index for {name} ({sym}): {e}")
-        return res

nsedash/core/nvidia_llm.py

@@ -1,933 +0,0 @@
-"""
-NVIDIA NIM LLM Wrapper with Intelligent Routing
-================================================
-Maintains a curated registry of all 121 NVIDIA NIM free-tier models,
-categorized and benchmarked by purpose. Provides intelligent routing
-based on the request type (coding, reasoning, chat, memory, judge,
-finance, vision, safety, translation, embedding).
-
-Model Registry is stored in: utils/config/nvidia_model_registry.json
-"""
-
-from typing import Any, Optional
-import requests
-import os
-import json
-import threading
-import datetime
-import time
-
-try:
-    from dotenv import load_dotenv, find_dotenv
-    env_file = find_dotenv(usecwd=True)
-    if env_file:
-        load_dotenv(env_file)
-    else:
-        load_dotenv()
-except ImportError:
-    pass
-
-try:
-    import keyring
-except ImportError:
-    keyring = None
-
-try:
-    from crewai.llms.base_llm import BaseLLM
-except ImportError:
-    class BaseLLM:
-        def __init__(self, **kwargs):
-            self.model = kwargs.get("model")
-            self.api_key = kwargs.get("api_key")
-
-# ---------------------------------------------------------------------------
-# Telemetry Token Tracking State
-# ---------------------------------------------------------------------------
-TOKEN_STATE_FILE = os.path.join(os.path.dirname(__file__), "token_state.json")
-MODEL_REGISTRY_FILE = os.path.join(os.path.dirname(__file__), "nvidia_model_registry.json")
-token_lock = threading.Lock()
-_WARNED_MISSING_KEY = False
-
-def load_token_usage() -> dict:
-    with token_lock:
-        if os.path.exists(TOKEN_STATE_FILE):
-            try:
-                with open(TOKEN_STATE_FILE, "r") as f:
-                    return json.load(f)
-            except Exception:
-                pass
-    return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
-
-def save_token_usage(usage: dict):
-    with token_lock:
-        try:
-            with open(TOKEN_STATE_FILE, "w") as f:
-                json.dump(usage, f, indent=4)
-        except Exception:
-            pass
-
-def add_tokens(prompt: int, completion: int, model_id: str = ""):
-    usage = load_token_usage()
-    usage["prompt_tokens"] += prompt
-    usage["completion_tokens"] += completion
-    usage["total_tokens"] += (prompt + completion)
-    # Per-model telemetry
-    per_model = usage.setdefault("per_model", {})
-    entry = per_model.setdefault(model_id, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0})
-    entry["prompt_tokens"] += prompt
-    entry["completion_tokens"] += completion
-    entry["total_tokens"] += (prompt + completion)
-    save_token_usage(usage)
-
-def get_token_usage() -> dict:
-    return load_token_usage()
-
-def reset_token_usage():
-    save_token_usage({"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0})
-
-
-# ---------------------------------------------------------------------------
-# Curated Model Registry — Built from deep research on 121 NVIDIA NIM models
-# ---------------------------------------------------------------------------
-# Categories: coding, reasoning, chat, memory, judge, finance, vision,
-#             safety, translation, embedding, creative, fast
-#
-# Ratings: 5=exceptional, 4=excellent, 3=good, 2=decent, 1=basic
-# Source: NVIDIA NIM catalog + community benchmarks (HumanEval, MMLU, etc.)
-# ---------------------------------------------------------------------------
-
-NVIDIA_MODEL_REGISTRY = {
-    # === CODING ===
-    # Best for: code generation, debugging, refactoring, code review, self-improvement agents
-    "coding": {
-        "primary": "deepseek-ai/deepseek-v4-pro",
-        "fallback": ["deepseek-ai/deepseek-v4-flash", "poolside/laguna-xs-2.1", "abacusai/dracarys-llama-3.1-70b-instruct", "nvidia/nv-embedcode-7b-v1"],
-        "models": {
-            "deepseek-ai/deepseek-v4-pro": {
-                "rating": 5, "context_window": 163840, "description": "State-of-the-art coder. Outstanding performance on code generation and multi-step agent debugging.", "strengths": ["code_gen", "agentic", "debugging"]
-            },
-            "deepseek-ai/deepseek-v4-flash": {
-                "rating": 4, "context_window": 163840, "description": "High-throughput coding model. Ideal for low-latency coding loops.", "strengths": ["code_gen", "fast", "agentic"]
-            },
-            "poolside/laguna-xs-2.1": {
-                "rating": 5, "context_window": 32768, "description": "Poolside Laguna XS. Specially trained for agentic coding and long-horizon developer workflows.", "strengths": ["code_gen", "agentic", "planning"]
-            },
-            "abacusai/dracarys-llama-3.1-70b-instruct": {
-                "rating": 4, "context_window": 131072, "description": "Abacus.AI fine-tune of Llama 3.1 70B, optimized for complex code instruction following.", "strengths": ["code_gen", "instruction"]
-            },
-            "nvidia/nv-embedcode-7b-v1": {
-                "rating": 3, "context_window": 8192, "description": "NVIDIA specialized coding embedding model for semantic code search.", "strengths": ["embedding", "code_search"]
-            }
-        }
-    },
-
-    # === REASONING / COMPLEX TASKS ===
-    # Best for: multi-step reasoning, mathematical analysis, strategy planning, agentic workflows
-    "reasoning": {
-        "primary": "google/diffusiongemma-26b-a4b-it",
-        "fallback": ["meta/llama-3.1-70b-instruct", "minimaxai/minimax-m3", "z-ai/glm-5.2", "nvidia/nemotron-3-ultra-550b-a55b", "thinkingmachines/inkling", "meta/llama-3.3-70b-instruct", "nvidia/llama-3.3-nemotron-super-49b-v1.5", "meta/llama-3.1-8b-instruct"],
-        "models": {
-            "minimaxai/minimax-m3": {
-                "rating": 5, "context_window": 128000, "description": "MiniMax-M3 SOTA reasoning model. Exceptional financial analysis and long context planning.", "strengths": ["reasoning", "finance", "instruction"]
-            },
-            "google/diffusiongemma-26b-a4b-it": {
-                "rating": 5, "context_window": 128000, "description": "Google DiffusionGemma 26B A4B IT. Next-gen reasoning and fast inference.", "strengths": ["reasoning", "finance", "agentic"]
-            },
-            "z-ai/glm-5.2": {
-                "rating": 5, "context_window": 128000, "description": "GLM-5.2 SOTA reasoning model. Outstanding multi-step planning and financial instruction following.", "strengths": ["reasoning", "finance", "instruction"]
-            },
-            "nvidia/nemotron-3-ultra-550b-a55b": {
-                "rating": 5, "context_window": 128000, "description": "NVIDIA 550B reasoning model. Exceptional logical analysis.", "strengths": ["reasoning", "agentic", "math"]
-            },
-            "thinkingmachines/inkling": {
-                "rating": 4, "context_window": 32768, "description": "Inkling 975B MoE reasoning model. Excels at complex analysis.", "strengths": ["reasoning", "moe"]
-            },
-            "meta/llama-3.1-70b-instruct": {
-                "rating": 5, "context_window": 128000, "description": "Meta Llama 3.1 70B Instruct. High throughput, low latency.", "strengths": ["reasoning", "finance", "fast"]
-            },
-            "meta/llama-3.3-70b-instruct": {
-                "rating": 5, "context_window": 128000, "description": "Meta Llama 3.3 70B Instruct.", "strengths": ["reasoning", "finance"]
-            },
-            "nvidia/llama-3.3-nemotron-super-49b-v1.5": {
-                "rating": 4, "context_window": 128000, "description": "NVIDIA Nemotron Super 49B v1.5.", "strengths": ["reasoning", "agentic"]
-            },
-            "meta/llama-3.1-8b-instruct": {
-                "rating": 4, "context_window": 128000, "description": "Llama 3.1 8B Instruct. Fast fallback.", "strengths": ["fast", "general"]
-            }
-        }
-    },
-
-    # === CHAT / GENERAL PURPOSE ===
-    # Best for: conversational Q&A, summarization, instruction-following
-    "chat": {
-        "primary": "meta/llama-3.3-70b-instruct",
-        "fallback": ["meta/llama-3.1-70b-instruct", "meta/llama-3.1-8b-instruct", "minimaxai/minimax-m3", "mistralai/mistral-medium-3.5-128b", "google/gemma-4-31b-it", "mistralai/mistral-small-4-119b-2603", "qwen/qwen3.5-122b-a10b", "nvidia/nemotron-3-nano-30b-a3b", "mistralai/ministral-14b-instruct-2512", "bytedance/seed-oss-36b-instruct", "nvidia/nvidia-nemotron-nano-9b-v2", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "sarvamai/sarvam-m", "google/gemma-3n-e4b-it", "google/gemma-3n-e2b-it", "mistralai/mistral-nemotron", "nvidia/llama-3.1-nemotron-nano-8b-v1", "upstage/solar-10.7b-instruct", "mistralai/mixtral-8x7b-instruct-v0.1", "minimaxai/minimax-m2.7", "stepfun-ai/step-3.7-flash", "stepfun-ai/step-3.5-flash"],
-        "models": {
-            "meta/llama-3.3-70b-instruct": {
-                "rating": 5, "context_window": 128000, "description": "Meta Llama 3.3 70B. High-quality instruction following, default conversational model.", "strengths": ["chat", "general", "instruction"]
-            },
-            "meta/llama-3.1-70b-instruct": {
-                "rating": 4, "context_window": 128000, "description": "Meta Llama 3.1 70B. Excellent instruction execution and formatting.", "strengths": ["chat", "general"]
-            },
-            "meta/llama-3.1-8b-instruct": {
-                "rating": 4, "context_window": 128000, "description": "Llama 3.1 8B. Fast, reliable conversational chat model.", "strengths": ["chat", "fast", "general"]
-            },
-            "minimaxai/minimax-m3": {
-                "rating": 4, "context_window": 40960, "description": "MiniMax M3. Strong general assistant, optimized for long-form dialogue.", "strengths": ["chat", "dialogue"]
-            },
-            "minimaxai/minimax-m2.7": {
-                "rating": 4, "context_window": 40960, "description": "MiniMax M2.7. Fast conversational LLM.", "strengths": ["chat", "fast"]
-            },
-            "stepfun-ai/step-3.7-flash": {
-                "rating": 4, "context_window": 128000, "description": "StepFun 3.7 Flash. Ultra-fast high-concurrency conversational assistant.", "strengths": ["chat", "fast"]
-            },
-            "stepfun-ai/step-3.5-flash": {
-                "rating": 3, "context_window": 128000, "description": "StepFun 3.5 Flash model.", "strengths": ["chat", "fast"]
-            },
-            "mistralai/mistral-medium-3.5-128b": {
-                "rating": 4, "context_window": 128000, "description": "Mistral Medium 3.5. Balanced instruct model with great reasoning capacity.", "strengths": ["chat", "instruction"]
-            },
-            "google/gemma-4-31b-it": {
-                "rating": 4, "context_window": 128000, "description": "Google Gemma 4 31B instruction-tuned model. Top scores in general chat.", "strengths": ["chat", "instruction"]
-            },
-            "mistralai/mistral-small-4-119b-2603": {
-                "rating": 4, "context_window": 32768, "description": "Mistral Small 4 119B. High quality intermediate model for general instruct tasks.", "strengths": ["chat", "instruction"]
-            },
-            "qwen/qwen3.5-122b-a10b": {
-                "rating": 4, "context_window": 32768, "description": "Qwen3.5 122B MoE model, highly conversational.", "strengths": ["chat", "moe"]
-            },
-            "nvidia/nemotron-3-nano-30b-a3b": {
-                "rating": 3, "context_window": 32768, "description": "NVIDIA Nemotron 3 Nano.", "strengths": ["chat", "fast"]
-            },
-            "mistralai/ministral-14b-instruct-2512": {
-                "rating": 3, "context_window": 32768, "description": "Ministral 14B instruction model.", "strengths": ["chat", "fast"]
-            },
-            "bytedance/seed-oss-36b-instruct": {
-                "rating": 4, "context_window": 32768, "description": "ByteDance Seed OSS 36B.", "strengths": ["chat", "general"]
-            },
-            "nvidia/nvidia-nemotron-nano-9b-v2": {
-                "rating": 3, "context_window": 32768, "description": "NVIDIA Nemotron Nano 9B v2.", "strengths": ["chat", "fast"]
-            },
-            "openai/gpt-oss-20b": {
-                "rating": 4, "context_window": 32768, "description": "OpenAI GPT-OSS 20B model.", "strengths": ["chat", "general"]
-            },
-            "openai/gpt-oss-120b": {
-                "rating": 4, "context_window": 32768, "description": "OpenAI GPT-OSS 120B model.", "strengths": ["chat", "general"]
-            },
-            "sarvamai/sarvam-m": {
-                "rating": 3, "context_window": 8192, "description": "Sarvam-M multilingual chat model.", "strengths": ["chat", "multilingual"]
-            },
-            "google/gemma-3n-e4b-it": {
-                "rating": 4, "context_window": 128000, "description": "Google Gemma 3N E4B model.", "strengths": ["chat", "general"]
-            },
-            "google/gemma-3n-e2b-it": {
-                "rating": 4, "context_window": 128000, "description": "Google Gemma 3N E2B model.", "strengths": ["chat", "general"]
-            },
-            "mistralai/mistral-nemotron": {
-                "rating": 4, "context_window": 128000, "description": "Mistral-Nemotron collaboration model.", "strengths": ["chat", "general"]
-            },
-            "nvidia/llama-3.1-nemotron-nano-8b-v1": {
-                "rating": 3, "context_window": 128000, "description": "NVIDIA tuned Llama 3.1 Nano 8B.", "strengths": ["chat", "fast"]
-            },
-            "upstage/solar-10.7b-instruct": {
-                "rating": 3, "context_window": 4096, "description": "Upstage Solar 10.7B instruction-tuned model.", "strengths": ["chat", "instruction"]
-            },
-            "mistralai/mixtral-8x7b-instruct-v0.1": {
-                "rating": 3, "context_window": 32768, "description": "Mixtral 8x7B MoE model.", "strengths": ["chat", "moe"]
-            }
-        }
-    },
-
-    # === MEMORY / SUMMARIZATION ===
-    # Best for: MEMORY.md updates, long-context summarization, KB compression
-    "memory": {
-        "primary": "meta/llama-4-maverick-17b-128e-instruct",
-        "fallback": ["moonshotai/kimi-k2.6"],
-        "models": {
-            "meta/llama-4-maverick-17b-128e-instruct": {
-                "rating": 5, "context_window": 1000000, "description": "Llama 4 Maverick. 1M token context, perfect for large codebase digests.", "strengths": ["summarization", "long_context", "memory"]
-            },
-            "moonshotai/kimi-k2.6": {
-                "rating": 4, "context_window": 131072, "description": "Kimi K2.6. 128K context window, highly accurate long-document analysis.", "strengths": ["summarization", "long_context"]
-            }
-        }
-    },
-
-    # === SAFETY / CONTENT MODERATION ===
-    # Best for: guardrails, PII detection, content safety checks, topic control
-    "safety": {
-        "primary": "nvidia/nemotron-3.5-content-safety",
-        "fallback": ["meta/llama-guard-4-12b", "nvidia/gliner-pii", "nvidia/llama-3.1-nemotron-safety-guard-8b-v3"],
-        "models": {
-            "nvidia/nemotron-3.5-content-safety": {
-                "rating": 5, "context_window": 4096, "description": "NVIDIA specialized content safety model.", "strengths": ["safety", "moderation"]
-            },
-            "meta/llama-guard-4-12b": {
-                "rating": 4, "context_window": 128000, "description": "Llama Guard 4 12B safety classifier.", "strengths": ["safety", "moderation"]
-            },
-            "nvidia/gliner-pii": {
-                "rating": 4, "context_window": 8192, "description": "GLiNER PII recognition model for anonymization.", "strengths": ["safety", "pii_redaction"]
-            },
-            "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": {
-                "rating": 4, "context_window": 8192, "description": "NVIDIA safety guard 8B v3.", "strengths": ["safety", "moderation"]
-            }
-        }
-    },
-
-    # === TRANSLATION ===
-    "translation": {
-        "primary": "nvidia/riva-translate-4b-instruct-v1.1",
-        "models": {
-            "nvidia/riva-translate-4b-instruct-v1.1": {
-                "rating": 4, "context_window": 4096, "description": "Riva machine translation model.", "strengths": ["translation"]
-            }
-        }
-    },
-
-    # === EMBEDDING ===
-    # Best for: vector embeddings for semantic search, RAG, document retrieval
-    "embedding": {
-        "primary": "nvidia/nv-embedcode-7b-v1",
-        "fallback": ["nvidia/nemotron-3-embed-1b", "nvidia/rerank-qa-mistral-4b"],
-        "models": {
-            "nvidia/nv-embedcode-7b-v1": {
-                "rating": 4, "context_window": 8192, "description": "NVIDIA specialized code embedding model.", "strengths": ["embedding", "code_search"]
-            },
-            "nvidia/nemotron-3-embed-1b": {
-                "rating": 3, "context_window": 512, "description": "Nemotron-3 1B embedding model.", "strengths": ["embedding"]
-            },
-            "nvidia/rerank-qa-mistral-4b": {
-                "rating": 4, "context_window": 4096, "description": "Mistral-based QA reranker model.", "strengths": ["rerank", "rag"]
-            }
-        }
-    },
-
-    # === VISION / MULTIMODAL ===
-    # Best for: image understanding, chart analysis, VLM-based tasks
-    "vision": {
-        "primary": "meta/llama-3.2-90b-vision-instruct",
-        "fallback": ["meta/llama-3.2-11b-vision-instruct", "google/diffusiongemma-26b-a4b-it", "google/paligemma", "nvidia/nemotron-nano-12b-v2-vl", "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"],
-        "models": {
-            "meta/llama-3.2-90b-vision-instruct": {
-                "rating": 5, "context_window": 128000, "description": "Meta Llama 3.2 90B Vision. High-end image and chart understanding.", "strengths": ["vision", "chart_analysis"]
-            },
-            "meta/llama-3.2-11b-vision-instruct": {
-                "rating": 4, "context_window": 128000, "description": "Meta Llama 3.2 11B Vision. Efficient image captioning and Q&A.", "strengths": ["vision", "fast"]
-            },
-            "google/diffusiongemma-26b-a4b-it": {
-                "rating": 4, "context_window": 8192, "description": "DiffusionGemma 26B multimodal visual instruction model.", "strengths": ["vision", "diffusion"]
-            },
-            "google/paligemma": {
-                "rating": 3, "context_window": 4096, "description": "Google PaliGemma multimodal model.", "strengths": ["vision", "vqa"]
-            },
-            "nvidia/nemotron-nano-12b-v2-vl": {
-                "rating": 4, "context_window": 32768, "description": "NVIDIA Nemotron Nano 12B VL model.", "strengths": ["vision", "vl"]
-            },
-            "nvidia/llama-3.1-nemotron-nano-vl-8b-v1": {
-                "rating": 3, "context_window": 128000, "description": "NVIDIA tuned Llama 3.1 Nano VL 8B.", "strengths": ["vision", "fast"]
-            }
-        }
-    },
-
-    # === FAST / LIGHTWEIGHT ===
-    # Best for: quick classification, token-budget-conscious tasks, high-throughput pipelines
-    "fast": {
-        "primary": "meta/llama-3.2-3b-instruct",
-        "fallback": ["meta/llama-3.2-1b-instruct", "nvidia/nemotron-mini-4b-instruct", "google/gemma-2-2b-it"],
-        "models": {
-            "meta/llama-3.2-3b-instruct": {
-                "rating": 4, "context_window": 128000, "description": "Llama 3.2 3B. Fast, lightweight edge-tier chatbot.", "strengths": ["fast", "lightweight"]
-            },
-            "meta/llama-3.2-1b-instruct": {
-                "rating": 3, "context_window": 128000, "description": "Llama 3.2 1B. Compact model for low token footprint.", "strengths": ["fast", "lightweight"]
-            },
-            "nvidia/nemotron-mini-4b-instruct": {
-                "rating": 3, "context_window": 4096, "description": "Nemotron Mini 4B instruction model.", "strengths": ["fast", "compact"]
-            },
-            "google/gemma-2-2b-it": {
-                "rating": 3, "context_window": 8192, "description": "Google Gemma-2 2B instruct.", "strengths": ["fast", "compact"]
-            }
-        }
-    },
-
-    # === SCIENCE & SPECIALIZED ===
-    "science": {
-        "primary": "nvidia/esmfold",
-        "fallback": ["nvidia/esm2-650m"],
-        "models": {
-            "nvidia/esmfold": {
-                "rating": 5, "context_window": 1024, "description": "ESMFold model for direct protein structure prediction from sequence.", "strengths": ["science", "protein_folding"]
-            },
-            "nvidia/esm2-650m": {
-                "rating": 4, "context_window": 1024, "description": "ESM-2 650M parameter protein language model.", "strengths": ["science", "protein_language"]
-            }
-        }
-    },
-
-    # === MULTIMODAL VISION / VEHICLES ===
-    "multimodal_vision": {
-        "primary": "nvidia/cosmos3-nano",
-        "fallback": ["nvidia/cosmos3-nano-reasoner", "nvidia/ai-synthetic-video-detector", "nvidia/active-speaker-detection", "nvidia/ising-calibration-1-35b-a3b", "nvidia/cosmos-transfer2.5-2b", "nvidia/streampetr", "nvidia/cosmos-transfer1-7b", "nvidia/sparsedrive", "nvidia/bevformer"],
-        "models": {
-            "nvidia/cosmos3-nano": {
-                "rating": 5, "context_window": 8192, "description": "Cosmos-3 Nano spatial video model.", "strengths": ["video", "spatial"]
-            },
-            "nvidia/cosmos3-nano-reasoner": {
-                "rating": 5, "context_window": 8192, "description": "Cosmos-3 Nano reasoning visual agent.", "strengths": ["video", "reasoning"]
-            },
-            "nvidia/ai-synthetic-video-detector": {
-                "rating": 4, "context_window": 4096, "description": "Synthetic video generation detector.", "strengths": ["video", "detection"]
-            },
-            "nvidia/active-speaker-detection": {
-                "rating": 4, "context_window": 4096, "description": "Maxine active speaker detection.", "strengths": ["video", "maxine"]
-            },
-            "nvidia/ising-calibration-1-35b-a3b": {
-                "rating": 3, "context_window": 4096, "description": "Ising calibration vision model.", "strengths": ["video"]
-            },
-            "nvidia/cosmos-transfer2.5-2b": {
-                "rating": 4, "context_window": 4096, "description": "Cosmos Transfer 2.5 2B video diffusion helper.", "strengths": ["video"]
-            },
-            "nvidia/streampetr": {
-                "rating": 4, "context_window": 4096, "description": "StreamPETR 3D object detection for autonomous driving.", "strengths": ["driving", "3d_detection"]
-            },
-            "nvidia/cosmos-transfer1-7b": {
-                "rating": 4, "context_window": 4096, "description": "Cosmos Transfer 1 7B video model.", "strengths": ["video"]
-            },
-            "nvidia/sparsedrive": {
-                "rating": 4, "context_window": 4096, "description": "SparseDrive planning for autonomous vehicles.", "strengths": ["driving"]
-            },
-            "nvidia/bevformer": {
-                "rating": 4, "context_window": 4096, "description": "BEVFormer bird's-eye-view camera perception.", "strengths": ["driving", "perception"]
-            }
-        }
-    },
-
-    # === AUDIO / SPEECH ===
-    "audio": {
-        "primary": "nvidia/nemotron-voicechat",
-        "fallback": ["nvidia/magpie-tts-zeroshot", "nvidia/background-noise-removal", "nvidia/studio-voice-enhance"],
-        "models": {
-            "nvidia/nemotron-voicechat": {
-                "rating": 5, "context_window": 4096, "description": "Nemotron model optimized for voice-based conversational turn-taking.", "strengths": ["voice", "chat"]
-            },
-            "nvidia/magpie-tts-zeroshot": {
-                "rating": 4, "context_window": 4096, "description": "Magpie zero-shot Text-to-Speech synthesis.", "strengths": ["voice", "tts"]
-            },
-            "nvidia/background-noise-removal": {
-                "rating": 4, "context_window": 4096, "description": "Maxine background noise removal audio NIM.", "strengths": ["audio", "maxine"]
-            },
-            "nvidia/studio-voice-enhance": {
-                "rating": 4, "context_window": 4096, "description": "Maxine Studio voice enhance audio NIM.", "strengths": ["audio", "maxine"]
-            }
-        }
-    }
-}
-
-# Keyword → category routing table for automatic dispatch
-ROUTING_KEYWORDS = {
-    "coding":            ["code", "python", "function", "class", "debug", "refactor", "implement", "syntax", "algorithm", "programming", "script", "module", "bug", "fix", "rewrite", "agent_workflow", "agent"],
-    "reasoning":         ["reason", "analyze", "plan", "strategy", "optimize", "complex", "math", "calculate", "evaluate", "multi-step", "explain", "compare", "decide", "judge", "score", "audit", "rate", "hallucin", "assess", "review trajectory", "llm-as-a-judge", "finance", "stock", "trading", "market", "portfolio", "financial", "investment", "equity", "backtest", "capital"],
-    "memory":            ["summarize", "memory", "kb", "knowledge", "log", "history", "update memory", "compress", "recall", "persist"],
-    "chat":              ["chat", "write", "creative", "document", "report", "essay", "story", "generate text"],
-    "safety":            ["safety", "harmful", "pii", "content moderation", "guardrail", "toxic", "filter", "censor"],
-    "fast":              ["quick", "fast", "brief", "short", "classify", "yes/no", "simple", "ping"],
-    "science":           ["science", "protein", "esmfold", "esm2", "biology", "molecule", "structure", "medical", "health", "clinical", "patient", "diagnosis", "symptom", "drug"],
-    "vision":            ["image", "chart", "graph", "visual", "picture", "plot", "screenshot", "vision"],
-    "multimodal_vision": ["cosmos", "video", "driving", "bevformer", "sparsedrive", "calibration"],
-    "audio":             ["voice", "tts", "speech", "noise", "audio", "sound", "enhance"],
-    "translation":       ["translate", "riva", "language", "bilingual", "multilingual"],
-}
-
-
-def route_request(prompt: str, hint: Optional[str] = None) -> tuple[str, str]:
-    """
-    Determine the best category and model for a given prompt.
-    Returns (category, model_id).
-    - hint: explicit category override (e.g. 'coding', 'judge', 'memory')
-    """
-    if hint and hint in NVIDIA_MODEL_REGISTRY:
-        cat = hint
-        return cat, NVIDIA_MODEL_REGISTRY[cat]["primary"]
-
-    prompt_lower = prompt.lower()
-    scores = {}
-    for category, keywords in ROUTING_KEYWORDS.items():
-        score = sum(1 for kw in keywords if kw in prompt_lower)
-        if score > 0:
-            scores[category] = score
-
-    if scores:
-        best_cat = max(scores, key=scores.get)
-        return best_cat, NVIDIA_MODEL_REGISTRY[best_cat]["primary"]
-
-    # Default: fast general chat
-    return "chat", NVIDIA_MODEL_REGISTRY["chat"]["primary"]
-
-
-def get_model_registry() -> dict:
-    """Return the full model registry dict."""
-    return NVIDIA_MODEL_REGISTRY
-
-
-def get_best_model(category: str) -> str:
-    """Return the primary model ID for a given category."""
-    # Handle remapped/deprecated categories gracefully
-    category_map = {
-        "judge": "reasoning",
-        "finance": "reasoning",
-        "creative": "chat",
-        "medical": "science"
-    }
-    mapped_cat = category_map.get(category, category)
-    entry = NVIDIA_MODEL_REGISTRY.get(mapped_cat)
-    if entry:
-        return entry["primary"]
-    return "meta/llama-3.1-8b-instruct"
-
-
-def list_categories() -> list:
-    """Return all routing categories."""
-    return list(NVIDIA_MODEL_REGISTRY.keys())
-
-
-def save_model_registry():
-    """Persist registry to JSON for external inspection / KB."""
-    os.makedirs(os.path.dirname(MODEL_REGISTRY_FILE), exist_ok=True)
-    try:
-        with open(MODEL_REGISTRY_FILE, "w") as f:
-            json.dump({
-                "updated_at": datetime.datetime.now().isoformat(),
-                "total_nvidia_nim_models": 74,
-                "categories": list(NVIDIA_MODEL_REGISTRY.keys()),
-                "registry": NVIDIA_MODEL_REGISTRY,
-                "routing_keywords": ROUTING_KEYWORDS,
-            }, f, indent=2)
-        print(f"[NVIDIA LLM] Model registry saved to {MODEL_REGISTRY_FILE}")
-    except Exception as e:
-        print(f"[NVIDIA LLM] Failed to save registry: {e}")
-
-
-# ---------------------------------------------------------------------------
-# Keyring resolution and API status testing
-# ---------------------------------------------------------------------------
-_NVIDIA_API_STATUS = {
-    "tested": False,
-    "success": False,
-    "message": "Not initialized",
-    "source": "None"
-}
-
-def resolve_nvidia_api_key() -> tuple[str | None, str]:
-    """
-    Fetch NVIDIA API Key from environment variable, .env files, or system keyring.
-    Returns (api_key, source_description).
-    """
-    # 1. Environment variable
-    key = os.getenv("NVIDIA_API_KEY") or os.getenv("NVIDIA_NIM_API_KEY")
-    if key:
-        clean_key = str(key).replace("\n", "").replace("\r", "").strip()
-        if clean_key:
-            return clean_key, "Environment Variable"
-
-    # 2. Check dotenv explicitly
-    try:
-        from dotenv import find_dotenv, dotenv_values
-        found = find_dotenv(usecwd=True)
-        if found:
-            vals = dotenv_values(found)
-            key = vals.get("NVIDIA_API_KEY") or vals.get("NVIDIA_NIM_API_KEY")
-            if key:
-                clean_key = str(key).replace("\n", "").replace("\r", "").strip()
-                if clean_key:
-                    os.environ["NVIDIA_API_KEY"] = clean_key
-                    return clean_key, f"Dotenv File ({os.path.basename(found)})"
-    except Exception:
-        pass
-
-    # 3. System Keyring
-    if keyring:
-        candidate_services = ["system", "nvidia", "nsedash", "NVIDIA_API_KEY", "NVIDIA", "nvidia_nim"]
-        candidate_usernames = ["NVIDIA_API_KEY", "api_key", "key", "default"]
-        for svc in candidate_services:
-            for usr in candidate_usernames:
-                try:
-                    secret = keyring.get_password(svc, usr)
-                    if secret:
-                        clean_secret = str(secret).replace("\n", "").replace("\r", "").strip()
-                        if clean_secret:
-                            os.environ["NVIDIA_API_KEY"] = clean_secret
-                            return clean_secret, f"System Keyring ({svc}/{usr})"
-                except Exception:
-                    pass
-
-    return None, "None"
-
-
-def test_nvidia_api_connection(api_key: str = None) -> tuple[bool, str]:
-    """
-    Test NVIDIA NIM API connection with the resolved API key.
-    Returns (success: bool, message: str).
-    """
-    key = api_key or os.getenv("NVIDIA_API_KEY")
-    if not key:
-        return False, "NVIDIA_API_KEY is not set in environment or system keyring"
-
-    headers = {
-        "Authorization": f"Bearer {key.strip()}",
-        "Accept": "application/json"
-    }
-    try:
-        resp = requests.get("https://integrate.api.nvidia.com/v1/models", headers=headers, timeout=5)
-        if resp.status_code == 200:
-            return True, "API connection tested & successful"
-        elif resp.status_code == 401:
-            return False, "NVIDIA API Key Unauthorized (HTTP 401)"
-        else:
-            return False, f"NVIDIA API test returned HTTP status {resp.status_code}"
-    except Exception as err:
-        return False, f"NVIDIA API connection test failed: {err}"
-
-
-def init_nvidia_api_status(force_retest: bool = False) -> dict:
-    """Initialize key resolution and API connection test once at startup."""
-    global _NVIDIA_API_STATUS
-    if _NVIDIA_API_STATUS["tested"] and not force_retest:
-        return _NVIDIA_API_STATUS
-
-    key, source = resolve_nvidia_api_key()
-    if not key:
-        _NVIDIA_API_STATUS = {
-            "tested": True,
-            "success": False,
-            "message": "NVIDIA_API_KEY missing in environment and system keyring",
-            "source": source
-        }
-        return _NVIDIA_API_STATUS
-
-    success, msg = test_nvidia_api_connection(key)
-    _NVIDIA_API_STATUS = {
-        "tested": True,
-        "success": success,
-        "message": msg,
-        "source": source
-    }
-    if success:
-        print(f"[NVIDIA LLM] API Key loaded from {source}. Connection tested and successful.")
-    else:
-        print(f"[NVIDIA LLM] API Key loaded from {source}. Connection test warning: {msg}")
-    return _NVIDIA_API_STATUS
-
-
-def get_nvidia_api_status() -> dict:
-    """Return the cached or freshly initialized NVIDIA API status."""
-    return init_nvidia_api_status()
-
-
-# ---------------------------------------------------------------------------
-# NvidiaLLM — CrewAI-compatible wrapper with intelligent routing
-# ---------------------------------------------------------------------------
-class NvidiaLLM(BaseLLM):
-    """
-    CrewAI-compatible LLM wrapper for NVIDIA NIM API.
-    Supports intelligent routing to specialist models by task category.
-    """
-
-    def __init__(self, **kwargs):
-        # Resolve API key from environment variable first, then fallback to keyring
-        api_key = kwargs.pop("api_key", None)
-        if not api_key:
-            api_key, _ = resolve_nvidia_api_key()
-
-        if api_key:
-            api_key = str(api_key).replace("\n", "").replace("\r", "").strip()
-
-        if not api_key:
-            global _WARNED_MISSING_KEY
-            if not _WARNED_MISSING_KEY:
-                print("WARNING: NVIDIA_API_KEY is not set. Free NVIDIA NIM API features require NVIDIA_API_KEY in environment or keyring.")
-                _WARNED_MISSING_KEY = True
-
-        # Default to fast chat model
-        kwargs["api_key"] = api_key
-        kwargs.setdefault("model", "meta/llama-3.1-8b-instruct")
-        super().__init__(**kwargs)
-
-        self.max_tokens = kwargs.get("max_tokens", 4096)
-        self.temperature = kwargs.get("temperature", 1.0)
-        self.top_p = kwargs.get("top_p", 0.95)
-        self.timeout = kwargs.get("timeout", 120)
-
-    def _raw_call(self, model_id: str, formatted_messages: list, max_tokens: int, timeout: int = None) -> str:
-        """Make a single raw API call with explicit model_id."""
-        effective_timeout = timeout or self.timeout
-        invoke_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Accept": "application/json",
-            "Content-Type": "application/json",
-        }
-        payload = {
-            "model": model_id,
-            "messages": formatted_messages,
-            "max_tokens": max_tokens,
-            "temperature": self.temperature,
-            "top_p": self.top_p,
-            "stream": False,
-        }
-
-        resp = requests.post(invoke_url, headers=headers, json=payload, timeout=effective_timeout)
-        resp.raise_for_status()
-        resp_data = resp.json()
-
-        # Track tokens per model
-        usage = resp_data.get("usage", {})
-        pt = usage.get("prompt_tokens", 0)
-        ct = usage.get("completion_tokens", 0)
-        if pt > 0 or ct > 0:
-            add_tokens(pt, ct, model_id=model_id)
-
-        if "choices" in resp_data and len(resp_data["choices"]) > 0:
-            message = resp_data["choices"][0].get("message", {})
-            if message.get("content"):
-                return message["content"].strip()
-        return ""
-
-    def call(self, messages: list[dict], callbacks: list[Any] | None = None, **kwargs: Any) -> str:
-        """
-        Main entry point. Supports:
-          - hint="coding"/"judge"/"memory" etc. for explicit routing
-          - Automatic keyword-based routing if hint not provided
-          - Fallback chain through category fallback models on failure
-        """
-        # Budget guardrail
-        current_usage = get_token_usage()
-        if current_usage.get("total_tokens", 0) > 10000000:
-            print("[AGENTS DIAG ERROR] Token budget exceeded (10M limit)! Throttling LLM call.")
-            return "[LLM ERROR] Token budget exceeded. API call blocked by guardrail."
-
-        # Normalize messages
-        if isinstance(messages, str):
-            formatted_messages = [{"role": "user", "content": messages}]
-            prompt_text = messages
-        elif isinstance(messages, list):
-            formatted_messages = messages
-            prompt_text = " ".join(m.get("content", "") for m in messages if isinstance(m, dict))
-        else:
-            formatted_messages = [{"role": "user", "content": str(messages)}]
-            prompt_text = str(messages)
-
-        max_tokens = kwargs.get("max_tokens") or self.max_tokens
-
-        # --- Intelligent Routing ---
-        hint = kwargs.get("hint")  # caller can pass hint="coding" etc.
-
-        # Per-category dynamic timeouts (large models need more time on free tier)
-        CATEGORY_TIMEOUTS = {
-            "coding":    180,
-            "reasoning": 300,   # 300s (5 minutes)
-            "memory":    180,
-            "judge":     120,
-            "finance":   300,   # 300s (5 minutes)
-            "vision":    90,
-            "safety":    30,
-            "fast":      20,
-            "chat":      60,
-            "creative":  120,
-            "medical":   60,
-            "embedding": 30,
-        }
-
-        # Per-category max_tokens caps
-        CATEGORY_MAX_TOKENS = {
-            "coding":    8192,
-            "reasoning": 32768,  # 32K context / output
-            "memory":    8192,
-            "judge":     2048,
-            "finance":   32768,  # 32K context / output
-            "chat":      2048,
-            "fast":      512,
-            "creative":  4096,
-            "medical":   2048,
-            "safety":    256,
-            "embedding": 512,
-        }
-
-        # --- Intelligent Routing & Explicit Overrides ---
-        hint = kwargs.get("hint")  # caller can pass hint="coding" etc.
-        explicit_model = kwargs.get("model") or kwargs.get("selected_model") or (self.model if self.model != "meta/llama-3.1-8b-instruct" else None)
-        explicit_timeout = kwargs.get("timeout")
-
-        # Determine category, primary model, and fallbacks
-        if explicit_model:
-            category = hint or "reasoning"
-            model_id = explicit_model
-            registry_entry = NVIDIA_MODEL_REGISTRY.get(category, {})
-            all_fallbacks = registry_entry.get("fallback", [])
-            fallbacks = [m for m in all_fallbacks if m != model_id]
-        else:
-            category, model_id = route_request(prompt_text, hint=hint)
-            registry_entry = NVIDIA_MODEL_REGISTRY.get(category, {})
-            fallbacks = registry_entry.get("fallback", [])
-
-        # Build candidate chain starting with primary/selected model
-        candidates = []
-        for cand in [model_id] + fallbacks:
-            if cand and cand not in candidates:
-                candidates.append(cand)
-
-        # Apply timeout override or category-tuned timeout
-        if explicit_timeout:
-            try:
-                effective_timeout = int(explicit_timeout)
-            except (ValueError, TypeError):
-                effective_timeout = CATEGORY_TIMEOUTS.get(category, self.timeout)
-        else:
-            effective_timeout = CATEGORY_TIMEOUTS.get(category, self.timeout)
-
-        effective_max_tokens = min(max_tokens, CATEGORY_MAX_TOKENS.get(category, max_tokens))
-
-        print(f"[NVIDIA LLM ROUTER] Primary/Selected={model_id} | Category={category} | Total Candidates={len(candidates)} | Overall Budget={effective_timeout}s")
-
-        last_error = None
-        start_t = time.time()
-        detailed_req = kwargs.get("detailed", False)
-
-        for idx, candidate in enumerate(candidates):
-            elapsed_so_far = time.time() - start_t
-            remaining_budget = effective_timeout - elapsed_so_far
-            if remaining_budget <= 3:
-                print(f"[NVIDIA LLM ROUTER] Overall timeout budget ({effective_timeout}s) nearly exhausted ({elapsed_so_far:.1f}s elapsed). Stopping candidate loop.")
-                break
-
-            # Cap each candidate call to 35s max to move quickly to responsive models on free tier
-            cand_timeout = min(35, int(remaining_budget))
-            print(f"[NVIDIA LLM ROUTER] Candidate [{idx+1}/{len(candidates)}]: {candidate} (call timeout={cand_timeout}s, remaining budget={int(remaining_budget)}s)")
-
-            try:
-                result = self._raw_call_detailed(candidate, formatted_messages, effective_max_tokens, timeout=cand_timeout)
-                if result and result.get("text"):
-                    if candidate != model_id:
-                        print(f"[NVIDIA LLM ROUTER] Fallback model successfully generated report: {candidate}")
-                    result["elapsed_seconds"] = round(time.time() - start_t, 2)
-                    return result if detailed_req else result["text"]
-            except requests.exceptions.HTTPError as e:
-                last_error = f"[LLM HTTP ERROR] {e}"
-                if hasattr(e, "response") and e.response is not None and e.response.status_code == 401:
-                    err_msg = "[LLM ERROR] Unauthorized: Invalid NVIDIA_API_KEY."
-                    return {"text": err_msg, "model_id": candidate} if detailed_req else err_msg
-                print(f"[NVIDIA LLM ROUTER] Model {candidate} failed ({e}), trying next candidate...")
-            except requests.exceptions.ConnectionError:
-                last_error = "[LLM OFFLINE] Cannot connect to the NVIDIA API."
-                print(f"[NVIDIA LLM ROUTER] Connection error for {candidate}, trying next candidate...")
-            except requests.exceptions.Timeout:
-                last_error = f"[LLM TIMEOUT] Model {candidate} timed out after {cand_timeout}s."
-                print(f"[NVIDIA LLM ROUTER] Model {candidate} timed out after {cand_timeout}s, trying next candidate...")
-            except Exception as e:
-                last_error = f"[LLM ERROR] {str(e)}"
-                print(f"[NVIDIA LLM ROUTER] Model {candidate} error: {e}, trying next candidate...")
-
-        err_out = last_error or "[LLM ERROR] All models in fallback chain failed or budget exhausted."
-        return {"text": err_out, "model_id": model_id} if detailed_req else err_out
-
-    def _raw_call_detailed(self, model_id: str, formatted_messages: list, max_tokens: int, timeout: int = None) -> dict:
-        """Make a raw API call and return text + usage metadata."""
-        effective_timeout = timeout or self.timeout
-        invoke_url = "https://integrate.api.nvidia.com/v1/chat/completions"
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Accept": "application/json",
-            "Content-Type": "application/json",
-        }
-        payload = {
-            "model": model_id,
-            "messages": formatted_messages,
-            "max_tokens": max_tokens,
-            "temperature": self.temperature,
-            "top_p": self.top_p,
-            "stream": False,
-        }
-
-        resp = requests.post(invoke_url, headers=headers, json=payload, timeout=effective_timeout)
-        resp.raise_for_status()
-        resp_data = resp.json()
-
-        usage = resp_data.get("usage", {})
-        pt = usage.get("prompt_tokens", 0)
-        ct = usage.get("completion_tokens", 0)
-        if pt > 0 or ct > 0:
-            add_tokens(pt, ct, model_id=model_id)
-
-        content = ""
-        if "choices" in resp_data and len(resp_data["choices"]) > 0:
-            message = resp_data["choices"][0].get("message", {})
-            if message.get("content"):
-                content = message["content"].strip()
-
-        return {
-            "text": content,
-            "model_id": model_id,
-            "prompt_tokens": pt,
-            "completion_tokens": ct,
-            "total_tokens": pt + ct,
-        }
-
-    def supports_function_calling(self) -> bool:
-        return False
-
-    def supports_stop_words(self) -> bool:
-        return False
-
-
-# ---------------------------------------------------------------------------
-# Convenience factory functions for specific categories
-# ---------------------------------------------------------------------------
-def get_coding_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for coding tasks."""
-    return NvidiaLLM(model=get_best_model("coding"), **kwargs)
-
-def get_reasoning_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for complex reasoning."""
-    return NvidiaLLM(model=get_best_model("reasoning"), **kwargs)
-
-def get_memory_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for memory/summarization."""
-    return NvidiaLLM(model=get_best_model("memory"), **kwargs)
-
-def get_judge_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for LLM-as-a-judge evaluation."""
-    return NvidiaLLM(model=get_best_model("judge"), **kwargs)
-
-def get_finance_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for financial analysis."""
-    return NvidiaLLM(model=get_best_model("finance"), **kwargs)
-
-def get_fast_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for fast/lightweight tasks."""
-    return NvidiaLLM(model=get_best_model("fast"), **kwargs)
-
-def get_creative_llm(**kwargs) -> NvidiaLLM:
-    """Return an NvidiaLLM pre-configured for creative writing and documentation."""
-    return NvidiaLLM(model=get_best_model("creative"), **kwargs)
-
-
-# ---------------------------------------------------------------------------
-# Default singleton (chat model, auto-routing enabled)
-# ---------------------------------------------------------------------------
-_llm_instance = None
-
-def get_nvidia_llm() -> NvidiaLLM:
-    global _llm_instance
-    if _llm_instance is None:
-        _llm_instance = NvidiaLLM()
-    return _llm_instance
-
-
-# ---------------------------------------------------------------------------
-# Eager initialization on main thread + save registry to JSON
-# ---------------------------------------------------------------------------
-try:
-    init_nvidia_api_status()
-    get_nvidia_llm()
-    save_model_registry()
-except Exception as e:
-    print(f"[NVIDIA LLM] Initialization warning: {e}")

nsedash/core/nvidia_model_registry.json

@@ -1,981 +0,0 @@
-{
-  "updated_at": "2026-07-30T01:20:19.883433",
-  "total_nvidia_nim_models": 74,
-  "categories": [
-    "coding",
-    "reasoning",
-    "chat",
-    "memory",
-    "safety",
-    "translation",
-    "embedding",
-    "vision",
-    "fast",
-    "science",
-    "multimodal_vision",
-    "audio"
-  ],
-  "registry": {
-    "coding": {
-      "primary": "deepseek-ai/deepseek-v4-pro",
-      "fallback": [
-        "deepseek-ai/deepseek-v4-flash",
-        "poolside/laguna-xs-2.1",
-        "abacusai/dracarys-llama-3.1-70b-instruct",
-        "nvidia/nv-embedcode-7b-v1"
-      ],
-      "models": {
-        "deepseek-ai/deepseek-v4-pro": {
-          "rating": 5,
-          "context_window": 163840,
-          "description": "State-of-the-art coder. Outstanding performance on code generation and multi-step agent debugging.",
-          "strengths": [
-            "code_gen",
-            "agentic",
-            "debugging"
-          ]
-        },
-        "deepseek-ai/deepseek-v4-flash": {
-          "rating": 4,
-          "context_window": 163840,
-          "description": "High-throughput coding model. Ideal for low-latency coding loops.",
-          "strengths": [
-            "code_gen",
-            "fast",
-            "agentic"
-          ]
-        },
-        "poolside/laguna-xs-2.1": {
-          "rating": 5,
-          "context_window": 32768,
-          "description": "Poolside Laguna XS. Specially trained for agentic coding and long-horizon developer workflows.",
-          "strengths": [
-            "code_gen",
-            "agentic",
-            "planning"
-          ]
-        },
-        "abacusai/dracarys-llama-3.1-70b-instruct": {
-          "rating": 4,
-          "context_window": 131072,
-          "description": "Abacus.AI fine-tune of Llama 3.1 70B, optimized for complex code instruction following.",
-          "strengths": [
-            "code_gen",
-            "instruction"
-          ]
-        },
-        "nvidia/nv-embedcode-7b-v1": {
-          "rating": 3,
-          "context_window": 8192,
-          "description": "NVIDIA specialized coding embedding model for semantic code search.",
-          "strengths": [
-            "embedding",
-            "code_search"
-          ]
-        }
-      }
-    },
-    "reasoning": {
-      "primary": "google/diffusiongemma-26b-a4b-it",
-      "fallback": [
-        "meta/llama-3.1-70b-instruct",
-        "minimaxai/minimax-m3",
-        "z-ai/glm-5.2",
-        "nvidia/nemotron-3-ultra-550b-a55b",
-        "thinkingmachines/inkling",
-        "meta/llama-3.3-70b-instruct",
-        "nvidia/llama-3.3-nemotron-super-49b-v1.5",
-        "meta/llama-3.1-8b-instruct"
-      ],
-      "models": {
-        "minimaxai/minimax-m3": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "MiniMax-M3 SOTA reasoning model. Exceptional financial analysis and long context planning.",
-          "strengths": [
-            "reasoning",
-            "finance",
-            "instruction"
-          ]
-        },
-        "google/diffusiongemma-26b-a4b-it": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "Google DiffusionGemma 26B A4B IT. Next-gen reasoning and fast inference.",
-          "strengths": [
-            "reasoning",
-            "finance",
-            "agentic"
-          ]
-        },
-        "z-ai/glm-5.2": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "GLM-5.2 SOTA reasoning model. Outstanding multi-step planning and financial instruction following.",
-          "strengths": [
-            "reasoning",
-            "finance",
-            "instruction"
-          ]
-        },
-        "nvidia/nemotron-3-ultra-550b-a55b": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "NVIDIA 550B reasoning model. Exceptional logical analysis.",
-          "strengths": [
-            "reasoning",
-            "agentic",
-            "math"
-          ]
-        },
-        "thinkingmachines/inkling": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "Inkling 975B MoE reasoning model. Excels at complex analysis.",
-          "strengths": [
-            "reasoning",
-            "moe"
-          ]
-        },
-        "meta/llama-3.1-70b-instruct": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "Meta Llama 3.1 70B Instruct. High throughput, low latency.",
-          "strengths": [
-            "reasoning",
-            "finance",
-            "fast"
-          ]
-        },
-        "meta/llama-3.3-70b-instruct": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "Meta Llama 3.3 70B Instruct.",
-          "strengths": [
-            "reasoning",
-            "finance"
-          ]
-        },
-        "nvidia/llama-3.3-nemotron-super-49b-v1.5": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "NVIDIA Nemotron Super 49B v1.5.",
-          "strengths": [
-            "reasoning",
-            "agentic"
-          ]
-        },
-        "meta/llama-3.1-8b-instruct": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Llama 3.1 8B Instruct. Fast fallback.",
-          "strengths": [
-            "fast",
-            "general"
-          ]
-        }
-      }
-    },
-    "chat": {
-      "primary": "meta/llama-3.3-70b-instruct",
-      "fallback": [
-        "meta/llama-3.1-70b-instruct",
-        "meta/llama-3.1-8b-instruct",
-        "minimaxai/minimax-m3",
-        "mistralai/mistral-medium-3.5-128b",
-        "google/gemma-4-31b-it",
-        "mistralai/mistral-small-4-119b-2603",
-        "qwen/qwen3.5-122b-a10b",
-        "nvidia/nemotron-3-nano-30b-a3b",
-        "mistralai/ministral-14b-instruct-2512",
-        "bytedance/seed-oss-36b-instruct",
-        "nvidia/nvidia-nemotron-nano-9b-v2",
-        "openai/gpt-oss-20b",
-        "openai/gpt-oss-120b",
-        "sarvamai/sarvam-m",
-        "google/gemma-3n-e4b-it",
-        "google/gemma-3n-e2b-it",
-        "mistralai/mistral-nemotron",
-        "nvidia/llama-3.1-nemotron-nano-8b-v1",
-        "upstage/solar-10.7b-instruct",
-        "mistralai/mixtral-8x7b-instruct-v0.1",
-        "minimaxai/minimax-m2.7",
-        "stepfun-ai/step-3.7-flash",
-        "stepfun-ai/step-3.5-flash"
-      ],
-      "models": {
-        "meta/llama-3.3-70b-instruct": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "Meta Llama 3.3 70B. High-quality instruction following, default conversational model.",
-          "strengths": [
-            "chat",
-            "general",
-            "instruction"
-          ]
-        },
-        "meta/llama-3.1-70b-instruct": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Meta Llama 3.1 70B. Excellent instruction execution and formatting.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "meta/llama-3.1-8b-instruct": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Llama 3.1 8B. Fast, reliable conversational chat model.",
-          "strengths": [
-            "chat",
-            "fast",
-            "general"
-          ]
-        },
-        "minimaxai/minimax-m3": {
-          "rating": 4,
-          "context_window": 40960,
-          "description": "MiniMax M3. Strong general assistant, optimized for long-form dialogue.",
-          "strengths": [
-            "chat",
-            "dialogue"
-          ]
-        },
-        "minimaxai/minimax-m2.7": {
-          "rating": 4,
-          "context_window": 40960,
-          "description": "MiniMax M2.7. Fast conversational LLM.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "stepfun-ai/step-3.7-flash": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "StepFun 3.7 Flash. Ultra-fast high-concurrency conversational assistant.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "stepfun-ai/step-3.5-flash": {
-          "rating": 3,
-          "context_window": 128000,
-          "description": "StepFun 3.5 Flash model.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "mistralai/mistral-medium-3.5-128b": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Mistral Medium 3.5. Balanced instruct model with great reasoning capacity.",
-          "strengths": [
-            "chat",
-            "instruction"
-          ]
-        },
-        "google/gemma-4-31b-it": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Google Gemma 4 31B instruction-tuned model. Top scores in general chat.",
-          "strengths": [
-            "chat",
-            "instruction"
-          ]
-        },
-        "mistralai/mistral-small-4-119b-2603": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "Mistral Small 4 119B. High quality intermediate model for general instruct tasks.",
-          "strengths": [
-            "chat",
-            "instruction"
-          ]
-        },
-        "qwen/qwen3.5-122b-a10b": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "Qwen3.5 122B MoE model, highly conversational.",
-          "strengths": [
-            "chat",
-            "moe"
-          ]
-        },
-        "nvidia/nemotron-3-nano-30b-a3b": {
-          "rating": 3,
-          "context_window": 32768,
-          "description": "NVIDIA Nemotron 3 Nano.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "mistralai/ministral-14b-instruct-2512": {
-          "rating": 3,
-          "context_window": 32768,
-          "description": "Ministral 14B instruction model.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "bytedance/seed-oss-36b-instruct": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "ByteDance Seed OSS 36B.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "nvidia/nvidia-nemotron-nano-9b-v2": {
-          "rating": 3,
-          "context_window": 32768,
-          "description": "NVIDIA Nemotron Nano 9B v2.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "openai/gpt-oss-20b": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "OpenAI GPT-OSS 20B model.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "openai/gpt-oss-120b": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "OpenAI GPT-OSS 120B model.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "sarvamai/sarvam-m": {
-          "rating": 3,
-          "context_window": 8192,
-          "description": "Sarvam-M multilingual chat model.",
-          "strengths": [
-            "chat",
-            "multilingual"
-          ]
-        },
-        "google/gemma-3n-e4b-it": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Google Gemma 3N E4B model.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "google/gemma-3n-e2b-it": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Google Gemma 3N E2B model.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "mistralai/mistral-nemotron": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Mistral-Nemotron collaboration model.",
-          "strengths": [
-            "chat",
-            "general"
-          ]
-        },
-        "nvidia/llama-3.1-nemotron-nano-8b-v1": {
-          "rating": 3,
-          "context_window": 128000,
-          "description": "NVIDIA tuned Llama 3.1 Nano 8B.",
-          "strengths": [
-            "chat",
-            "fast"
-          ]
-        },
-        "upstage/solar-10.7b-instruct": {
-          "rating": 3,
-          "context_window": 4096,
-          "description": "Upstage Solar 10.7B instruction-tuned model.",
-          "strengths": [
-            "chat",
-            "instruction"
-          ]
-        },
-        "mistralai/mixtral-8x7b-instruct-v0.1": {
-          "rating": 3,
-          "context_window": 32768,
-          "description": "Mixtral 8x7B MoE model.",
-          "strengths": [
-            "chat",
-            "moe"
-          ]
-        }
-      }
-    },
-    "memory": {
-      "primary": "meta/llama-4-maverick-17b-128e-instruct",
-      "fallback": [
-        "moonshotai/kimi-k2.6"
-      ],
-      "models": {
-        "meta/llama-4-maverick-17b-128e-instruct": {
-          "rating": 5,
-          "context_window": 1000000,
-          "description": "Llama 4 Maverick. 1M token context, perfect for large codebase digests.",
-          "strengths": [
-            "summarization",
-            "long_context",
-            "memory"
-          ]
-        },
-        "moonshotai/kimi-k2.6": {
-          "rating": 4,
-          "context_window": 131072,
-          "description": "Kimi K2.6. 128K context window, highly accurate long-document analysis.",
-          "strengths": [
-            "summarization",
-            "long_context"
-          ]
-        }
-      }
-    },
-    "safety": {
-      "primary": "nvidia/nemotron-3.5-content-safety",
-      "fallback": [
-        "meta/llama-guard-4-12b",
-        "nvidia/gliner-pii",
-        "nvidia/llama-3.1-nemotron-safety-guard-8b-v3"
-      ],
-      "models": {
-        "nvidia/nemotron-3.5-content-safety": {
-          "rating": 5,
-          "context_window": 4096,
-          "description": "NVIDIA specialized content safety model.",
-          "strengths": [
-            "safety",
-            "moderation"
-          ]
-        },
-        "meta/llama-guard-4-12b": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Llama Guard 4 12B safety classifier.",
-          "strengths": [
-            "safety",
-            "moderation"
-          ]
-        },
-        "nvidia/gliner-pii": {
-          "rating": 4,
-          "context_window": 8192,
-          "description": "GLiNER PII recognition model for anonymization.",
-          "strengths": [
-            "safety",
-            "pii_redaction"
-          ]
-        },
-        "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": {
-          "rating": 4,
-          "context_window": 8192,
-          "description": "NVIDIA safety guard 8B v3.",
-          "strengths": [
-            "safety",
-            "moderation"
-          ]
-        }
-      }
-    },
-    "translation": {
-      "primary": "nvidia/riva-translate-4b-instruct-v1.1",
-      "models": {
-        "nvidia/riva-translate-4b-instruct-v1.1": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Riva machine translation model.",
-          "strengths": [
-            "translation"
-          ]
-        }
-      }
-    },
-    "embedding": {
-      "primary": "nvidia/nv-embedcode-7b-v1",
-      "fallback": [
-        "nvidia/nemotron-3-embed-1b",
-        "nvidia/rerank-qa-mistral-4b"
-      ],
-      "models": {
-        "nvidia/nv-embedcode-7b-v1": {
-          "rating": 4,
-          "context_window": 8192,
-          "description": "NVIDIA specialized code embedding model.",
-          "strengths": [
-            "embedding",
-            "code_search"
-          ]
-        },
-        "nvidia/nemotron-3-embed-1b": {
-          "rating": 3,
-          "context_window": 512,
-          "description": "Nemotron-3 1B embedding model.",
-          "strengths": [
-            "embedding"
-          ]
-        },
-        "nvidia/rerank-qa-mistral-4b": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Mistral-based QA reranker model.",
-          "strengths": [
-            "rerank",
-            "rag"
-          ]
-        }
-      }
-    },
-    "vision": {
-      "primary": "meta/llama-3.2-90b-vision-instruct",
-      "fallback": [
-        "meta/llama-3.2-11b-vision-instruct",
-        "google/diffusiongemma-26b-a4b-it",
-        "google/paligemma",
-        "nvidia/nemotron-nano-12b-v2-vl",
-        "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"
-      ],
-      "models": {
-        "meta/llama-3.2-90b-vision-instruct": {
-          "rating": 5,
-          "context_window": 128000,
-          "description": "Meta Llama 3.2 90B Vision. High-end image and chart understanding.",
-          "strengths": [
-            "vision",
-            "chart_analysis"
-          ]
-        },
-        "meta/llama-3.2-11b-vision-instruct": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Meta Llama 3.2 11B Vision. Efficient image captioning and Q&A.",
-          "strengths": [
-            "vision",
-            "fast"
-          ]
-        },
-        "google/diffusiongemma-26b-a4b-it": {
-          "rating": 4,
-          "context_window": 8192,
-          "description": "DiffusionGemma 26B multimodal visual instruction model.",
-          "strengths": [
-            "vision",
-            "diffusion"
-          ]
-        },
-        "google/paligemma": {
-          "rating": 3,
-          "context_window": 4096,
-          "description": "Google PaliGemma multimodal model.",
-          "strengths": [
-            "vision",
-            "vqa"
-          ]
-        },
-        "nvidia/nemotron-nano-12b-v2-vl": {
-          "rating": 4,
-          "context_window": 32768,
-          "description": "NVIDIA Nemotron Nano 12B VL model.",
-          "strengths": [
-            "vision",
-            "vl"
-          ]
-        },
-        "nvidia/llama-3.1-nemotron-nano-vl-8b-v1": {
-          "rating": 3,
-          "context_window": 128000,
-          "description": "NVIDIA tuned Llama 3.1 Nano VL 8B.",
-          "strengths": [
-            "vision",
-            "fast"
-          ]
-        }
-      }
-    },
-    "fast": {
-      "primary": "meta/llama-3.2-3b-instruct",
-      "fallback": [
-        "meta/llama-3.2-1b-instruct",
-        "nvidia/nemotron-mini-4b-instruct",
-        "google/gemma-2-2b-it"
-      ],
-      "models": {
-        "meta/llama-3.2-3b-instruct": {
-          "rating": 4,
-          "context_window": 128000,
-          "description": "Llama 3.2 3B. Fast, lightweight edge-tier chatbot.",
-          "strengths": [
-            "fast",
-            "lightweight"
-          ]
-        },
-        "meta/llama-3.2-1b-instruct": {
-          "rating": 3,
-          "context_window": 128000,
-          "description": "Llama 3.2 1B. Compact model for low token footprint.",
-          "strengths": [
-            "fast",
-            "lightweight"
-          ]
-        },
-        "nvidia/nemotron-mini-4b-instruct": {
-          "rating": 3,
-          "context_window": 4096,
-          "description": "Nemotron Mini 4B instruction model.",
-          "strengths": [
-            "fast",
-            "compact"
-          ]
-        },
-        "google/gemma-2-2b-it": {
-          "rating": 3,
-          "context_window": 8192,
-          "description": "Google Gemma-2 2B instruct.",
-          "strengths": [
-            "fast",
-            "compact"
-          ]
-        }
-      }
-    },
-    "science": {
-      "primary": "nvidia/esmfold",
-      "fallback": [
-        "nvidia/esm2-650m"
-      ],
-      "models": {
-        "nvidia/esmfold": {
-          "rating": 5,
-          "context_window": 1024,
-          "description": "ESMFold model for direct protein structure prediction from sequence.",
-          "strengths": [
-            "science",
-            "protein_folding"
-          ]
-        },
-        "nvidia/esm2-650m": {
-          "rating": 4,
-          "context_window": 1024,
-          "description": "ESM-2 650M parameter protein language model.",
-          "strengths": [
-            "science",
-            "protein_language"
-          ]
-        }
-      }
-    },
-    "multimodal_vision": {
-      "primary": "nvidia/cosmos3-nano",
-      "fallback": [
-        "nvidia/cosmos3-nano-reasoner",
-        "nvidia/ai-synthetic-video-detector",
-        "nvidia/active-speaker-detection",
-        "nvidia/ising-calibration-1-35b-a3b",
-        "nvidia/cosmos-transfer2.5-2b",
-        "nvidia/streampetr",
-        "nvidia/cosmos-transfer1-7b",
-        "nvidia/sparsedrive",
-        "nvidia/bevformer"
-      ],
-      "models": {
-        "nvidia/cosmos3-nano": {
-          "rating": 5,
-          "context_window": 8192,
-          "description": "Cosmos-3 Nano spatial video model.",
-          "strengths": [
-            "video",
-            "spatial"
-          ]
-        },
-        "nvidia/cosmos3-nano-reasoner": {
-          "rating": 5,
-          "context_window": 8192,
-          "description": "Cosmos-3 Nano reasoning visual agent.",
-          "strengths": [
-            "video",
-            "reasoning"
-          ]
-        },
-        "nvidia/ai-synthetic-video-detector": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Synthetic video generation detector.",
-          "strengths": [
-            "video",
-            "detection"
-          ]
-        },
-        "nvidia/active-speaker-detection": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Maxine active speaker detection.",
-          "strengths": [
-            "video",
-            "maxine"
-          ]
-        },
-        "nvidia/ising-calibration-1-35b-a3b": {
-          "rating": 3,
-          "context_window": 4096,
-          "description": "Ising calibration vision model.",
-          "strengths": [
-            "video"
-          ]
-        },
-        "nvidia/cosmos-transfer2.5-2b": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Cosmos Transfer 2.5 2B video diffusion helper.",
-          "strengths": [
-            "video"
-          ]
-        },
-        "nvidia/streampetr": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "StreamPETR 3D object detection for autonomous driving.",
-          "strengths": [
-            "driving",
-            "3d_detection"
-          ]
-        },
-        "nvidia/cosmos-transfer1-7b": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Cosmos Transfer 1 7B video model.",
-          "strengths": [
-            "video"
-          ]
-        },
-        "nvidia/sparsedrive": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "SparseDrive planning for autonomous vehicles.",
-          "strengths": [
-            "driving"
-          ]
-        },
-        "nvidia/bevformer": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "BEVFormer bird's-eye-view camera perception.",
-          "strengths": [
-            "driving",
-            "perception"
-          ]
-        }
-      }
-    },
-    "audio": {
-      "primary": "nvidia/nemotron-voicechat",
-      "fallback": [
-        "nvidia/magpie-tts-zeroshot",
-        "nvidia/background-noise-removal",
-        "nvidia/studio-voice-enhance"
-      ],
-      "models": {
-        "nvidia/nemotron-voicechat": {
-          "rating": 5,
-          "context_window": 4096,
-          "description": "Nemotron model optimized for voice-based conversational turn-taking.",
-          "strengths": [
-            "voice",
-            "chat"
-          ]
-        },
-        "nvidia/magpie-tts-zeroshot": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Magpie zero-shot Text-to-Speech synthesis.",
-          "strengths": [
-            "voice",
-            "tts"
-          ]
-        },
-        "nvidia/background-noise-removal": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Maxine background noise removal audio NIM.",
-          "strengths": [
-            "audio",
-            "maxine"
-          ]
-        },
-        "nvidia/studio-voice-enhance": {
-          "rating": 4,
-          "context_window": 4096,
-          "description": "Maxine Studio voice enhance audio NIM.",
-          "strengths": [
-            "audio",
-            "maxine"
-          ]
-        }
-      }
-    }
-  },
-  "routing_keywords": {
-    "coding": [
-      "code",
-      "python",
-      "function",
-      "class",
-      "debug",
-      "refactor",
-      "implement",
-      "syntax",
-      "algorithm",
-      "programming",
-      "script",
-      "module",
-      "bug",
-      "fix",
-      "rewrite",
-      "agent_workflow",
-      "agent"
-    ],
-    "reasoning": [
-      "reason",
-      "analyze",
-      "plan",
-      "strategy",
-      "optimize",
-      "complex",
-      "math",
-      "calculate",
-      "evaluate",
-      "multi-step",
-      "explain",
-      "compare",
-      "decide",
-      "judge",
-      "score",
-      "audit",
-      "rate",
-      "hallucin",
-      "assess",
-      "review trajectory",
-      "llm-as-a-judge",
-      "finance",
-      "stock",
-      "trading",
-      "market",
-      "portfolio",
-      "financial",
-      "investment",
-      "equity",
-      "backtest",
-      "capital"
-    ],
-    "memory": [
-      "summarize",
-      "memory",
-      "kb",
-      "knowledge",
-      "log",
-      "history",
-      "update memory",
-      "compress",
-      "recall",
-      "persist"
-    ],
-    "chat": [
-      "chat",
-      "write",
-      "creative",
-      "document",
-      "report",
-      "essay",
-      "story",
-      "generate text"
-    ],
-    "safety": [
-      "safety",
-      "harmful",
-      "pii",
-      "content moderation",
-      "guardrail",
-      "toxic",
-      "filter",
-      "censor"
-    ],
-    "fast": [
-      "quick",
-      "fast",
-      "brief",
-      "short",
-      "classify",
-      "yes/no",
-      "simple",
-      "ping"
-    ],
-    "science": [
-      "science",
-      "protein",
-      "esmfold",
-      "esm2",
-      "biology",
-      "molecule",
-      "structure",
-      "medical",
-      "health",
-      "clinical",
-      "patient",
-      "diagnosis",
-      "symptom",
-      "drug"
-    ],
-    "vision": [
-      "image",
-      "chart",
-      "graph",
-      "visual",
-      "picture",
-      "plot",
-      "screenshot",
-      "vision"
-    ],
-    "multimodal_vision": [
-      "cosmos",
-      "video",
-      "driving",
-      "bevformer",
-      "sparsedrive",
-      "calibration"
-    ],
-    "audio": [
-      "voice",
-      "tts",
-      "speech",
-      "noise",
-      "audio",
-      "sound",
-      "enhance"
-    ],
-    "translation": [
-      "translate",
-      "riva",
-      "language",
-      "bilingual",
-      "multilingual"
-    ]
-  }
-}

nsedash/core/portfolio.py

@@ -1,311 +0,0 @@
-"""
-nsedash.core.portfolio
-======================
-ICICI portfolio import (PortFolioEqtSummary.csv mapped through
-NSEScripMaster.txt inside SecurityMaster.zip) and portfolio summary
-calculation. Parity with v4.5 ``load_portfolio_data`` and
-``update_portfolio_summary`` minus the UI coupling.
-"""
-
-from __future__ import annotations
-
-import logging
-import zipfile
-from pathlib import Path
-
-import pandas as pd
-
-from ..config import Paths
-
-
-class PortfolioImportError(Exception):
-    """Raised with a user-presentable message when the import cannot proceed."""
-
-
-def clean_symbol(sym: str) -> str:
-    """Clean standard ticker symbols from common suffixes like .NS and -EQ."""
-    sym = str(sym).strip().upper()
-    if sym.endswith(".NS"):
-        sym = sym[:-3]
-    if sym.endswith("-EQ"):
-        sym = sym[:-3]
-    return sym
-
-
-def identify_broker(file_name: str, columns: list[str]) -> str:
-    """Identify the broker based on filename and column header heuristics."""
-    filename_lower = file_name.lower()
-    normalized_cols = [str(c).strip().lower() for c in columns]
-
-    # Heuristics based on filename
-    if 'icici' in filename_lower or 'portfolioeqtsummary' in filename_lower:
-        return "ICICIDirect"
-    elif 'hdfc' in filename_lower:
-        return "HDFCSecurities"
-    elif 'groww' in filename_lower:
-        return "Groww"
-    elif 'angel' in filename_lower:
-        return "AngelOne"
-    elif 'upstox' in filename_lower:
-        return "Upstox"
-    elif 'zerodha' in filename_lower or 'holdings' in filename_lower:
-        return "Zerodha"
-
-    # Heuristics based on headers
-    if 'average cost price' in normalized_cols or 'stock symbol' in normalized_cols:
-        return "ICICIDirect"
-    elif 'avg. cost' in normalized_cols or 'qty.' in normalized_cols:
-        return "Zerodha"
-    elif 'trading_symbol' in normalized_cols or 'groww_locked_quantity' in normalized_cols:
-        return "Groww"
-    elif 'average acquisition price' in normalized_cols or 'scrip name' in normalized_cols:
-        return "AngelOne"
-    elif 'holding qty' in normalized_cols:
-        return "Upstox"
-    elif 'sip_indicator' in normalized_cols or 'mtf_indicator' in normalized_cols or 'company_name' in normalized_cols:
-        return "HDFCSecurities"
-
-    return "Generic"
-
-
-def identify_columns(df_columns: list[str]) -> tuple[str | None, str | None, str | None]:
-    """Intelligently map columns to Symbol, Quantity, and Price using candidates list."""
-    normalized_cols = [str(c).strip().lower() for c in df_columns]
-    col_map = {str(c).strip().lower(): c for c in df_columns}
-
-    symbol_candidates = ['stock symbol', 'instrument', 'trading_symbol', 'symbol', 'trading symbol', 'scrip name', 'stock name', 'company name', 'isin name']
-    qty_candidates = ['qty', 'qty.', 'quantity', 'available qty.', 'holding qty', 'current balance', 'available qty', 'qty_available', 'holding quantity', 'quantity']
-    price_candidates = ['average cost price', 'avg. cost', 'average_price', 'average price', 'avg price', 'buy average', 'average cost', 'average acquisition price', 'avg cost', 'buy price', 'avg_price', 'avg_cost']
-
-    symbol_col = None
-    qty_col = None
-    price_col = None
-
-    for cand in symbol_candidates:
-        if cand in normalized_cols:
-            symbol_col = col_map[cand]
-            break
-    if not symbol_col:
-        for c in normalized_cols:
-            if any(cand in c for cand in ['symbol', 'instrument', 'scrip', 'company_name']):
-                symbol_col = col_map[c]
-                break
-
-    for cand in qty_candidates:
-        if cand in normalized_cols:
-            qty_col = col_map[cand]
-            break
-    if not qty_col:
-        for c in normalized_cols:
-            if any(cand in c for cand in ['qty', 'quantity', 'balance']):
-                qty_col = col_map[c]
-                break
-
-    for cand in price_candidates:
-        if cand in normalized_cols:
-            price_col = col_map[cand]
-            break
-    if not price_col:
-        for c in normalized_cols:
-            if any(cand in c for cand in ['avg', 'price', 'cost', 'buy']):
-                price_col = col_map[c]
-                break
-
-    return symbol_col, qty_col, price_col
-
-
-def import_portfolio(file_path: Path | str | None, paths: Paths) -> tuple[list[dict], list[str], str]:
-    """
-    Import holdings from a file path (or fallback to paths.portfolio_file).
-    Identifies the broker, parses columns, cleans symbols, and returns
-    (holdings, unmapped_symbols, broker_name).
-    """
-    if file_path is None:
-        portfolio_file = paths.portfolio_file
-        # Check if CSV exists
-        csv_portfolio_file = Path(str(portfolio_file).replace('.parquet', '.csv'))
-        if csv_portfolio_file.exists():
-            try:
-                df_temp = pd.read_csv(csv_portfolio_file)
-                df_temp.to_parquet(portfolio_file, index=False)
-                csv_portfolio_file.unlink()
-                logging.info(f"Auto-converted portfolio CSV '{csv_portfolio_file}' to Parquet.")
-            except Exception as e:
-                logging.warning(f"Failed to auto-convert portfolio CSV: {e}")
-            file_path = portfolio_file
-        elif portfolio_file.exists():
-            file_path = portfolio_file
-        else:
-            raise PortfolioImportError(f"Portfolio file not found: {portfolio_file}")
-
-    file_path = Path(file_path)
-    is_parquet = file_path.suffix.lower() == '.parquet'
-
-    try:
-        if is_parquet:
-            df = pd.read_parquet(file_path)
-        else:
-            df = pd.read_csv(file_path)
-    except Exception as e:
-        raise PortfolioImportError(f"Failed to read portfolio file: {e}")
-
-    if df.empty:
-        raise PortfolioImportError("One of the CSV files is empty.")
-
-    # Strip columns
-    df.columns = df.columns.str.strip().str.replace('"', '')
-
-    # Check if this is a previously standardized file by us
-    if 'Broker' in df.columns and 'SYMBOL' in df.columns:
-        broker_name = df['Broker'].iloc[0] if not df['Broker'].empty else "Generic"
-        holdings = []
-        for _, row in df.iterrows():
-            holdings.append({
-                "SYMBOL": str(row['SYMBOL']),
-                "Quantity": float(row['Quantity']) if pd.notna(row['Quantity']) else 0.0,
-                "Price": float(row['Price']) if pd.notna(row['Price']) else 0.0,
-            })
-        return holdings, [], broker_name
-
-    broker_name = identify_broker(file_path.name, df.columns)
-
-    symbol_col, qty_col, price_col = identify_columns(df.columns)
-    if not symbol_col or not qty_col or not price_col:
-        if len(df.columns) >= 3:
-            symbol_col = df.columns[0]
-            qty_col = df.columns[1]
-            price_col = df.columns[2]
-        else:
-            raise PortfolioImportError(
-                f"Could not map columns for broker '{broker_name}'. Expected columns for Symbol, Quantity, and Price."
-            )
-
-    holdings = []
-    unmapped = []
-
-    if broker_name == "ICICIDirect":
-        # Load ICICI Direct specific mapping
-        mapping_zip_file = paths.security_master_zip
-        if not mapping_zip_file.exists():
-            raise PortfolioImportError(f"Mapping zip file not found: {mapping_zip_file}")
-
-        mapping_parquet = paths.data_dir / 'NSEScripMaster.parquet'
-        need_convert = True
-        if mapping_parquet.exists():
-            if mapping_parquet.stat().st_mtime > mapping_zip_file.stat().st_mtime:
-                need_convert = False
-
-        if need_convert:
-            with zipfile.ZipFile(mapping_zip_file, 'r') as z:
-                if 'NSEScripMaster.txt' not in z.namelist():
-                    raise PortfolioImportError("NSEScripMaster.txt not found in SecurityMaster.zip")
-                z.extract('NSEScripMaster.txt', path=paths.data_dir)
-            mapping_txt = paths.data_dir / 'NSEScripMaster.txt'
-            if not mapping_txt.exists():
-                raise PortfolioImportError(f"Mapping file not found: {mapping_txt}")
-            try:
-                mapping_df = pd.read_csv(mapping_txt)
-                mapping_df.to_parquet(mapping_parquet, index=False)
-                mapping_txt.unlink()
-                logging.info("Auto-converted mapping txt to NSEScripMaster.parquet.")
-            except Exception as e:
-                raise PortfolioImportError(f"Failed to convert mapping CSV to Parquet: {e}")
-
-        try:
-            mapping_df = pd.read_parquet(mapping_parquet)
-        except Exception as e:
-            raise PortfolioImportError(f"Failed to read mapping Parquet: {e}")
-
-        mapping_df.columns = mapping_df.columns.str.strip().str.replace('"', '')
-
-        try:
-            icici_to_nse = dict(zip(mapping_df['ShortName'].astype(str).str.strip(), mapping_df['ExchangeCode'].astype(str).str.strip()))
-        except KeyError as ke:
-            raise PortfolioImportError(f"Missing expected column in mapping file: {ke}")
-
-        for _, row in df.iterrows():
-            raw_sym = str(row[symbol_col]).strip()
-            nse_sym = icici_to_nse.get(raw_sym, None)
-            if nse_sym:
-                try:
-                    qty = float(row[qty_col]) if pd.notna(row[qty_col]) else 0.0
-                    price = float(row[price_col]) if pd.notna(row[price_col]) else 0.0
-                    holdings.append({
-                        "SYMBOL": nse_sym,
-                        "Quantity": qty,
-                        "Price": price
-                    })
-                except (ValueError, KeyError) as e:
-                    logging.warning(f"Failed to parse row for {raw_sym}: {e}")
-            else:
-                unmapped.append(raw_sym)
-    else:
-        # Standard cleaning for Zerodha, Groww, AngelOne, Upstox, HDFCSecurities, Generic
-        for _, row in df.iterrows():
-            raw_sym = str(row[symbol_col]).strip()
-            nse_sym = clean_symbol(raw_sym)
-            if nse_sym:
-                try:
-                    qty = float(row[qty_col]) if pd.notna(row[qty_col]) else 0.0
-                    price = float(row[price_col]) if pd.notna(row[price_col]) else 0.0
-                    holdings.append({
-                        "SYMBOL": nse_sym,
-                        "Quantity": qty,
-                        "Price": price
-                    })
-                except (ValueError, KeyError) as e:
-                    logging.warning(f"Failed to parse row for {raw_sym}: {e}")
-            else:
-                unmapped.append(raw_sym)
-
-    return holdings, unmapped, broker_name
-
-
-def import_icici_portfolio(paths: Paths) -> tuple[list[dict], list[str]]:
-    """Thin backwards-compatible wrapper calling import_portfolio."""
-    holdings, unmapped, _ = import_portfolio(None, paths)
-    return holdings, unmapped
-
-
-
-def compute_portfolio_summary(cells: list[dict]) -> dict:
-    """
-    Aggregate the portfolio metrics displayed in the top bar.
-
-    Each cell dict must provide: Quantity, Price (cost), current_cmp,
-    prev_close (0.0 when unknown — the caller applies the v4.5 fallback of
-    prev_close := current_cmp when no historic row exists).
-    """
-    total_invested = 0.0
-    total_current_value = 0.0
-    total_day_gain = 0.0
-    total_previous_close_value = 0.0
-    for c in cells:
-        qty = float(c.get('Quantity', 0.0) or 0.0)
-        cost_price = float(c.get('Price', 0.0) or 0.0)
-        current_cmp = float(c.get('current_cmp', 0.0) or 0.0)
-        prev_close = float(c.get('prev_close', 0.0) or 0.0)
-        if prev_close == 0.0 and current_cmp != 0.0:
-            prev_close = current_cmp
-        total_invested += qty * cost_price
-        total_current_value += qty * current_cmp
-        if prev_close > 0 and current_cmp > 0:
-            total_day_gain += qty * (current_cmp - prev_close)
-            total_previous_close_value += qty * prev_close
-    absolute_returns = total_current_value - total_invested
-    day_gain_pct = (
-        total_day_gain / total_previous_close_value * 100
-        if total_previous_close_value != 0 else 0.0
-    )
-    abs_returns_pct = (
-        absolute_returns / total_invested * 100
-        if total_invested != 0 else 0.0
-    )
-    return {
-        "invested": total_invested,
-        "current_value": total_current_value,
-        "day_gain_abs": total_day_gain,
-        "day_gain_pct": day_gain_pct,
-        "abs_returns_abs": absolute_returns,
-        "abs_returns_pct": abs_returns_pct,
-    }

nsedash/core/recommender_bridge.py

@@ -1,110 +0,0 @@
-"""
-nsedash.core.recommender_bridge
-===============================
-Bridge to the external ``recommender.SymbolRecommender`` module (LSTM
-buy/sell). v4.5 hard-imported ``recommender`` (and TensorFlow transitively)
-at module import time, so the entire dashboard failed to start when the
-optional deep-learning stack was absent (AUDIT FIX H-9). This bridge imports
-lazily and degrades to ``N/A`` recommendations when unavailable, while
-preserving full behaviour when the module is present.
-"""
-
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-
-from ..config import Recommendation
-
-_MIN_ROWS_FOR_PREDICTION = 61  # parity with v4.5 run_buy_sell_recommendation
-
-
-def _load_recommender(recommender_dir: Path):
-    try:
-        import sys
-        nsereco_path = str(Path(__file__).resolve().parents[2] / 'nsereco')
-        if nsereco_path not in sys.path:
-            sys.path.append(nsereco_path)
-        from recommender import SymbolRecommender  # noqa: PLC0415
-        return SymbolRecommender(recommender_dir)
-    except Exception as e:
-        logging.warning(f"SymbolRecommender unavailable: {e}")
-        return None
-
-
-def recommender_available(recommender_dir: Path) -> bool:
-    return _load_recommender(recommender_dir) is not None
-
-
-def run_recommendations(recommender_dir: Path, symbols: list[str],
-                        history_lookup) -> list[dict]:
-    """
-    Generate buy/sell predictions for the given symbols.
-
-    ``history_lookup(symbol)`` returns the symbol's historical DataFrame
-    sorted ascending by TRAN_DATE.
-
-    Returns a list of {'symbol','action','confidence'} sorted by confidence
-    descending — the same records v4.5 showed in the results dialog, plus
-    one entry per skipped symbol carrying Recommendation.NA.
-    """
-    recommender = _load_recommender(recommender_dir)
-    results: list[dict] = []
-    for symbol in symbols:
-        model_file = Path(recommender_dir) / f"{symbol}.h5"
-        if recommender is None or not model_file.exists():
-            if recommender is not None:
-                logging.warning(f"[{symbol}] No trained model found at {model_file}.")
-            results.append({
-                "symbol": symbol,
-                "action": Recommendation.NA.value,
-                "confidence": 0.0,
-                "predicted": False,
-            })
-            continue
-        symbol_df = history_lookup(symbol)
-        if symbol_df is not None and not symbol_df.empty \
-                and len(symbol_df) >= _MIN_ROWS_FOR_PREDICTION:
-            tracker = getattr(recommender, 'tracker_data', {}) or {}
-            entry = tracker.get(symbol, {})
-            confidence = entry.get('confidence_score', 0.0)
-            action = entry.get('recommendation', Recommendation.NA)
-            action = action.value if isinstance(action, Recommendation) else str(action)
-            results.append({
-                "symbol": symbol,
-                "action": action,
-                "confidence": float(confidence),
-                "predicted": True,
-            })
-        else:
-            logging.warning(f"[{symbol}] Skipping prediction: insufficient data.")
-            results.append({
-                "symbol": symbol,
-                "action": Recommendation.NA.value,
-                "confidence": 0.0,
-                "predicted": False,
-            })
-    results.sort(key=lambda r: r["confidence"], reverse=True)
-    return results
-
-
-def train_all_models(recommender_dir: Path, symbols: list[str],
-                     history_lookup) -> dict:
-    """
-    Train LSTM models for all symbols. v4.5 delegated to
-    ``SymbolRecommender.train_all_async(symbols, lookup, tk_root)``; here the
-    web job system supplies the asynchrony, so the synchronous variant is
-    preferred when exposed by the module, falling back to the async API with
-    a ``None`` UI handle.
-    """
-    recommender = _load_recommender(recommender_dir)
-    if recommender is None:
-        raise RuntimeError(
-            "SymbolRecommender module (and its TensorFlow dependency) is not "
-            "installed; deep-learning training is unavailable."
-        )
-    if hasattr(recommender, 'train_all'):
-        recommender.train_all(symbols, history_lookup)
-    else:
-        recommender.train_all_async(symbols, history_lookup, None)
-    return {"trained_symbols": len(symbols)}

nsedash/core/stock_data.py

@@ -1,504 +0,0 @@
-"""
-nsedash.core.stock_data
-=======================
-Headless data layer (no UI coupling). Owns the historical DataFrame and all
-derived analytics. Functional parity with v4.5 ``StockData`` minus the
-Tkinter dialogs, which are replaced by exceptions/return values surfaced to
-the web layer.
-
-Audit fixes applied here:
-  P-4  ``get_data_for_date`` no longer performs an O(rows) boolean scan of
-       the date slice per column assignment per symbol (O(S²) in v4.5);
-       results are accumulated per symbol and joined once.
-  P-5  The 5-day volume-change scan (30 per-day full-column comparisons per
-       symbol in v4.5) is replaced by a single windowed slice per symbol.
-  A-2  ``df['DAILY_RETURN'].fillna(0, inplace=True)`` (chained-assignment,
-       removed in pandas 3.x) replaced with safe column assignment.
-  P-6  The parquet cache is only rewritten when the source CSV has changed
-       (mtime check) instead of on every load.
-  D-1  Data-layer methods no longer call ``messagebox``; errors are logged
-       and raised/returned for the caller to present.
-  T-1  A re-entrant lock guards DataFrame replacement so background jobs and
-       request handlers cannot observe a half-swapped frame.
-"""
-
-from __future__ import annotations
-
-import gc
-import json
-import logging
-import threading
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-
-from ..config import (ANALYSIS_LIMIT, CSV_COLUMNS, CSV_DTYPES,
-                      DATA_RETENTION_DAYS, Paths)
-from ..logging_utils import elapsed_time_logger
-from .live_fetch import LiveFetcher
-
-
-class StockData:
-    """
-    Manages all data I/O, historical analysis, and real-time scraping.
-
-    Responsibilities (unchanged from v4.5):
-        - Loading / caching historical CSV data
-        - Computing returns, volatility, volume changes
-        - Real-time fetching (delegated to LiveFetcher)
-        - Watchlist JSON read/write helpers (delegated to watchlists module)
-        - ETF exclusion list management
-        - Moneycontrol score data access
-    """
-
-    def __init__(self, paths: Paths):
-        self.paths = paths
-        self.paths.ensure_dirs()
-        self._lock = threading.RLock()
-        self.df: pd.DataFrame = pd.DataFrame()
-        self.last_date = None
-        self.two_years_ago = None
-        self.etf_symbols = self.load_etf_symbols()
-        self.fetcher = LiveFetcher(prev_volume_lookup=self.previous_day_volume)
-        self.load_data()
-
-    # ------------------------------------------------------------------
-    # ETF Management
-    # ------------------------------------------------------------------
-
-    def load_etf_symbols(self) -> set:
-        """Load ETF symbols from the JSON exclusion list file."""
-        try:
-            if self.paths.etf_list_file.exists():
-                with open(self.paths.etf_list_file, 'r') as f:
-                    return set(json.load(f))
-            logging.warning(
-                f"ETF list file not found: {self.paths.etf_list_file}. "
-                "No ETFs will be excluded."
-            )
-            return set()
-        except Exception as e:
-            logging.error(
-                f"Error loading ETF symbols from {self.paths.etf_list_file}: {e}"
-            )
-            return set()
-
-    # ------------------------------------------------------------------
-    # Symbol Utilities
-    # ------------------------------------------------------------------
-
-    def get_all_symbols_from_df(self) -> list[str]:
-        """Return list of all unique stock symbols in the loaded DataFrame."""
-        with self._lock:
-            if not self.df.empty:
-                return self.df['SYMBOL'].unique().tolist()
-            return []
-
-    def symbol_history(self, symbol: str) -> pd.DataFrame:
-        """All rows for a symbol sorted ascending by TRAN_DATE."""
-        with self._lock:
-            return self.df[self.df['SYMBOL'] == symbol].sort_values('TRAN_DATE').copy()
-
-    def previous_day_volume(self, symbol: str) -> float:
-        """Latest historic NET_TRDQTY strictly before today (for live fetch)."""
-        with self._lock:
-            df = self.df
-        if df.empty:
-            return 0.0
-        rows = df[df['SYMBOL'] == symbol]
-        today = datetime.now().date()
-        prev = rows[rows['TRAN_DATE'].dt.date < today]
-        if prev.empty:
-            logging.warning(f"No previous day's historic volume found for {symbol}.")
-            return 0.0
-        return float(prev.sort_values('TRAN_DATE').iloc[-1]['NET_TRDQTY'])
-
-    # ------------------------------------------------------------------
-    # Real-time delegation (API parity with v4.5)
-    # ------------------------------------------------------------------
-
-    def fetch_real_time_data(self, symbols, preferred_source="Google"):
-        return self.fetcher.fetch_real_time_data(symbols, preferred_source)
-
-    # ------------------------------------------------------------------
-    # Data Loading
-    # ------------------------------------------------------------------
-
-    @elapsed_time_logger("Loading stock data...")
-    def load_data(self) -> None:
-        """
-        Load nse_processed_data.parquet, trim to the last 2 years,
-        compute DAILY_RETURN and VOLATILITY, and cache as parquet.
-        """
-        try:
-            pq_path = self.paths.data_dir / 'nse_processed_data.parquet'
-
-            if pq_path.exists():
-                logging.info(f"Loading data from Parquet: {pq_path}")
-                df = pd.read_parquet(pq_path, columns=CSV_COLUMNS)
-                # Ensure TRAN_DATE is datetime type
-                if not pd.api.types.is_datetime64_any_dtype(df['TRAN_DATE']):
-                    df['TRAN_DATE'] = pd.to_datetime(df['TRAN_DATE'])
-            else:
-                raise FileNotFoundError(f"Parquet file ({pq_path}) not found.")
-
-            last_date = df['TRAN_DATE'].max()
-            two_years_ago = last_date - timedelta(days=DATA_RETENTION_DAYS)
-            df = df[df['TRAN_DATE'] >= two_years_ago].copy()
-            df = df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True)
-
-            # Daily return (% change in closing price per symbol)
-            df['DAILY_RETURN'] = (
-                df.groupby('SYMBOL', observed=True)['CLOSE_PRICE'].pct_change() * 100
-            )
-            df['DAILY_RETURN'] = df['DAILY_RETURN'].fillna(0)  # AUDIT FIX A-2
-
-            # 30-day rolling volatility (std of daily returns)
-            df['VOLATILITY'] = df.groupby('SYMBOL', observed=True)['DAILY_RETURN'].transform(
-                lambda x: x.rolling(window=30, min_periods=1).std()
-            )
-            df['VOLATILITY'] = df['VOLATILITY'].fillna(0)
-
-            with self._lock:
-                self.df = df
-                self.last_date = last_date
-                self.two_years_ago = two_years_ago
-
-            # Keep extract file updated for compatibility
-            try:
-                pq = self.paths.extract_file
-                if not pq.exists() or pq.stat().st_mtime < pq_path.stat().st_mtime:
-                    df.to_parquet(pq, index=False)
-            except Exception as e:
-                logging.warning(f"Parquet cache write skipped: {e}")
-
-            logging.info(
-                f"Data loaded from {two_years_ago} with "
-                "DAILY_RETURN and VOLATILITY calculated."
-            )
-            gc.collect()
-        except Exception as e:
-            logging.error(f"Failed to load data: {e}")
-            raise
-
-    # ------------------------------------------------------------------
-    # Per-date enrichment
-    # ------------------------------------------------------------------
-
-    @elapsed_time_logger("Loading date data...")
-    def get_data_for_date(self, date) -> pd.DataFrame:
-        """
-        Return enriched DataFrame for a specific trading date, including:
-            WEEK_CHANGE, MONTH_CHANGE, THREE_MONTH_CHANGE, TWO_YEAR_CHANGE,
-            PRICE_HISTORY, VOLUME_HISTORY, AVG_VOLUME, VOLUME_CHANGES.
-        Raises ValueError if no rows exist for the date.
-        """
-        logging.info(f"Getting data for {date}")
-        if isinstance(date, datetime):
-            date = date.date()
-        if isinstance(date, pd.Timestamp):
-            date = date.date()
-        with self._lock:
-            df = self.df
-        date_data = df[df['TRAN_DATE'].dt.date == date].copy()
-        if date_data.empty:
-            raise ValueError(f"No data for {date}")
-
-        grouped = df.groupby('SYMBOL', observed=True)
-        results: dict[str, dict] = {}
-        symbol_errors: list[tuple[str, str]] = []
-
-        for symbol in date_data['SYMBOL'].unique():
-            try:
-                try:
-                    symbol_data = grouped.get_group(symbol)
-                except KeyError:
-                    raise ValueError(f"No historical data for symbol {symbol}")
-                if symbol_data.empty:
-                    raise ValueError(f"No historical data for symbol {symbol}")
-
-                curr_date = date_data.loc[
-                    date_data['SYMBOL'] == symbol, 'TRAN_DATE'
-                ].iloc[0]
-                dates = symbol_data['TRAN_DATE']
-                closes = symbol_data['CLOSE_PRICE']
-                res: dict = {}
-
-                def window_change(days_back: int) -> float:
-                    start = curr_date - timedelta(days=days_back)
-                    prices = closes[(dates <= curr_date) & (dates >= start)]
-                    if prices.empty:
-                        return 0.0
-                    return float((prices.iloc[-1] / prices.iloc[0] - 1) * 100)
-
-                res['WEEK_CHANGE'] = window_change(7)
-                res['MONTH_CHANGE'] = window_change(30)
-                res['THREE_MONTH_CHANGE'] = window_change(90)
-
-                # --- 2-year window: change, histories, average volume ---
-                two_years_ago = curr_date - timedelta(days=365 * 2)
-                mask_2y = (dates <= curr_date) & (dates >= two_years_ago)
-                two_year_data = symbol_data[mask_2y]
-                if not two_year_data.empty:
-                    res['TWO_YEAR_CHANGE'] = float(
-                        (two_year_data['CLOSE_PRICE'].iloc[-1] /
-                         two_year_data['CLOSE_PRICE'].iloc[0] - 1) * 100
-                    )
-                    res['PRICE_HISTORY'] = (
-                        two_year_data[['TRAN_DATE', 'CLOSE_PRICE']].values.tolist()
-                    )
-                    res['VOLUME_HISTORY'] = (
-                        two_year_data[['TRAN_DATE', 'NET_TRDQTY']].values.tolist()
-                    )
-                    res['AVG_VOLUME'] = float(two_year_data['NET_TRDQTY'].mean())
-                else:
-                    res['TWO_YEAR_CHANGE'] = 0.0
-                    res['PRICE_HISTORY'] = None
-                    res['VOLUME_HISTORY'] = None
-                    res['AVG_VOLUME'] = 0.0
-
-                # --- 5-day volume changes (last 6 trading days within a
-                #     30-calendar-day lookback, parity with v4.5) ---
-                lookback_start = curr_date - timedelta(days=29)
-                recent = symbol_data[
-                    (dates <= curr_date) & (dates > lookback_start - timedelta(days=1))
-                ]
-                volumes_asc = recent['NET_TRDQTY'].tail(6).astype(float).tolist()
-                while len(volumes_asc) < 6:
-                    volumes_asc.insert(0, 0.0)
-                volume_changes = []
-                for i in range(1, len(volumes_asc)):
-                    prev_v = volumes_asc[i - 1]
-                    change = (
-                        ((volumes_asc[i] - prev_v) / prev_v) * 100
-                        if prev_v != 0 else 0
-                    )
-                    volume_changes.append(round(change, 2))
-                while len(volume_changes) < 5:
-                    volume_changes.insert(0, 0.0)
-                res['VOLUME_CHANGES'] = volume_changes[-5:]
-
-                results[symbol] = res
-            except Exception as e:
-                logging.error(f"Failed to process calculations for {symbol}: {e}")
-                symbol_errors.append((symbol, str(e)))
-
-        # AUDIT FIX P-4: single vectorised join instead of per-symbol .loc
-        # scans. (Note: SYMBOL is categorical, so plain-string iteration is
-        # used — Series.map on a categorical would try to build categories
-        # from the mapped values, which fails for list-valued columns.)
-        row_symbols = date_data['SYMBOL'].astype(str).tolist()
-        for col, default in [
-            ('WEEK_CHANGE', 0.0), ('MONTH_CHANGE', 0.0),
-            ('THREE_MONTH_CHANGE', 0.0), ('TWO_YEAR_CHANGE', 0.0),
-            ('AVG_VOLUME', 0.0),
-        ]:
-            date_data[col] = pd.Series(
-                [results.get(s, {}).get(col, default) for s in row_symbols],
-                index=date_data.index, dtype=float,
-            )
-        for col in ('PRICE_HISTORY', 'VOLUME_HISTORY', 'VOLUME_CHANGES'):
-            date_data[col] = pd.Series(
-                [results.get(s, {}).get(col) for s in row_symbols],
-                index=date_data.index, dtype=object,
-            )
-
-        if symbol_errors:
-            logging.warning(
-                f"Completed date data load for {date} with "
-                f"{len(symbol_errors)} symbol failures. "
-                f"Failures: {symbol_errors[:10]}..."
-            )
-        logging.info(f"Successful to get data for {date}")
-        return date_data
-
-    # ------------------------------------------------------------------
-    # Volume analytics
-    # ------------------------------------------------------------------
-
-    @staticmethod
-    def _calculate_average_volume_change(symbol_data: pd.DataFrame,
-                                         days_back: int) -> float:
-        """
-        Average daily volume % change for a symbol over ``days_back`` days.
-        ``symbol_data`` must be sorted ascending by TRAN_DATE.
-        """
-        if symbol_data.empty or len(symbol_data) < 2:
-            return 0.0
-        end_date = symbol_data['TRAN_DATE'].iloc[-1]
-        start_date = end_date - timedelta(days=days_back)
-        period = symbol_data[
-            (symbol_data['TRAN_DATE'] <= end_date) &
-            (symbol_data['TRAN_DATE'] > start_date - timedelta(days=1))
-        ]
-        if len(period) < 2:
-            return 0.0
-        vols = period['NET_TRDQTY'].to_numpy(dtype=float)
-        prev, curr = vols[:-1], vols[1:]
-        with np.errstate(divide='ignore', invalid='ignore'):
-            changes = np.where(prev != 0, (curr - prev) / prev * 100, 0.0)
-        return float(changes.mean()) if changes.size else 0.0
-
-    @elapsed_time_logger("Fetching top volume movers...")
-    def get_top_volume_movers(self, duration: str, limit: int = ANALYSIS_LIMIT,
-                              volume_cutoff: float = 0) -> pd.DataFrame:
-        """
-        Rank stocks by average volume % change for the given duration
-        ('1D', '1W', '1M', '3M'). ETFs excluded. Returns DataFrame with
-        SYMBOL, VOLUME_CHANGE_PCT, SECURITY, AVG_VOLUME.
-        """
-        days_map = {"1D": 1, "1W": 7, "1M": 30, "3M": 90}
-        days_back = days_map.get(duration)
-        if days_back is None:
-            logging.error(f"Invalid duration for volume analysis: {duration}")
-            return pd.DataFrame()
-
-        with self._lock:
-            df = self.df
-        latest_date = df['TRAN_DATE'].max()
-        try:
-            all_latest = self.get_data_for_date(latest_date)
-        except Exception as e:
-            logging.error(
-                f"Error getting data for latest date in get_top_volume_movers: {e}"
-            )
-            raise RuntimeError(
-                "Could not retrieve necessary data for volume analysis."
-            ) from e
-        if all_latest.empty:
-            logging.warning(
-                f"No data available for {latest_date} to perform volume analysis."
-            )
-            return pd.DataFrame()
-
-        grouped = df.groupby('SYMBOL', observed=True)
-        top_movers = []
-        for _, row in all_latest.iterrows():
-            symbol = row['SYMBOL']
-            if symbol in self.etf_symbols:
-                continue
-            if "ETF" in str(row['SECURITY']).upper():
-                continue
-            avg_volume = row.get('AVG_VOLUME', 0) or 0
-            if avg_volume < volume_cutoff:
-                continue
-            if duration == "1D":
-                vc = row.get('VOLUME_CHANGES') or []
-                volume_pct_change = vc[4] if len(vc) >= 5 else 0.0
-            else:
-                try:
-                    hist = grouped.get_group(symbol)
-                except KeyError:
-                    hist = pd.DataFrame()
-                volume_pct_change = self._calculate_average_volume_change(
-                    hist, days_back
-                )
-            top_movers.append({
-                "SYMBOL": symbol,
-                "VOLUME_CHANGE_PCT": volume_pct_change,
-                "SECURITY": row['SECURITY'],
-                "AVG_VOLUME": avg_volume,
-            })
-        movers_df = pd.DataFrame(top_movers)
-        if movers_df.empty:
-            return pd.DataFrame()
-        return movers_df.sort_values(
-            by="VOLUME_CHANGE_PCT", ascending=False
-        ).head(limit)
-
-    # ------------------------------------------------------------------
-    # Moneycontrol scores
-    # ------------------------------------------------------------------
-
-    @elapsed_time_logger("Loading and sorting Moneycontrol data by score...")
-    def get_top_mc_score_movers(self, limit: int = ANALYSIS_LIMIT) -> pd.DataFrame:
-        """
-        Read nse_dashboard_mclink.json and return the top ``limit`` symbols
-        ranked by MC_SCORE descending.
-        """
-        try:
-            if not self.paths.mc_link_file.exists():
-                logging.warning(
-                    f"Moneycontrol link file not found: {self.paths.mc_link_file}."
-                )
-                return pd.DataFrame()
-            with open(self.paths.mc_link_file, 'r') as f:
-                mc_data = json.load(f)
-            processed = [
-                {"SYMBOL": symbol, "MC_SCORE": d['MC_SCORE'],
-                 "SECURITY": d.get('MC_COMPANY_NAME', '')}
-                for symbol, d in mc_data.items() if 'MC_SCORE' in d
-            ]
-            if not processed:
-                logging.warning("No valid MC_SCORE data found.")
-                return pd.DataFrame()
-            mc_df = pd.DataFrame(processed)
-            return mc_df.sort_values(by="MC_SCORE", ascending=False).head(limit)
-        except (json.JSONDecodeError, FileNotFoundError) as e:
-            logging.error(f"Error reading Moneycontrol data: {e}")
-            raise RuntimeError(f"Could not read Moneycontrol data: {e}") from e
-        except Exception as e:
-            logging.error(f"Unexpected error getting top MC score movers: {e}")
-            raise RuntimeError(f"Failed to analyze top MC scores: {e}") from e
-
-    # ------------------------------------------------------------------
-    # Moneycontrol link store (shared by all stock cards; AUDIT FIX P-7:
-    # v4.5 re-read the JSON from disk once per card per refresh)
-    # ------------------------------------------------------------------
-
-    def load_mc_link_data(self) -> dict:
-        """Load nse_dashboard_mclink.json; create an empty file if missing."""
-        try:
-            p = Path(self.paths.mc_link_file)
-            if p.exists():
-                with open(p, 'r') as f:
-                    return json.load(f)
-            with open(p, 'w') as f:
-                json.dump({}, f)
-            return {}
-        except Exception as e:
-            logging.error(
-                f"Error loading Moneycontrol links from {self.paths.mc_link_file}: {e}"
-            )
-            return {}
-
-    def save_mc_link_data(self, data: dict) -> None:
-        """Write the updated Moneycontrol links dict back to JSON."""
-        try:
-            with open(self.paths.mc_link_file, 'w') as f:
-                json.dump(data, f, indent=4)
-            print(f"[ACTIVITY] Moneycontrol links config file updated: {self.paths.mc_link_file.name}", flush=True)
-        except Exception as e:
-            logging.error(
-                f"Error saving Moneycontrol links to {self.paths.mc_link_file}: {e}"
-            )
-
-    def set_mc_link(self, symbol: str, security: str, link: str) -> dict:
-        """
-        Register/replace the Moneycontrol URL for a symbol (parity with the
-        v4.5 URL-prompt flow, including MC_COMPANY_CODE extraction).
-        """
-        input_link = (link or "").strip()
-        if not input_link:
-            raise ValueError("Please enter the Moneycontrol URL.")
-        url_parts = input_link.strip('/').split('/')
-        input_mc_code = next((p for p in reversed(url_parts) if p), None)
-        if not input_mc_code:
-            raise ValueError("Could not extract MC Company Code from the URL.")
-        mc_links = self.load_mc_link_data()
-        if symbol in mc_links:
-            mc_links[symbol]["MC_LINK"] = input_link
-            mc_links[symbol]["MC_COMPANY_CODE"] = input_mc_code
-            mc_links[symbol]["MC_COMPANY_NAME"] = security
-        else:
-            mc_links[symbol] = {
-                "MC_ATH": 0.0, "MC_ATL": 0.0,
-                "MC_COMPANY_CODE": input_mc_code,
-                "MC_COMPANY_NAME": security,
-                "MC_FV": 0.0, "MC_LINK": input_link,
-                "MC_MCAP": 0.0, "MC_SCORE": 0.0, "MC_STATUS": False,
-            }
-        self.save_mc_link_data(mc_links)
-        return mc_links[symbol]

nsedash/core/token_state.json

@@ -1,37 +0,0 @@
-{
-    "prompt_tokens": 186398,
-    "completion_tokens": 72204,
-    "total_tokens": 258602,
-    "per_model": {
-        "nvidia/llama-3.3-nemotron-super-49b-v1": {
-            "prompt_tokens": 5935,
-            "completion_tokens": 1999,
-            "total_tokens": 7934
-        },
-        "nvidia/nemotron-3-ultra-550b-a55b": {
-            "prompt_tokens": 8822,
-            "completion_tokens": 8192,
-            "total_tokens": 17014
-        },
-        "nvidia/llama-3.3-nemotron-super-49b-v1.5": {
-            "prompt_tokens": 15581,
-            "completion_tokens": 11471,
-            "total_tokens": 27052
-        },
-        "meta/llama-3.1-70b-instruct": {
-            "prompt_tokens": 17996,
-            "completion_tokens": 5482,
-            "total_tokens": 23478
-        },
-        "thinkingmachines/inkling": {
-            "prompt_tokens": 3426,
-            "completion_tokens": 4096,
-            "total_tokens": 7522
-        },
-        "google/diffusiongemma-26b-a4b-it": {
-            "prompt_tokens": 134638,
-            "completion_tokens": 40964,
-            "total_tokens": 175602
-        }
-    }
-}

nsedash/core/user_auth.py

@@ -1,835 +0,0 @@
-"""
-nsedash.core.user_auth
-=======================
-User authentication and session management for nsedash — ported from
-the mlbench ``security/app_security.py`` module and adapted for the
-nsedash Flask API.
-
-Authentication flow
---------------------
-1. ``POST /api/auth/login``  — userid + password → bcrypt verify → TOTP required
-2. ``POST /api/auth/totp``   — userid + TOTP code → issues a session token
-3. All subsequent API calls send ``X-Session-Token`` header.
-4. ``SessionManager.validate_session()`` resolves the token → userid.
-5. ``_svc(userid)`` creates / retrieves the per-user ``DashboardService``.
-
-Seed users (from mlbench users.db)
-------------------------------------
-The file ``security/users.db`` is created on first run using the same
-schema as mlbench.  The initial user set is:
-  admin, sam, mainak, tunir, rama
-
-Schema of users.db
---------------------
-{
-  "users": {
-    "<userid>": {
-      "userid": str,
-      "password_hash": str | null,
-      "email": str | null,
-      "totp_secret": str | null,
-      "totp_secret_encrypted": str | null,
-      "is_admin": bool,
-      "created_at": ISO8601,
-      "last_login": ISO8601 | null,
-      "login_count": int,
-      "active": bool,
-      "login_status": "online" | "offline"
-    }
-  }
-}
-"""
-
-from __future__ import annotations
-
-import base64
-import json
-import logging
-import os
-import secrets
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
-
-log = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# File paths (relative to the app base_dir — set via UserAuth.init())
-# ---------------------------------------------------------------------------
-
-_SECURITY_DIR: Path | None = None
-_USERS_DB_FILE: Path | None = None
-_ENCRYPTION_KEY_FILE: Path | None = None
-_SESSION_TOKENS_FILE: Path | None = None
-_HF_SYNC = None
-
-import re
-
-SESSION_EXPIRATION_HOURS = 24
-TOTP_WINDOW = 1  # ±1 30-second window
-
-
-def validate_password_policy(password: str) -> Tuple[bool, str]:
-    """
-    Validates password against policy:
-      - Minimum length: 8 characters
-      - At least 1 uppercase letter (A-Z)
-      - At least 1 lowercase letter (a-z)
-      - At least 1 digit (0-9)
-      - At least 1 special character (!@#$%^&*()_+-=[]{}|;:,.<>?)
-    """
-    if not password or len(password) < 8:
-        return False, "Password must be at least 8 characters long."
-    if not re.search(r"[A-Z]", password):
-        return False, "Password must contain at least 1 uppercase letter (A-Z)."
-    if not re.search(r"[a-z]", password):
-        return False, "Password must contain at least 1 lowercase letter (a-z)."
-    if not re.search(r"[0-9]", password):
-        return False, "Password must contain at least 1 number (0-9)."
-    if not re.search(r"[^A-Za-z0-9]", password):
-        return False, "Password must contain at least 1 special character (e.g. !@#$%^&*)."
-    return True, "OK"
-
-
-def init(base_dir: Path | str, hf_sync=None) -> None:
-    """
-    Initialise paths. Must be called once from ``create_app()`` before
-    any auth function is used.
-    """
-    global _SECURITY_DIR, _USERS_DB_FILE, _ENCRYPTION_KEY_FILE, _SESSION_TOKENS_FILE, _HF_SYNC
-    base_dir = Path(base_dir)
-    _SECURITY_DIR = base_dir / "processed" / "security"
-    _SECURITY_DIR.mkdir(parents=True, exist_ok=True)
-    _USERS_DB_FILE = _SECURITY_DIR / "users.db"
-    _ENCRYPTION_KEY_FILE = _SECURITY_DIR / "encryption_key.key"
-    if not _ENCRYPTION_KEY_FILE.exists() and (_SECURITY_DIR / ".encryption_key").exists():
-        _ENCRYPTION_KEY_FILE = _SECURITY_DIR / ".encryption_key"
-    _SESSION_TOKENS_FILE = _SECURITY_DIR / ".session_tokens.json"
-    _HF_SYNC = hf_sync
-
-    # Initialise encrypted SQLite database (userdata.db)
-    from .userdata_db import init as init_userdata_db
-    init_userdata_db(base_dir, hf_sync=hf_sync)
-
-    log.info("[Auth] Initialised security dir: %s (hf_sync linked: %s)", _SECURITY_DIR, hf_sync is not None)
-
-
-def _check_init():
-    if _USERS_DB_FILE is None:
-        raise RuntimeError("user_auth.init(base_dir) must be called first")
-
-
-# ---------------------------------------------------------------------------
-# Encryption (Fernet — same key format as mlbench)
-# ---------------------------------------------------------------------------
-
-class EncryptionManager:
-    """Handles encryption/decryption of TOTP secrets."""
-
-    @staticmethod
-    def _get_or_create_key() -> bytes:
-        _check_init()
-        from cryptography.fernet import Fernet
-        if _ENCRYPTION_KEY_FILE.exists():
-            return _ENCRYPTION_KEY_FILE.read_bytes()
-        key = Fernet.generate_key()
-        _ENCRYPTION_KEY_FILE.write_bytes(key)
-        log.info("[Auth] Encryption key created.")
-        if _HF_SYNC:
-            _HF_SYNC.push_file_async(_ENCRYPTION_KEY_FILE)
-        return key
-
-    @staticmethod
-    def encrypt(plaintext: str) -> str:
-        try:
-            from cryptography.fernet import Fernet
-            key = EncryptionManager._get_or_create_key()
-            ciphertext = Fernet(key).encrypt(plaintext.encode())
-            return base64.b64encode(ciphertext).decode()
-        except Exception as exc:
-            log.error("[Auth] Encryption error: %s", exc)
-            return plaintext
-
-    @staticmethod
-    def decrypt(ciphertext: str) -> str | None:
-        try:
-            from cryptography.fernet import Fernet
-            key = EncryptionManager._get_or_create_key()
-            raw = base64.b64decode(ciphertext.encode())
-            return Fernet(key).decrypt(raw).decode()
-        except Exception as exc:
-            log.error("[Auth] Decryption error: %s", exc)
-            return None
-
-
-# ---------------------------------------------------------------------------
-# User Database
-# ---------------------------------------------------------------------------
-
-# Seed users ported from mlbench users.db.
-# Passwords are null (set on first login via TOTP setup flow).
-# Only the admin entry has the real bcrypt hash from mlbench.
-_SEED_USERS: Dict[str, Any] = {
-    "admin": {
-        "userid": "admin",
-        "password_hash": "pbkdf2:sha256:1000000$3QTFDU2NQmIiNJCs$8e2fa3fd5daae1c5270426e2c2080c600a11fbeabee071f115ec5f584098571b",
-        "email": None,
-        "totp_secret": None,
-        "totp_secret_encrypted": None,
-        "is_admin": True,
-        "created_at": "2026-02-28T16:05:18.324809",
-        "last_login": None,
-        "login_count": 0,
-        "active": True,
-        "login_status": "offline",
-    },
-    "sam": {
-        "userid": "sam",
-        "password_hash": "pbkdf2:sha256:1000000$zAgOwq9jWdoLseRy$bec16031d940efcdb7b83b5d34735b3a11bcf4ce613ff7c5e0252c01d98cbdd5",
-        "email": None,
-        "totp_secret": None,
-        "totp_secret_encrypted": None,
-        "is_admin": False,
-        "created_at": "2026-03-26T00:31:49.856999",
-        "last_login": None,
-        "login_count": 0,
-        "active": True,
-        "login_status": "offline",
-    },
-    "mainak": {
-        "userid": "mainak",
-        "password_hash": None,
-        "email": None,
-        "totp_secret": None,
-        "totp_secret_encrypted": None,
-        "is_admin": False,
-        "created_at": "2026-02-28T20:58:29.175636",
-        "last_login": None,
-        "login_count": 0,
-        "active": True,
-        "login_status": "offline",
-    },
-    "tunir": {
-        "userid": "tunir",
-        "password_hash": None,
-        "email": None,
-        "totp_secret": None,
-        "totp_secret_encrypted": None,
-        "is_admin": False,
-        "created_at": "2026-02-28T20:58:42.323454",
-        "last_login": None,
-        "login_count": 0,
-        "active": True,
-        "login_status": "offline",
-    },
-    "rama": {
-        "userid": "rama",
-        "password_hash": None,
-        "email": None,
-        "totp_secret": None,
-        "totp_secret_encrypted": None,
-        "is_admin": False,
-        "created_at": "2026-02-28T23:18:58.881545",
-        "last_login": None,
-        "login_count": 0,
-        "active": True,
-        "login_status": "offline",
-    },
-}
-
-
-class UserDatabase:
-    """JSON-backed user database — same schema as mlbench users.db."""
-
-    @staticmethod
-    def load() -> Dict[str, Any]:
-        _check_init()
-        if not _USERS_DB_FILE.exists():
-            return UserDatabase._create_defaults()
-        try:
-            return json.loads(_USERS_DB_FILE.read_text())
-        except Exception as exc:
-            log.error("[Auth] Failed to load users.db: %s", exc)
-            return UserDatabase._create_defaults()
-
-    @staticmethod
-    def _create_defaults() -> Dict[str, Any]:
-        data = {"users": _SEED_USERS.copy()}
-        UserDatabase.save(data)
-        log.warning("[Auth] Created default users.db from seed (admin, sam, mainak, tunir, rama)")
-        return data
-
-    @staticmethod
-    def save(data: Dict[str, Any]) -> bool:
-        _check_init()
-        try:
-            _USERS_DB_FILE.write_text(json.dumps(data, indent=2))
-            if _HF_SYNC:
-                _HF_SYNC.push_file_async(_USERS_DB_FILE)
-            return True
-        except Exception as exc:
-            log.error("[Auth] Failed to save users.db: %s", exc)
-            return False
-
-    @staticmethod
-    def get_user(userid: str) -> Optional[Dict[str, Any]]:
-        u = UserDatabase.load().get("users", {}).get(userid)
-        if u and userid == "admin":
-            u["is_admin"] = True
-        return u
-
-    @staticmethod
-    def update_user(userid: str, updates: Dict[str, Any]) -> bool:
-        data = UserDatabase.load()
-        if userid not in data.get("users", {}):
-            return False
-        data["users"][userid].update(updates)
-        if userid == "admin":
-            data["users"][userid]["is_admin"] = True
-        return UserDatabase.save(data)
-
-    @staticmethod
-    def create_user(userid: str, is_admin: bool = False) -> bool:
-        if UserDatabase.get_user(userid):
-            return False
-        data = UserDatabase.load()
-        data["users"][userid] = {
-            "userid": userid,
-            "password_hash": None,
-            "email": None,
-            "totp_secret": None,
-            "totp_secret_encrypted": None,
-            "is_admin": True if userid == "admin" else is_admin,
-            "created_at": datetime.now().isoformat(),
-            "last_login": None,
-            "login_count": 0,
-            "active": True,
-            "login_status": "offline",
-        }
-        return UserDatabase.save(data)
-
-    @staticmethod
-    def delete_user(userid: str) -> bool:
-        if userid == "admin":
-            return False
-        data = UserDatabase.load()
-        if userid in data.get("users", {}):
-            del data["users"][userid]
-            return UserDatabase.save(data)
-        return False
-
-    @staticmethod
-    def reset_totp(userid: str) -> bool:
-        data = UserDatabase.load()
-        if userid in data.get("users", {}):
-            data["users"][userid]["password_hash"] = None
-            data["users"][userid]["totp_secret"] = None
-            data["users"][userid]["totp_secret_encrypted"] = None
-            if userid == "admin":
-                data["users"][userid]["is_admin"] = True
-            return UserDatabase.save(data)
-        return False
-
-    @staticmethod
-    def get_all_users() -> Dict[str, Any]:
-        users = UserDatabase.load().get("users", {})
-        return {
-            uid: {
-                "userid": uid,
-                "is_admin": True if uid == "admin" else u.get("is_admin", False),
-                "active": u.get("active"),
-                "last_login": u.get("last_login"),
-                "login_count": u.get("login_count", 0),
-                "login_status": u.get("login_status", "offline"),
-            }
-            for uid, u in users.items()
-        }
-
-
-# ---------------------------------------------------------------------------
-# Authentication
-# ---------------------------------------------------------------------------
-
-class AuthenticationService:
-    """Userid + bcrypt password + TOTP — same flow as mlbench."""
-
-    @staticmethod
-    def login(userid: str, password: str) -> Tuple[bool, str, bool]:
-        """
-        Verify password.
-
-        Returns
-        -------
-        (success, message, totp_required)
-        """
-        user = UserDatabase.get_user(userid)
-        if not user:
-            ActivityLogger.log_activity(userid, "login", "failed", "User not found")
-            return False, "User not found.", False
-        if not user.get("active", True):
-            ActivityLogger.log_activity(userid, "login", "failed", "Account inactive")
-            return False, "Account inactive.", False
-
-        from werkzeug.security import generate_password_hash, check_password_hash
-
-        if user.get("password_hash") is None:
-            # First login — set password and generate TOTP secret
-            pw_hash = generate_password_hash(password, method="pbkdf2:sha256")
-            import pyotp
-            totp_secret = pyotp.random_base32()
-            enc = EncryptionManager.encrypt(totp_secret)
-            UserDatabase.update_user(userid, {
-                "password_hash": pw_hash,
-                "totp_secret": totp_secret,
-                "totp_secret_encrypted": enc,
-            })
-            log.info("[Auth] First login for '%s' — TOTP provisioned.", userid)
-            ActivityLogger.log_activity(userid, "login", "first_totp_setup", "First TOTP setup triggered")
-            return True, "first_login", True   # caller should show QR
-
-        if not check_password_hash(user["password_hash"], password):
-            ActivityLogger.log_activity(userid, "login", "failed", "Invalid password")
-            return False, "Invalid password.", False
-
-        # Validate password policy compliance
-        valid_pw, _ = validate_password_policy(password)
-        if not valid_pw or user.get("force_password_change"):
-            UserDatabase.update_user(userid, {"force_password_change": True})
-
-        # If TOTP not yet set up (e.g. admin copied over without secret)
-        if not user.get("totp_secret"):
-            try:
-                import pyotp
-            except ImportError:
-                ActivityLogger.log_activity(userid, "login", "error", "pyotp missing")
-                return False, "Server error: pyotp library is missing", False
-            
-            totp_secret = pyotp.random_base32()
-            enc = EncryptionManager.encrypt(totp_secret)
-            if not UserDatabase.update_user(userid, {
-                "totp_secret": totp_secret,
-                "totp_secret_encrypted": enc,
-            }):
-                ActivityLogger.log_activity(userid, "login", "error", "failed_to_save_totp")
-                return False, "Server error: Failed to save TOTP data", False
-                
-            ActivityLogger.log_activity(userid, "login", "totp_setup", "TOTP setup triggered")
-            return True, "totp_setup", True
-
-        ActivityLogger.log_activity(userid, "login", "password_verified", "Password verified successfully")
-        return True, "ok", True
-
-
-def change_user_password(userid: str, current_password: str | None, new_password: str, confirm_password: str, is_admin_override: bool = False) -> Tuple[bool, str]:
-    """
-    Change user password enforcing:
-      1. Reconfirmation entry (new_password == confirm_password)
-      2. Password policy (min 8 chars, 1 upper, 1 lower, 1 digit, 1 special char)
-      3. Current password verification (unless admin override)
-    """
-    if new_password != confirm_password:
-        return False, "New password and confirm password do not match."
-
-    valid, msg = validate_password_policy(new_password)
-    if not valid:
-        return False, msg
-
-    user = UserDatabase.get_user(userid)
-    if not user:
-        return False, "User not found."
-
-    from werkzeug.security import generate_password_hash, check_password_hash
-
-    if not is_admin_override:
-        if not current_password or not user.get("password_hash"):
-            return False, "Current password is required."
-        if not check_password_hash(user["password_hash"], current_password):
-            return False, "Current password is incorrect."
-
-    new_hash = generate_password_hash(new_password, method="pbkdf2:sha256")
-    UserDatabase.update_user(userid, {
-        "password_hash": new_hash,
-        "force_password_change": False
-    })
-    ActivityLogger.log_activity(userid, "change_password", "success", "Password updated successfully")
-    return True, "Password changed successfully."
-
-    @staticmethod
-    def get_totp_qr(userid: str) -> str | None:
-        """Return base64 PNG data-URI for TOTP QR code."""
-        user = UserDatabase.get_user(userid)
-        if not user:
-            return None
-        secret = user.get("totp_secret")
-        if not secret and user.get("totp_secret_encrypted"):
-            secret = EncryptionManager.decrypt(user["totp_secret_encrypted"])
-        if not secret:
-            return None
-        try:
-            import pyotp
-            import qrcode
-        except ImportError as exc:
-            log.error("[Auth] Missing dependencies for QR generation: %s", exc)
-            return None
-
-        try:
-            from io import BytesIO
-            uri = pyotp.TOTP(secret).provisioning_uri(
-                name=userid, issuer_name="NSE Dashboard"
-            )
-            qr = qrcode.make(uri)
-            buf = BytesIO()
-            qr.save(buf, format="PNG")
-            return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
-        except Exception as exc:
-            log.error("[Auth] QR generation failed for '%s': %s", userid, exc)
-            return None
-
-    @staticmethod
-    def verify_totp(userid: str, code: str) -> Tuple[bool, str]:
-        """Verify TOTP code and — on success — issue a session token."""
-        user = UserDatabase.get_user(userid)
-        if not user:
-            ActivityLogger.log_activity(userid, "totp_verification", "failed", "User not found")
-            return False, "User not found."
-        secret = user.get("totp_secret")
-        if not secret and user.get("totp_secret_encrypted"):
-            secret = EncryptionManager.decrypt(user["totp_secret_encrypted"])
-        if not secret:
-            ActivityLogger.log_activity(userid, "totp_verification", "failed", "TOTP not configured")
-            return False, "TOTP not configured."
-        try:
-            import pyotp
-            totp = pyotp.TOTP(secret)
-            if not totp.verify(code, valid_window=TOTP_WINDOW):
-                ActivityLogger.log_activity(userid, "totp_verification", "failed", "Invalid TOTP code")
-                return False, "Invalid TOTP code."
-        except Exception as exc:
-            ActivityLogger.log_activity(userid, "totp_verification", "error", f"TOTP error: {exc}")
-            return False, f"TOTP error: {exc}"
-
-        # Success — update login metadata
-        cnt = user.get("login_count", 0)
-        UserDatabase.update_user(userid, {
-            "last_login": datetime.now().isoformat(),
-            "login_count": cnt + 1,
-            "login_status": "online",
-        })
-        token = SessionManager.create_session(userid)
-        ActivityLogger.log_activity(userid, "login", "success", "TOTP verified, logged in")
-        return True, token
-
-
-
-# ---------------------------------------------------------------------------
-# Session Manager
-# ---------------------------------------------------------------------------
-
-class SessionManager:
-    """File-backed session tokens — same approach as mlbench SessionManager."""
-
-    @staticmethod
-    def _load() -> Dict[str, Any]:
-        _check_init()
-        if not _SESSION_TOKENS_FILE or not _SESSION_TOKENS_FILE.exists():
-            return {}
-        try:
-            return json.loads(_SESSION_TOKENS_FILE.read_text())
-        except Exception:
-            return {}
-
-    @staticmethod
-    def _save(sessions: Dict[str, Any]) -> None:
-        _check_init()
-        try:
-            _SESSION_TOKENS_FILE.write_text(json.dumps(sessions, indent=2))
-            if _HF_SYNC:
-                _HF_SYNC.push_file_async(_SESSION_TOKENS_FILE)
-        except Exception as exc:
-            log.error("[Auth] Failed to save sessions: %s", exc)
-
-    @staticmethod
-    def create_session(userid: str) -> str:
-        token = secrets.token_urlsafe(32)
-        sessions = SessionManager._load()
-        sessions[token] = {
-            "userid": userid,
-            "created_at": datetime.now().isoformat(),
-            "last_activity": datetime.now().isoformat(),
-        }
-        SessionManager._save(sessions)
-        log.info("[Auth] Session created for '%s'", userid)
-        return token
-
-    @staticmethod
-    def validate_session(token: str) -> Tuple[bool, Optional[str]]:
-        """Returns (valid, userid)."""
-        if not token:
-            return False, None
-        sessions = SessionManager._load()
-        entry = sessions.get(token)
-        if not entry:
-            return False, None
-        created = datetime.fromisoformat(entry["created_at"])
-        if (datetime.now() - created).total_seconds() > SESSION_EXPIRATION_HOURS * 3600:
-            del sessions[token]
-            SessionManager._save(sessions)
-            return False, None
-        # Touch last_activity
-        entry["last_activity"] = datetime.now().isoformat()
-        SessionManager._save(sessions)
-        return True, entry["userid"]
-
-    @staticmethod
-    def destroy_session(token: str) -> bool:
-        sessions = SessionManager._load()
-        if token in sessions:
-            userid = sessions[token].get("userid", "?")
-            del sessions[token]
-            SessionManager._save(sessions)
-            # Mark user offline
-            UserDatabase.update_user(userid, {"login_status": "offline"})
-            log.info("[Auth] Session destroyed for '%s'", userid)
-            ActivityLogger.log_activity(userid, "logout", "success", "Logged out successfully")
-            return True
-        return False
-
-    @staticmethod
-    def get_userid_sessions(userid: str) -> list[str]:
-        """Return all active token strings for a given userid."""
-        sessions = SessionManager._load()
-        return [tok for tok, d in sessions.items() if d.get("userid") == userid]
-        
-    @staticmethod
-    def clear_sessions_for_user(userid: str) -> int:
-        """Deletes all active sessions for a specific user, forcing them to log in again."""
-        sessions = SessionManager._load()
-        tokens_to_delete = [tok for tok, d in sessions.items() if d.get("userid") == userid]
-        for tok in tokens_to_delete:
-            del sessions[tok]
-        if tokens_to_delete:
-            SessionManager._save(sessions)
-            log.info("[Auth] Cleared %d sessions for '%s'", len(tokens_to_delete), userid)
-        
-        # Always force offline status to resolve "ghost" logins
-        UserDatabase.update_user(userid, {"login_status": "offline"})
-        return len(tokens_to_delete)
-
-    @staticmethod
-    def cleanup_expired() -> int:
-        sessions = SessionManager._load()
-        expired = [
-            tok for tok, d in sessions.items()
-            if (datetime.now() - datetime.fromisoformat(d["created_at"])).total_seconds()
-               > SESSION_EXPIRATION_HOURS * 3600
-        ]
-        for tok in expired:
-            del sessions[tok]
-        if expired:
-            SessionManager._save(sessions)
-        return len(expired)
-
-    @staticmethod
-    def invalidate_all_sessions_on_startup() -> int:
-        """
-        Invalidates all active session tokens and updates all users as offline
-        on application startup to enforce fresh per-session logins.
-        """
-        sessions = SessionManager._load()
-        count = len(sessions)
-        SessionManager._save({})
-        
-        # Update all users to offline status and ensure admin preserves is_admin
-        data = UserDatabase.load()
-        users = data.get("users", {})
-        updated = False
-        for userid, user_info in users.items():
-            if user_info.get("login_status") != "offline":
-                user_info["login_status"] = "offline"
-                updated = True
-            if userid == "admin" and not user_info.get("is_admin"):
-                user_info["is_admin"] = True
-                updated = True
-        if updated:
-            UserDatabase.save(data)
-            
-        log.info("[Auth Startup] Invalidated %d active sessions. Updated users to offline status.", count)
-        return count
-
-
-def invalidate_all_sessions_on_startup() -> int:
-    return SessionManager.invalidate_all_sessions_on_startup()
-
-
-
-# ---------------------------------------------------------------------------
-# Activity Logger
-# ---------------------------------------------------------------------------
-
-class ActivityLogger:
-    """Logs user activities for audit trail (parity with mlbench)."""
-
-    @staticmethod
-    def log_activity(userid: str, action: str, status: str, details: str = "", duration_seconds: Optional[int] = None):
-        _check_init()
-        try:
-            log_file = _SECURITY_DIR / "activity_log.jsonl"
-            log_entry = {
-                'timestamp': datetime.now().isoformat(),
-                'userid': userid,
-                'action': action,
-                'status': status,
-                'details': details
-            }
-            if duration_seconds is not None:
-                log_entry['duration_seconds'] = duration_seconds
-            
-            with open(log_file, 'a') as f:
-                f.write(json.dumps(log_entry) + '\n')
-            log.info("[Activity] Logged: %s - %s - %s", userid, action, status)
-            if _HF_SYNC:
-                _HF_SYNC.push_file_async(log_file)
-        except Exception as e:
-            log.error("[Activity] Error logging activity: %s", e)
-
-    @staticmethod
-    def get_activity_logs(userid: Optional[str] = None, limit: int = 1000) -> list:
-        _check_init()
-        try:
-            log_file = _SECURITY_DIR / "activity_log.jsonl"
-            logs = []
-            if log_file.exists():
-                with open(log_file, 'r') as f:
-                    for line in f:
-                        if line.strip():
-                            log_entry = json.loads(line)
-                            if userid is None or log_entry.get('userid') == userid:
-                                logs.append(log_entry)
-            return logs[-limit:]
-        except Exception as e:
-            log.error("[Activity] Error reading logs: %s", e)
-            return []
-
-
-# ---------------------------------------------------------------------------
-# Encrypted User Data Sync in users.db
-# ---------------------------------------------------------------------------
-
-def backup_session_to_db(userid: str, base_dir: Path | str) -> bool:
-    """Read the session files (config, state, watchlists, portfolio) and save encrypted to users.db."""
-    _check_init()
-    if userid == "admin":
-        return True # Admin uses global shared files, not isolated/encrypted
-
-    from ..config import Paths
-    paths = Paths(base_dir=base_dir, session_token=userid)
-    
-    # Read files
-    try:
-        state_data = None
-        if paths.state_file.exists():
-            try:
-                state_data = json.loads(paths.state_file.read_text())
-            except Exception:
-                pass
-
-        config_data = None
-        if paths.config_file.exists():
-            try:
-                config_data = json.loads(paths.config_file.read_text())
-            except Exception:
-                pass
-
-        portfolio_data = None
-        if paths.portfolio_file.exists():
-            try:
-                # Read binary Parquet file and encode to base64 string
-                binary_data = paths.portfolio_file.read_bytes()
-                portfolio_data = base64.b64encode(binary_data).decode('utf-8')
-            except Exception as exc:
-                log.error("[Auth] Failed to backup portfolio Parquet: %s", exc)
-
-        watchlists_data = {}
-        if paths.watchlist_dir.exists():
-            for item in paths.watchlist_dir.iterdir():
-                if item.is_file() and item.suffix == ".json":
-                    try:
-                        watchlists_data[item.name] = json.loads(item.read_text())
-                    except Exception:
-                        pass
-
-        user_data = {
-            "state": state_data,
-            "config": config_data,
-            "portfolio": portfolio_data,
-            "watchlists": watchlists_data,
-        }
-
-        # Encrypt & save
-        plain = json.dumps(user_data)
-        enc = EncryptionManager.encrypt(plain)
-        UserDatabase.update_user(userid, {"user_data_encrypted": enc})
-        return True
-    except Exception as exc:
-        log.error("[Auth] Failed to backup session files for user %s: %s", userid, exc)
-        return False
-
-
-def restore_session_from_db(userid: str, base_dir: Path | str) -> bool:
-    """Load and decrypt the user's data from users.db and write to the isolated session files."""
-    _check_init()
-    if userid == "admin":
-        return False # Admin uses global shared files, no restore needed
-
-    user = UserDatabase.get_user(userid)
-    if not user:
-        return False
-    
-    enc = user.get("user_data_encrypted")
-    if not enc:
-        return False # No data saved yet
-    
-    try:
-        plain = EncryptionManager.decrypt(enc)
-        if not plain:
-            return False
-        
-        user_data = json.loads(plain)
-        from ..config import Paths
-        paths = Paths(base_dir=base_dir, session_token=userid)
-        paths.ensure_dirs()
-
-        # Restore state
-        if user_data.get("state"):
-            paths.state_file.write_text(json.dumps(user_data["state"], indent=2))
-
-        # Restore config
-        if user_data.get("config"):
-            paths.config_file.write_text(json.dumps(user_data["config"], indent=2))
-
-        # Restore portfolio
-        if user_data.get("portfolio"):
-            try:
-                # Decode base64 to binary bytes and save
-                port_raw = user_data["portfolio"]
-                try:
-                    binary_data = base64.b64decode(port_raw.encode('utf-8'))
-                    paths.portfolio_file.write_bytes(binary_data)
-                except Exception:
-                    # Fallback to legacy plain text write if it was not base64
-                    paths.portfolio_file.write_text(port_raw)
-            except Exception as exc:
-                log.error("[Auth] Failed to restore portfolio Parquet: %s", exc)
-
-        # Restore watchlists
-        if user_data.get("watchlists"):
-            for filename, wl_content in user_data["watchlists"].items():
-                file_path = paths.watchlist_dir / filename
-                file_path.write_text(json.dumps(wl_content, indent=2))
-
-        log.info("[Auth] Restored session files from users.db for user %s", userid)
-        return True
-    except Exception as exc:
-        log.error("[Auth] Failed to restore session files for user %s: %s", userid, exc)
-        return False
-

nsedash/core/userdata_db.py

@@ -1,206 +0,0 @@
-"""
-nsedash.core.userdata_db
-========================
-Encrypted SQLite database module (userdata.db) for per-user private Watchlists 1–5.
-Uses Fernet AES-256 encryption via EncryptionManager to encrypt watchlist contents.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import sqlite3
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-log = logging.getLogger(__name__)
-
-_USERDATA_DB_FILE: Path | None = None
-_SECURITY_DIR: Path | None = None
-_HF_SYNC = None
-
-
-def init(base_dir: Path | str, hf_sync=None) -> None:
-    """Initialise SQLite database paths for userdata.db."""
-    global _USERDATA_DB_FILE, _SECURITY_DIR, _HF_SYNC
-    base_dir = Path(base_dir)
-    _SECURITY_DIR = base_dir / "processed" / "security"
-    _SECURITY_DIR.mkdir(parents=True, exist_ok=True)
-    _USERDATA_DB_FILE = _SECURITY_DIR / "userdata.db"
-    _HF_SYNC = hf_sync
-    
-    UserDataDB.ensure_tables()
-    UserDataDB.perform_initial_reset_and_migration(base_dir)
-    log.info("[UserDataDB] Initialised encrypted SQLite database: %s", _USERDATA_DB_FILE)
-
-
-class UserDataDB:
-    """Encrypted SQLite database manager for per-user private watchlists (slots 1..5)."""
-
-    @staticmethod
-    def _get_connection() -> sqlite3.Connection:
-        if _USERDATA_DB_FILE is None:
-            raise RuntimeError("userdata_db.init(base_dir) must be called first")
-        conn = sqlite3.connect(_USERDATA_DB_FILE)
-        conn.row_factory = sqlite3.Row
-        return conn
-
-    @staticmethod
-    def ensure_tables() -> None:
-        """Create user_watchlists table if it does not exist."""
-        conn = UserDataDB._get_connection()
-        try:
-            with conn:
-                conn.execute("""
-                    CREATE TABLE IF NOT EXISTS user_watchlists (
-                        userid TEXT NOT NULL,
-                        slot INTEGER NOT NULL,
-                        name TEXT NOT NULL,
-                        encrypted_data TEXT NOT NULL,
-                        updated_at TEXT NOT NULL,
-                        PRIMARY KEY (userid, slot)
-                    );
-                """)
-            log.info("[UserDataDB] Schema verified for user_watchlists table in userdata.db.")
-        finally:
-            conn.close()
-
-    @staticmethod
-    def save_watchlist(userid: str, slot: int, name: str, symbols: List[str]) -> bool:
-        """Encrypt and save a user's private watchlist (slots 1..5)."""
-        if slot < 1 or slot > 5:
-            log.warning("[UserDataDB] Slot %d is outside private range 1..5 for user %s", slot, userid)
-            return False
-
-        from .user_auth import EncryptionManager
-        payload = {
-            "name": name or f"Watchlist {slot}",
-            "symbols": list(symbols or [])
-        }
-        plain_json = json.dumps(payload)
-        encrypted_data = EncryptionManager.encrypt(plain_json)
-        now_iso = datetime.now().isoformat()
-
-        conn = UserDataDB._get_connection()
-        try:
-            with conn:
-                conn.execute("""
-                    INSERT INTO user_watchlists (userid, slot, name, encrypted_data, updated_at)
-                    VALUES (?, ?, ?, ?, ?)
-                    ON CONFLICT(userid, slot) DO UPDATE SET
-                        name = excluded.name,
-                        encrypted_data = excluded.encrypted_data,
-                        updated_at = excluded.updated_at;
-                """, (userid, slot, name or f"Watchlist {slot}", encrypted_data, now_iso))
-            log.info("[UserDataDB] Saved encrypted watchlist slot %d for user %s (%d symbols)", slot, userid, len(symbols))
-            if _HF_SYNC and _USERDATA_DB_FILE:
-                _HF_SYNC.push_file_async(_USERDATA_DB_FILE)
-            return True
-        except Exception as exc:
-            log.error("[UserDataDB] Error saving watchlist for user %s slot %d: %s", userid, slot, exc, exc_info=True)
-            return False
-        finally:
-            conn.close()
-
-    @staticmethod
-    def get_watchlist(userid: str, slot: int) -> Optional[Dict[str, Any]]:
-        """Fetch and decrypt a user's private watchlist (slots 1..5)."""
-        if slot < 1 or slot > 5:
-            return None
-
-        from .user_auth import EncryptionManager
-        conn = UserDataDB._get_connection()
-        try:
-            cur = conn.execute(
-                "SELECT name, encrypted_data FROM user_watchlists WHERE userid = ? AND slot = ?",
-                (userid, slot)
-            )
-            row = cur.fetchone()
-            if not row:
-                return None
-            
-            name = row["name"]
-            enc_data = row["encrypted_data"]
-            plain_json = EncryptionManager.decrypt(enc_data)
-            if not plain_json:
-                log.warning("[UserDataDB] Decryption returned empty string for user %s slot %d", userid, slot)
-                return {"name": name, "symbols": []}
-
-            data = json.loads(plain_json)
-            return {
-                "name": data.get("name", name),
-                "symbols": data.get("symbols", [])
-            }
-        except Exception as exc:
-            log.error("[UserDataDB] Error fetching watchlist for user %s slot %d: %s", userid, slot, exc, exc_info=True)
-            return None
-        finally:
-            conn.close()
-
-    @staticmethod
-    def get_all_user_watchlists(userid: str) -> Dict[int, Dict[str, Any]]:
-        """Return all watchlists 1..5 for userid."""
-        result = {}
-        for slot in range(1, 6):
-            wl = UserDataDB.get_watchlist(userid, slot)
-            if wl:
-                result[slot] = wl
-            else:
-                result[slot] = {"name": f"Watchlist {slot}", "symbols": []}
-        return result
-
-    @staticmethod
-    def reset_watchlists_except(allowed_users: List[str] = None) -> int:
-        """
-        Resets Watchlists 1–5 to blank (empty symbols list) for all users
-        EXCEPT those listed in allowed_users (default: ['admin', 'sam']).
-        """
-        allowed = set(allowed_users or ["admin", "sam"])
-        from .user_auth import UserDatabase
-        all_users = UserDatabase.get_all_users()
-        
-        reset_count = 0
-        for uid in all_users.keys():
-            if uid not in allowed:
-                for slot in range(1, 6):
-                    UserDataDB.save_watchlist(uid, slot, f"Watchlist {slot}", [])
-                reset_count += 1
-        return reset_count
-
-    @staticmethod
-    def perform_initial_reset_and_migration(base_dir: Path | str) -> None:
-        """
-        Migrates existing Watchlists 1..5 for admin and sam into userdata.db (if not present),
-        and resets Watchlists 1..5 to blank for all other users.
-        """
-        from .user_auth import UserDatabase
-        all_users = UserDatabase.get_all_users()
-        base_dir = Path(base_dir)
-        wl_dir = base_dir / "processed" / "watchlists"
-
-        for uid in all_users.keys():
-            if uid in ("admin", "sam"):
-                # Migrate existing JSON files for admin / sam if slot not yet populated in userdata.db
-                user_wl_dir = wl_dir / f"session_{uid}" if uid != "admin" else wl_dir
-                for slot in range(1, 6):
-                    existing_wl = UserDataDB.get_watchlist(uid, slot)
-                    if existing_wl is None:
-                        json_file = user_wl_dir / f"nse_dashboard_watchlist{slot}.json"
-                        name = f"Watchlist {slot}"
-                        symbols = []
-                        if json_file.exists():
-                            try:
-                                data = json.loads(json_file.read_text())
-                                name = data.get("name", name)
-                                symbols_raw = data.get("symbols", [])
-                                symbols = [s.get("SYMBOL") if isinstance(s, dict) else str(s) for s in symbols_raw]
-                            except Exception:
-                                pass
-                        UserDataDB.save_watchlist(uid, slot, name, symbols)
-                        log.info("[UserDataDB] Migrated slot %d for '%s' to userdata.db (%d symbols)", slot, uid, len(symbols))
-            else:
-                # Reset slots 1..5 to blank for non-exempt users
-                for slot in range(1, 6):
-                    UserDataDB.save_watchlist(uid, slot, f"Watchlist {slot}", [])
-                log.info("[UserDataDB] Reset Watchlists 1–5 to blank for user '%s'", uid)

nsedash/core/watchlists.py

@@ -1,262 +0,0 @@
-"""
-nsedash.core.watchlists
-=======================
-Watchlist persistence: 20 named slots, the default dashboard state file,
-analysis-cache watchlists, and application config. Functional parity with
-the v4.5 methods ``_save_watchlist_json``, ``_load_symbols_from_json``,
-``_load_analysis_json_if_recent``, ``load_watchlist_names``,
-``save_dashboard_state``, ``load_config`` and ``save_config``.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from datetime import datetime
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import pandas as pd
-
-from ..config import WATCHLIST_SLOTS, Paths
-
-if TYPE_CHECKING:
-    from .hf_sync import HFDataSync, _NoOpSync
-
-
-class WatchlistStore:
-
-    def __init__(self, paths: Paths, hf_sync=None):
-        self.paths = paths
-        # hf_sync is a HFDataSync or _NoOpSync — push_file_async is always safe to call
-        self._hf = hf_sync
-
-    def _backup(self) -> None:
-        if self.paths.session_token:
-            from . import user_auth
-            user_auth.backup_session_to_db(self.paths.session_token, self.paths.base_dir)
-
-    # ------------------------------------------------------------------
-    # Low-level JSON helpers (parity with StockData._* in v4.5)
-    # ------------------------------------------------------------------
-
-    def save_watchlist_json(self, df_to_save: pd.DataFrame,
-                            file_path: Path, name: str) -> None:
-        """Save a DataFrame as a watchlist JSON with name and timestamp."""
-        watchlist_data = {
-            "name": name,
-            "symbols": [
-                {"SYMBOL": row['SYMBOL'], "Quantity": 0.0, "Price": 0.0}
-                for _, row in df_to_save.iterrows()
-            ],
-            "last_updated": datetime.now().isoformat(),
-        }
-        try:
-            file_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(file_path, 'w') as f:
-                json.dump(watchlist_data, f, indent=4)
-            print(f"[ACTIVITY] Session watchlist JSON updated: {file_path.name}", flush=True)
-            logging.info(f"Saved '{name}' watchlist to {file_path}")
-            if self._hf:
-                self._hf.push_file_async(file_path)
-            self._backup()
-        except Exception as e:
-            logging.error(f"Error saving '{name}' watchlist to {file_path}: {e}")
-
-    def load_symbols_from_json(self, file_path: Path,
-                               current_data_tran_date=None
-                               ) -> tuple[str, list[dict]]:
-        """
-        Load symbol entries from a watchlist JSON file.
-        Returns (watchlist_name, [{'SYMBOL','Quantity','Price'}, ...]).
-        If ``current_data_tran_date`` is provided and the file's
-        ``last_updated`` predates it, returns a cache miss (name, []).
-        Returns ("", []) on error.
-        """
-        file_path = Path(file_path)
-        import re
-        m = re.search(r"nse_dashboard_watchlist([1-5])\.json$", file_path.name)
-        if m:
-            slot = int(m.group(1))
-            userid = self.paths.session_token or "admin"
-            from .userdata_db import UserDataDB
-            wl = UserDataDB.get_watchlist(userid, slot)
-            if wl is not None:
-                symbols = [{"SYMBOL": s, "Quantity": 0.0, "Price": 0.0} for s in wl.get("symbols", [])]
-                return wl.get("name", f"Watchlist {slot}"), symbols
-
-        if not file_path.exists():
-            logging.info(f"JSON file not found: {file_path}")
-            return "", []
-        try:
-            with open(file_path, 'r') as f:
-                data = json.load(f)
-            watchlist_name = data.get('name', file_path.stem)
-            if current_data_tran_date:
-                last_updated_str = data.get('last_updated')
-                if not last_updated_str:
-                    logging.warning(
-                        f"'{file_path}' has no 'last_updated' field. Recalculating."
-                    )
-                    return watchlist_name, []
-                last_updated_date = datetime.fromisoformat(last_updated_str).date()
-                ref = current_data_tran_date
-                ref_date = ref.date() if hasattr(ref, 'date') else ref
-                if last_updated_date < ref_date:
-                    logging.info(f"'{file_path}' is outdated. Recalculating.")
-                    return watchlist_name, []
-            symbols_data = []
-            for entry in data.get('symbols', []):
-                if isinstance(entry, dict) and 'SYMBOL' in entry:
-                    symbols_data.append({
-                        "SYMBOL": entry['SYMBOL'],
-                        "Quantity": entry.get('Quantity', 0.0),
-                        "Price": entry.get('Price', 0.0),
-                    })
-                elif isinstance(entry, str):
-                    symbols_data.append(
-                        {"SYMBOL": entry, "Quantity": 0.0, "Price": 0.0}
-                    )
-                else:
-                    logging.warning(f"Skipping invalid entry in JSON file: {entry}")
-            return watchlist_name, symbols_data
-        except (json.JSONDecodeError, ValueError) as e:
-            logging.warning(f"Error parsing '{file_path}': {e}.")
-            return "", []
-        except Exception as e:
-            logging.error(f"Unexpected error loading JSON from {file_path}: {e}.")
-            return "", []
-
-    def load_analysis_json_if_recent(self, file_path: Path,
-                                     current_data_tran_date) -> list[str] | None:
-        """Symbol list from an analysis-cache watchlist if fresh, else None."""
-        _name, loaded = self.load_symbols_from_json(file_path, current_data_tran_date)
-        if loaded:
-            return [item['SYMBOL'] for item in loaded]
-        return None
-
-    # ------------------------------------------------------------------
-    # Named slots (1..20)
-    # ------------------------------------------------------------------
-
-    def load_watchlist_names(self) -> list[str]:
-        """Read watchlist names from slot files and UserDataDB for slots 1..5."""
-        names = [f"Watchlist {i}" for i in range(1, WATCHLIST_SLOTS + 1)]
-        userid = self.paths.session_token or "admin"
-        from .userdata_db import UserDataDB
-        
-        for i in range(WATCHLIST_SLOTS):
-            slot = i + 1
-            if 1 <= slot <= 5:
-                wl = UserDataDB.get_watchlist(userid, slot)
-                if wl and wl.get("name"):
-                    names[i] = wl["name"]
-                    continue
-            file_path = self.paths.watchlist_slot(slot)
-            try:
-                with open(file_path, "r") as f:
-                    data = json.load(f)
-                names[i] = data.get("name", names[i])
-            except (FileNotFoundError, json.JSONDecodeError):
-                continue
-        return names
-
-    def save_slot(self, slot: int, name: str, symbols: list[dict]) -> Path:
-        """Write a full watchlist to a numbered slot and UserDataDB if slot 1..5."""
-        if not (1 <= slot <= WATCHLIST_SLOTS):
-            raise ValueError(f"Watchlist slot must be 1..{WATCHLIST_SLOTS}")
-        file_path = self.paths.watchlist_slot(slot)
-        with open(file_path, "w") as f:
-            json.dump({
-                "name": name,
-                "symbols": symbols,
-                "last_updated": datetime.now().isoformat(),
-            }, f, indent=2)
-        
-        if 1 <= slot <= 5:
-            userid = self.paths.session_token or "admin"
-            from .userdata_db import UserDataDB
-            sym_list = [s.get("SYMBOL") if isinstance(s, dict) else str(s) for s in symbols]
-            UserDataDB.save_watchlist(userid, slot, name, sym_list)
-
-        print(f"[ACTIVITY] Session watchlist slot {slot} updated: {file_path.name}", flush=True)
-        if self._hf:
-            self._hf.push_file_async(file_path)
-        self._backup()
-        return file_path
-
-    def add_symbols_to_slot(self, slot: int, symbols: list[str]) -> tuple[str, int]:
-        """
-        Merge symbols into a numbered slot (dedup). Returns
-        (watchlist_name, number_of_new_symbols_added).
-        """
-        file_path = self.paths.watchlist_slot(slot)
-        existing_name, existing = self.load_symbols_from_json(file_path)
-        watchlist_name = existing_name or f"Watchlist {slot}"
-        existing_set = {e['SYMBOL'] for e in existing if 'SYMBOL' in e}
-        added = 0
-        for s in symbols:
-            if s not in existing_set:
-                existing.append({"SYMBOL": s, "Quantity": 0.0, "Price": 0.0})
-                existing_set.add(s)
-                added += 1
-        if added > 0:
-            self.save_slot(slot, watchlist_name, existing)
-        return watchlist_name, added
-
-    # ------------------------------------------------------------------
-    # Dashboard state + config
-    # ------------------------------------------------------------------
-
-    def save_dashboard_state(self, cells: list[dict]) -> None:
-        """Auto-save current stocks to the main state file between sessions."""
-        try:
-            state = {
-                'symbols': cells,
-                'last_updated': datetime.now().isoformat(),
-            }
-            with open(self.paths.state_file, 'w') as f:
-                json.dump(state, f, indent=2)
-            print(f"[ACTIVITY] Session dashboard state file updated: {self.paths.state_file.name}", flush=True)
-            if self._hf:
-                self._hf.push_file_async(self.paths.state_file)
-            self._backup()
-        except Exception as e:
-            logging.error(f"Failed to save main dashboard state file: {e}")
-
-    def load_config(self) -> dict:
-        """Load persisted UI config (window geometry kept for compatibility)."""
-        defaults = {
-            'window_size': '1280x800',
-            'window_position': '+100+100',
-            'current_watchlist_name': "Default List",
-        }
-        try:
-            with open(self.paths.config_file, 'r') as f:
-                config = json.load(f)
-            return {**defaults, **config}
-        except FileNotFoundError:
-            return defaults
-        except Exception as e:
-            logging.error(f"Failed to load config: {e}")
-            return defaults
-
-    def save_config(self, current_watchlist_name: str,
-                    window_size: str = '1280x800',
-                    window_position: str = '+100+100') -> None:
-        """Persist config. Geometry fields retained for file compatibility."""
-        try:
-            config = {
-                'window_size': window_size,
-                'window_position': window_position,
-                'last_updated': datetime.now().isoformat(),
-                'current_watchlist_name': current_watchlist_name,
-            }
-            with open(self.paths.config_file, 'w') as f:
-                json.dump(config, f, indent=2)
-            print(f"[ACTIVITY] Session configuration file updated: {self.paths.config_file.name}", flush=True)
-            if self._hf:
-                self._hf.push_file_async(self.paths.config_file)
-            self._backup()
-        except Exception as e:
-            logging.error(f"Failed to save config: {e}")

nsedash/crew/agents.py

@@ -1,221 +0,0 @@
-"""
-nsedash.crew.agents
-===================
-Domain agents. Each agent's tools are deterministic Python callables over
-the core services — no LLM. The analysis tools reproduce, byte-for-byte in
-behaviour, the compute paths of v4.5's ``filter_stocks_by_change``,
-``filter_stocks_by_volume_change``, ``filter_stocks_by_change_return`` and
-``load_top_score_stocks``, including their JSON cache-freshness logic.
-"""
-
-from __future__ import annotations
-
-import logging
-from datetime import datetime
-
-from ..config import ANALYSIS_LIMIT, Paths
-from ..core.portfolio import import_portfolio
-from ..core.recommender_bridge import run_recommendations, train_all_models
-from ..core.stock_data import StockData
-from ..core.watchlists import WatchlistStore
-from .framework import Agent, Tool
-
-PRICE_CHANGE_COLUMNS = {
-    "1D": "DAILY_CHANGE", "1W": "WEEK_CHANGE",
-    "1M": "MONTH_CHANGE", "3M": "THREE_MONTH_CHANGE",
-}
-RETURN_COLUMNS = {
-    "1W": "WEEK_CHANGE", "1M": "MONTH_CHANGE", "3M": "THREE_MONTH_CHANGE",
-}
-
-
-def build_agents(stock_data: StockData, watchlists: WatchlistStore,
-                 paths: Paths) -> dict[str, Agent]:
-    """Construct the full agent roster wired to live service instances."""
-
-    # ------------------------------------------------------------------
-    # Data steward tools
-    # ------------------------------------------------------------------
-
-    def reload_csv() -> dict:
-        stock_data.load_data()
-        return {"last_date": str(stock_data.last_date)}
-
-    def enrich_date(date) -> "object":
-        return stock_data.get_data_for_date(date)
-
-    # ------------------------------------------------------------------
-    # Market watcher tools
-    # ------------------------------------------------------------------
-
-    def live_quotes(symbols: list[str], preferred_source: str = "Google") -> dict:
-        data, source = stock_data.fetch_real_time_data(symbols, preferred_source)
-        return {"quotes": data, "source": source}
-
-    # ------------------------------------------------------------------
-    # Analyst tools (cache-first, parity with v4.5)
-    # ------------------------------------------------------------------
-
-    def price_change_analysis(duration: str, change_type: str,
-                              threshold: float, selected_date) -> dict:
-        entry = paths.hist_price_files.get((duration, change_type))
-        if not entry:
-            raise ValueError("Invalid duration or change type.")
-        json_name = f"{duration} {change_type} Price Portfolio"
-        current_max = stock_data.df['TRAN_DATE'].max()
-        cached = watchlists.load_analysis_json_if_recent(entry, current_max)
-        if cached:
-            return {"symbols": cached, "json_name": json_name, "cached": True}
-        change_column = PRICE_CHANGE_COLUMNS.get(duration)
-        if not change_column:
-            raise ValueError("Invalid duration specified.")
-        data = stock_data.get_data_for_date(selected_date)
-        if data.empty:
-            return {"symbols": [], "json_name": json_name, "cached": False}
-        if change_column == "DAILY_CHANGE":
-            # 1D change is derived, matching the card's daily-change formula.
-            data = data.copy()
-            data["DAILY_CHANGE"] = (
-                (data['CLOSE_PRICE'] / data['PREV_CL_PR'] - 1) * 100
-            ).where(data['PREV_CL_PR'] != 0, 0.0)
-        if change_column not in data.columns:
-            raise ValueError(f"'{change_column}' data not available.")
-        filtered = data.dropna(subset=[change_column]).copy()
-        if change_type == "Max":
-            filtered = filtered[filtered[change_column] >= threshold] \
-                .sort_values(by=change_column, ascending=False)
-        else:
-            filtered = filtered[filtered[change_column] <= threshold] \
-                .sort_values(by=change_column, ascending=True)
-        top = filtered.head(ANALYSIS_LIMIT).copy()
-        watchlists.save_watchlist_json(top, entry, json_name)
-        return {"symbols": top['SYMBOL'].tolist(),
-                "json_name": json_name, "cached": False}
-
-    def volume_change_analysis(duration: str, volume_cutoff: float) -> dict:
-        entry = paths.hist_vol_files.get(duration)
-        if not entry:
-            raise ValueError("Invalid duration specified.")
-        json_name = f"{duration} Max Volume Portfolio"
-        current_max = stock_data.df['TRAN_DATE'].max()
-        cached = watchlists.load_analysis_json_if_recent(entry, current_max)
-        if cached:
-            return {"symbols": cached, "json_name": json_name, "cached": True}
-        movers = stock_data.get_top_volume_movers(
-            duration, limit=ANALYSIS_LIMIT, volume_cutoff=volume_cutoff
-        )
-        if movers.empty:
-            return {"symbols": [], "json_name": json_name, "cached": False}
-        watchlists.save_watchlist_json(movers, entry, json_name)
-        return {"symbols": movers['SYMBOL'].tolist(),
-                "json_name": json_name, "cached": False}
-
-    def return_analysis(duration: str, selected_date) -> dict:
-        entry = paths.max_return_files.get(duration)
-        if not entry:
-            raise ValueError("Invalid duration specified.")
-        json_name = f"{duration} Max Return Portfolio"
-        current_max = stock_data.df['TRAN_DATE'].max()
-        cached = watchlists.load_analysis_json_if_recent(entry, current_max)
-        if cached:
-            return {"symbols": cached, "json_name": json_name, "cached": True}
-        change_column = RETURN_COLUMNS.get(duration)
-        if not change_column:
-            raise ValueError("Invalid duration specified.")
-        data = stock_data.get_data_for_date(selected_date)
-        if data.empty:
-            return {"symbols": [], "json_name": json_name, "cached": False}
-        if change_column not in data.columns:
-            raise ValueError(f"'{change_column}' data not available.")
-        filtered = data.dropna(subset=[change_column]).copy()
-        filtered = filtered[filtered[change_column] > 0] \
-            .sort_values(by=change_column, ascending=False)
-        top = filtered.head(ANALYSIS_LIMIT).copy()
-        watchlists.save_watchlist_json(top, entry, json_name)
-        return {"symbols": top['SYMBOL'].tolist(),
-                "json_name": json_name, "cached": False}
-
-    def top_score_analysis() -> dict:
-        entry = paths.max_score_watchlist_file
-        json_name = "Top 50 MC Score"
-        cached = watchlists.load_analysis_json_if_recent(entry, datetime.now())
-        if cached:
-            return {"symbols": cached, "json_name": json_name, "cached": True}
-        top = stock_data.get_top_mc_score_movers(limit=ANALYSIS_LIMIT)
-        if top.empty:
-            return {"symbols": [], "json_name": json_name, "cached": False}
-        watchlists.save_watchlist_json(top, entry, json_name)
-        return {"symbols": top['SYMBOL'].tolist(),
-                "json_name": json_name, "cached": False}
-
-    # ------------------------------------------------------------------
-    # Portfolio manager tools
-    # ------------------------------------------------------------------
-
-    def portfolio_import(filepath: str | None = None) -> dict:
-        holdings, unmapped, broker_name = import_portfolio(filepath, paths)
-        return {"holdings": holdings, "unmapped": unmapped, "broker_name": broker_name}
-
-    # ------------------------------------------------------------------
-    # Recommender tools
-    # ------------------------------------------------------------------
-
-    def recommend(symbols: list[str]) -> list[dict]:
-        return run_recommendations(
-            paths.recommender_dir, symbols, stock_data.symbol_history
-        )
-
-    def train_models() -> dict:
-        symbols = stock_data.get_all_symbols_from_df()
-        logging.info(f"Training LSTM models for {len(symbols)} symbols.")
-        return train_all_models(
-            paths.recommender_dir, symbols, stock_data.symbol_history
-        )
-
-    # ------------------------------------------------------------------
-    # Agent roster
-    # ------------------------------------------------------------------
-
-    return {
-        "data_steward": Agent(
-            role="Data Steward",
-            goal="Own the historical dataset: load, trim, enrich, and cache.",
-            backstory="Deterministic custodian of nse_processed_data.csv.",
-            tools=[
-                Tool("reload_csv", reload_csv, "Reload the source CSV"),
-                Tool("enrich_date", enrich_date, "Enriched frame for a date"),
-            ],
-        ),
-        "market_watcher": Agent(
-            role="Market Watcher",
-            goal="Fetch live quotes with Google↔Yahoo failover.",
-            backstory="Scrapes real-time prices concurrently with timeouts.",
-            tools=[Tool("live_quotes", live_quotes, "Fetch live quotes")],
-        ),
-        "analyst": Agent(
-            role="Quant Analyst",
-            goal="Rank the universe by price, volume, return and MC score.",
-            backstory="Cache-first analytics identical to dashboard v4.5.",
-            tools=[
-                Tool("price_change_analysis", price_change_analysis),
-                Tool("volume_change_analysis", volume_change_analysis),
-                Tool("return_analysis", return_analysis),
-                Tool("top_score_analysis", top_score_analysis),
-            ],
-        ),
-        "portfolio_manager": Agent(
-            role="Portfolio Manager",
-            goal="Import broker holdings and value the book.",
-            backstory="Maps ICICI short names to NSE symbols.",
-            tools=[Tool("portfolio_import", portfolio_import)],
-        ),
-        "recommender": Agent(
-            role="Recommender",
-            goal="Surface LSTM buy/sell calls when models exist.",
-            backstory="Bridges to the optional recommender module.",
-            tools=[
-                Tool("recommend", recommend),
-                Tool("train_models", train_models),
-            ],
-        ),
-    }

nsedash/crew/crews.py

@@ -1,128 +0,0 @@
-"""
-nsedash.crew.crews
-==================
-Pre-wired crews the dashboard service kicks off for its operation
-categories. Each crew is a deterministic pipeline over the agents in
-``nsedash.crew.agents`` — no LLM involved.
-"""
-
-from __future__ import annotations
-
-from .framework import Agent, Crew, Process, Task
-
-
-class CrewFactory:
-    """Builds crews bound to the shared agent roster."""
-
-    def __init__(self, agents: dict[str, Agent]):
-        self.agents = agents
-
-    def live_refresh_crew(self, symbols: list[str], preferred_source: str) -> Crew:
-        """One live polling tick: fetch quotes for the displayed symbols."""
-        watcher = self.agents["market_watcher"]
-        return Crew(
-            agents=[watcher],
-            tasks=[Task(
-                description="Fetch live quotes for displayed symbols",
-                agent=watcher, tool="live_quotes", output_key="live",
-                inputs={"symbols": symbols, "preferred_source": preferred_source},
-            )],
-            name="live_refresh",
-        )
-
-    def price_analysis_crew(self, duration: str, change_type: str,
-                            threshold: float, selected_date) -> Crew:
-        analyst = self.agents["analyst"]
-        return Crew(
-            agents=[analyst],
-            tasks=[Task(
-                description=f"{duration} {change_type} price-change screen",
-                agent=analyst, tool="price_change_analysis",
-                output_key="analysis",
-                inputs={"duration": duration, "change_type": change_type,
-                        "threshold": threshold, "selected_date": selected_date},
-            )],
-            name="price_analysis",
-        )
-
-    def volume_analysis_crew(self, duration: str, volume_cutoff: float) -> Crew:
-        analyst = self.agents["analyst"]
-        return Crew(
-            agents=[analyst],
-            tasks=[Task(
-                description=f"{duration} max volume screen",
-                agent=analyst, tool="volume_change_analysis",
-                output_key="analysis",
-                inputs={"duration": duration, "volume_cutoff": volume_cutoff},
-            )],
-            name="volume_analysis",
-        )
-
-    def return_analysis_crew(self, duration: str, selected_date) -> Crew:
-        analyst = self.agents["analyst"]
-        return Crew(
-            agents=[analyst],
-            tasks=[Task(
-                description=f"{duration} max return simulation",
-                agent=analyst, tool="return_analysis", output_key="analysis",
-                inputs={"duration": duration, "selected_date": selected_date},
-            )],
-            name="return_analysis",
-        )
-
-    def top_score_crew(self) -> Crew:
-        analyst = self.agents["analyst"]
-        return Crew(
-            agents=[analyst],
-            tasks=[Task(
-                description="Top-50 Moneycontrol score screen",
-                agent=analyst, tool="top_score_analysis", output_key="analysis",
-            )],
-            name="top_score",
-        )
-
-    def refresh_crew(self, selected_date) -> Crew:
-        """Full dashboard refresh: reload CSV then re-enrich the date."""
-        steward = self.agents["data_steward"]
-        return Crew(
-            agents=[steward],
-            tasks=[
-                Task(description="Reload historical CSV",
-                     agent=steward, tool="reload_csv", output_key="reload"),
-                Task(description="Re-enrich the selected date",
-                     agent=steward, tool="enrich_date", output_key="date_data",
-                     inputs={"date": selected_date}),
-            ],
-            process=Process.sequential,
-            name="dashboard_refresh",
-        )
-
-    def portfolio_crew(self, filepath: str | None = None) -> Crew:
-        pm = self.agents["portfolio_manager"]
-        return Crew(
-            agents=[pm],
-            tasks=[Task(description="Import portfolio",
-                        agent=pm, tool="portfolio_import",
-                        output_key="portfolio",
-                        inputs={"filepath": filepath})],
-            name="portfolio_import",
-        )
-
-    def recommendation_crew(self, symbols: list[str]) -> Crew:
-        rec = self.agents["recommender"]
-        return Crew(
-            agents=[rec],
-            tasks=[Task(description="Buy/sell recommendations",
-                        agent=rec, tool="recommend", output_key="recommendations",
-                        inputs={"symbols": symbols})],
-            name="recommendation",
-        )
-
-    def training_crew(self) -> Crew:
-        rec = self.agents["recommender"]
-        return Crew(
-            agents=[rec],
-            tasks=[Task(description="Train all LSTM models",
-                        agent=rec, tool="train_models", output_key="training")],
-            name="training",
-        )

nsedash/crew/framework.py

@@ -1,151 +0,0 @@
-"""
-nsedash.crew.framework
-======================
-A minimal crewAI-style agentic framework implemented in pure Python.
-
-Deliberately NO LLM anywhere: agents are deterministic executors whose
-"reasoning" is plain Python code. The surface mirrors crewAI's mental model
-(Agent / Task / Crew / Process, ``kickoff()``) so orchestration reads the
-same way, while every task resolves to a concrete Python callable (a Tool).
-
-Concurrency: crews execute on an asyncio event loop. Blocking tool
-functions are dispatched to a thread-pool executor, so Process.parallel
-crews run their tasks genuinely concurrently and never block the caller's
-event loop (used by the web layer's background job system).
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import time
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass, field
-from enum import Enum
-from typing import Any, Callable
-
-
-class Process(str, Enum):
-    sequential = "sequential"
-    parallel = "parallel"
-
-
-@dataclass
-class Tool:
-    """A named deterministic capability: a plain Python callable."""
-    name: str
-    func: Callable[..., Any]
-    description: str = ""
-
-    def run(self, *args, **kwargs) -> Any:
-        return self.func(*args, **kwargs)
-
-
-@dataclass
-class Agent:
-    """
-    A deterministic worker with a role, a goal, and a toolbox.
-    ``execute(tool_name, ...)`` dispatches to the named tool. There is no
-    model call and no free-form planning — dispatch is exact-match.
-    """
-    role: str
-    goal: str
-    backstory: str = ""
-    tools: list[Tool] = field(default_factory=list)
-
-    def tool(self, name: str) -> Tool:
-        for t in self.tools:
-            if t.name == name:
-                return t
-        raise KeyError(f"Agent '{self.role}' has no tool named '{name}'")
-
-    def execute(self, tool_name: str, /, *args, **kwargs) -> Any:
-        start = time.time()
-        try:
-            result = self.tool(tool_name).run(*args, **kwargs)
-            return result
-        finally:
-            logging.info(
-                f"AGENT[{self.role}] tool={tool_name} "
-                f"elapsed={time.time() - start:.2f}s"
-            )
-
-
-@dataclass
-class Task:
-    """
-    A unit of work bound to an agent. ``tool`` names the agent tool to run;
-    ``inputs`` may be a dict, or a callable(context) -> dict evaluated at
-    execution time so later tasks can consume earlier task outputs.
-    ``output_key`` names the result in the crew context.
-    """
-    description: str
-    agent: Agent
-    tool: str
-    output_key: str
-    inputs: dict | Callable[[dict], dict] | None = None
-
-    def resolve_inputs(self, context: dict) -> dict:
-        if self.inputs is None:
-            return {}
-        if callable(self.inputs):
-            return self.inputs(context)
-        return dict(self.inputs)
-
-
-class Crew:
-    """
-    Orchestrates tasks across agents.
-
-    Process.sequential — tasks run in declared order; each task sees prior
-    outputs in the shared context.
-    Process.parallel   — tasks run concurrently; each task sees only the
-    initial inputs.
-    """
-
-    _executor = ThreadPoolExecutor(max_workers=16, thread_name_prefix="crew")
-
-    def __init__(self, agents: list[Agent], tasks: list[Task],
-                 process: Process = Process.sequential, name: str = "crew"):
-        self.agents = agents
-        self.tasks = tasks
-        self.process = process
-        self.name = name
-
-    async def _run_task(self, task: Task, context: dict) -> Any:
-        loop = asyncio.get_running_loop()
-        kwargs = task.resolve_inputs(context)
-        return await loop.run_in_executor(
-            self._executor,
-            lambda: task.agent.execute(task.tool, **kwargs),
-        )
-
-    async def kickoff_async(self, inputs: dict | None = None) -> dict:
-        context: dict = dict(inputs or {})
-        logging.info(f"CREW[{self.name}] kickoff process={self.process.value} "
-                     f"tasks={len(self.tasks)}")
-        if self.process is Process.sequential:
-            for task in self.tasks:
-                context[task.output_key] = await self._run_task(task, context)
-        else:
-            results = await asyncio.gather(
-                *(self._run_task(t, context) for t in self.tasks)
-            )
-            for task, result in zip(self.tasks, results):
-                context[task.output_key] = result
-        return context
-
-    def kickoff(self, inputs: dict | None = None) -> dict:
-        """
-        Synchronous entry point. Runs on a private event loop when no loop
-        is active in the current thread; otherwise callers should await
-        ``kickoff_async`` directly.
-        """
-        try:
-            asyncio.get_running_loop()
-        except RuntimeError:
-            return asyncio.run(self.kickoff_async(inputs))
-        raise RuntimeError(
-            "Crew.kickoff() called from a running event loop; "
-            "use `await crew.kickoff_async(...)` instead."
-        )

nsedash/logging_utils.py

@@ -1,82 +0,0 @@
-"""
-nsedash.logging_utils
-=====================
-Application + metrics logging, and the elapsed-time decorator.
-
-The Tkinter ``wait_dialog_decorator`` from v4.5 has no equivalent in a
-server process; its UX role (a modal "please wait" indicator) is fulfilled
-by the web job system (``nsedash.web.background``) which the frontend polls
-to show a spinner. ``elapsed_time_logger`` is retained with identical log
-format: ``timestamp | function | message | N.NN sec``.
-"""
-
-from __future__ import annotations
-
-import logging
-import threading
-import time
-from datetime import datetime
-from functools import wraps
-from pathlib import Path
-
-_metrics_lock = threading.Lock()
-_metrics_file: Path | None = None
-
-
-def setup_logging(log_file: Path, metrics_file: Path) -> None:
-    """Configure the root logger to write to the dashboard log file."""
-    global _metrics_file
-    _metrics_file = metrics_file
-    log_file.parent.mkdir(parents=True, exist_ok=True)
-    root = logging.getLogger()
-    root.setLevel(logging.INFO)
-    # Avoid duplicate handlers on repeated app-factory calls (tests).
-    if not any(
-        isinstance(h, logging.FileHandler)
-        and getattr(h, 'baseFilename', None) == str(log_file)
-        for h in root.handlers
-    ):
-        handler = logging.FileHandler(str(log_file))
-        handler.setFormatter(
-            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        )
-        root.addHandler(handler)
-
-
-def log_operation(operation_name: str, elapsed_time: float) -> None:
-    """Append an operation timing entry to the metrics log (thread-safe)."""
-    if _metrics_file is None:
-        return
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    line = f"{timestamp} | {operation_name} | {elapsed_time:.2f} sec\n"
-    # AUDIT FIX M-10: original opened the file unsynchronised from multiple
-    # threads; interleaved writes could corrupt lines.
-    with _metrics_lock:
-        with open(_metrics_file, "a") as fh:
-            fh.write(line)
-
-
-def log_dashboard_message(message: str) -> None:
-    """Write a progress message to the dashboard log."""
-    logging.info(f"DASHBOARD_PROGRESS: {message}")
-
-
-def elapsed_time_logger(message: str = "Processing..."):
-    """Decorator: logs wall-clock execution time of the wrapped callable."""
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            start_time = time.time()
-            function_name = func.__name__
-            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            try:
-                result = func(*args, **kwargs)
-            finally:
-                elapsed = time.time() - start_time
-                logging.info(
-                    f"{timestamp} | {function_name} | {message} | {elapsed:.2f} sec"
-                )
-                log_operation(function_name, elapsed)
-            return result
-        return wrapper
-    return decorator

nsedash/web/__init__.py

@@ -1,112 +0,0 @@
-"""
-nsedash.web
-===========
-Flask application factory. Wires the service layer, agent roster, crews,
-async runtime, job manager, live scheduler and script runner into
-``app.extensions["nsedash"]``.
-
-HF Persistence
---------------
-When ``SPACE_ID`` is set (HuggingFace Space), ``HFDataSync`` is
-activated:
-- On startup: pulls all processed files from ``Sam-max1/nsedash_data``
-  *before* ``DashboardService`` is created so the service sees fresh data.
-- On every file write: pushes the changed file asynchronously (3-second
-  debounce, fire-and-forget background thread).
-- On session cleanup: removes per-user files from the dataset.
-
-Multi-User Auth
----------------
-``user_auth.init(base_dir)`` is called once to set file paths, then the
-``UserDatabase`` is loaded (creating the seed users.db if absent).
-"""
-
-from __future__ import annotations
-
-import os
-
-from flask import Flask
-
-from ..config import HF_DATASET_REPO, Paths
-from ..core.dashboard_service import DashboardService
-from ..core.hf_sync import make_sync
-from ..core import user_auth
-from ..crew.agents import build_agents
-from ..crew.crews import CrewFactory
-from ..logging_utils import setup_logging
-from .api import api
-from .background import (AsyncRuntime, JobManager, LiveRefreshScheduler,
-                          ScriptRunner)
-from .routes import pages
-
-
-def create_app(base_dir=None) -> Flask:
-    paths = Paths(base_dir)
-    paths.ensure_dirs()
-    setup_logging(paths.log_file, paths.metrics_file)
-
-    # ------------------------------------------------------------------
-    # 1. HF Dataset sync — startup pull BEFORE DashboardService / auth init
-    # ------------------------------------------------------------------
-    hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HF_PRIVATE_TOKEN")
-    hf_sync = make_sync(data_dir=paths.data_dir, repo_id=HF_DATASET_REPO, token=hf_token)
-    hf_sync.pull_on_startup()
-
-    # ------------------------------------------------------------------
-    # 2. Initialise user auth (file paths, seed users.db, link hf_sync)
-    # ------------------------------------------------------------------
-    user_auth.init(paths.base_dir, hf_sync=hf_sync)
-    user_auth.invalidate_all_sessions_on_startup()
-
-
-    # ------------------------------------------------------------------
-    # 3. Service / crew layer
-    # ------------------------------------------------------------------
-    service = DashboardService(paths, hf_sync=hf_sync)
-    agents = build_agents(service.stock_data, service.watchlists, paths)
-    crews = CrewFactory(agents)
-    runtime = AsyncRuntime()
-    jobs = JobManager(runtime)
-    scheduler = LiveRefreshScheduler(runtime, service.load_live_data, base_dir=paths.base_dir)
-    scripts = ScriptRunner()
-
-    app = Flask(__name__)
-    app.extensions["nsedash"] = {
-        "paths": paths,
-        "service": service,
-        "services": {"admin": service},
-        "base_dir": paths.base_dir,
-        "agents": agents,
-        "crews": crews,
-        "runtime": runtime,
-        "jobs": jobs,
-        "scheduler": scheduler,
-        "scripts": scripts,
-        "hf_sync": hf_sync,
-    }
-    app.register_blueprint(pages)
-    app.register_blueprint(api)
-
-    # Start inactivity cleanup daemon thread and stats reporting thread
-    from .api import start_cleanup_thread, start_stats_thread
-    start_cleanup_thread(app)
-    start_stats_thread(app)
-
-    @app.after_request
-    def add_security_headers(response):
-        response.headers["Content-Security-Policy"] = (
-            "default-src 'self'; "
-            "script-src 'self' https://cdnjs.cloudflare.com; "
-            "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
-            "font-src 'self' https://fonts.gstatic.com; "
-            "img-src 'self' data:; "
-            "connect-src 'self';"
-        )
-        response.headers["X-Frame-Options"] = "SAMEORIGIN"
-        response.headers["X-Content-Type-Options"] = "nosniff"
-        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
-        response.headers["Permissions-Policy"] = "geolocation=(), camera=(), microphone=()"
-        return response
-
-    return app
-

nsedash/web/api.py

@@ -1,1196 +0,0 @@
-"""
-nsedash.web.api
-===============
-REST API. Every v4.5 dashboard action maps to an endpoint. Long-running
-operations (analysis crews, refresh, portfolio import, recommendations,
-training) are submitted to the JobManager and polled by the frontend —
-the web equivalent of the v4.5 wait-dialog + worker-thread pattern.
-
-Authentication
---------------
-All endpoints (except /api/auth/* and /api/status) require a valid
-``X-Session-Token`` header issued by the login + TOTP flow.
-The token resolves to a ``userid`` via ``user_auth.SessionManager``,
-which is then used as the per-user isolation key in ``_svc()``.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import signal
-import subprocess
-import threading
-import time
-from pathlib import Path
-
-import pandas as pd
-from flask import Blueprint, current_app, jsonify, request, send_file
-from werkzeug.utils import secure_filename
-
-from ..config import UTILITY_SCRIPTS, WATCHLIST_SLOTS
-from ..core import user_auth
-
-api = Blueprint("api", __name__, url_prefix="/api")
-
-# In-memory activity tracker (userid -> last_active timestamp)
-_active_sessions: dict[str, float] = {}
-SESSION_TIMEOUT_SECONDS = 15 * 60  # 15 minutes
-
-# In-memory tracker for recently inactivated sessions to notify UI
-_inactive_tokens: dict[str, float] = {}
-
-# Hourly session stats
-total_connections_this_hour = 0
-total_disconnections_this_hour = 0
-
-# Routes exempt from auth check
-_AUTH_EXEMPT = {
-    "/api/auth/login",
-    "/api/auth/totp",
-    "/api/auth/logout",
-    "/api/auth/status",
-    "/api/status",
-}
-
-
-@api.before_request
-def check_auth():
-    if current_app.testing:
-        return
-    if request.path in _AUTH_EXEMPT:
-        return
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token")
-             or request.args.get("session_token"))
-    if not token:
-        return jsonify({"error": "Authentication required", "auth_required": True}), 401
-    valid, _userid = user_auth.SessionManager.validate_session(token)
-    if not valid:
-        if token:
-            now = time.time()
-            # Prune old entries to prevent leaks
-            for k in list(_inactive_tokens.keys()):
-                if now - _inactive_tokens[k] > 3600:
-                    _inactive_tokens.pop(k, None)
-            if token in _inactive_tokens:
-                _inactive_tokens.pop(token, None)
-                return jsonify({"error": "Session terminated due to inactivity", "auth_required": True, "reason": "inactivity"}), 401
-        return jsonify({"error": "Session expired or invalid", "auth_required": True}), 401
-
-
-# ---------------------------------------------------------------------------
-# Auth endpoints
-# ---------------------------------------------------------------------------
-
-@api.post("/auth/login")
-def auth_login():
-    """Step 1: userid + password -> returns totp_required and optional QR."""
-    body = request.get_json(force=True)
-    userid = (body.get("userid") or "").strip()
-    password = body.get("password") or ""
-    if not userid or not password:
-        return jsonify({"ok": False, "message": "userid and password are required."}), 400
-
-    success, message, totp_required = user_auth.AuthenticationService.login(userid, password)
-    if not success:
-        return jsonify({"ok": False, "message": message}), 401
-
-    resp = {"ok": True, "userid": userid, "totp_required": totp_required, "message": message}
-    if message in ("first_login", "totp_setup"):
-        resp["qr"] = user_auth.AuthenticationService.get_totp_qr(userid)
-        user = user_auth.UserDatabase.get_user(userid) or {}
-        secret = user.get("totp_secret")
-        if not secret and user.get("totp_secret_encrypted"):
-            secret = user_auth.EncryptionManager.decrypt(user["totp_secret_encrypted"])
-        resp["secret"] = secret
-    return jsonify(resp)
-
-
-@api.post("/auth/totp")
-def auth_totp():
-    """Step 2: userid + TOTP code -> issues a session token."""
-    body = request.get_json(force=True)
-    userid = (body.get("userid") or "").strip()
-    code = (body.get("code") or "").strip()
-    if not userid or not code:
-        return jsonify({"ok": False, "message": "userid and code are required."}), 400
-
-    success, result = user_auth.AuthenticationService.verify_totp(userid, code)
-    if not success:
-        return jsonify({"ok": False, "message": result}), 401
-
-    user = user_auth.UserDatabase.get_user(userid) or {}
-    force_pw = bool(user.get("force_password_change", False))
-
-    _active_sessions[userid] = time.time()
-    global total_connections_this_hour
-    total_connections_this_hour += 1
-    print(f"[SESSION] Login: {userid} (IP: {request.remote_addr})", flush=True)
-    logging.info("Login: %s from %s", userid, request.remote_addr)
-    return jsonify({"ok": True, "session_token": result, "userid": userid, "force_password_change": force_pw})
-
-
-@api.post("/auth/logout")
-def auth_logout():
-    """Destroy the current session."""
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token"))
-    if token:
-        valid, userid = user_auth.SessionManager.validate_session(token)
-        user_auth.SessionManager.destroy_session(token)
-        if userid:
-            _active_sessions.pop(userid, None)
-            global total_disconnections_this_hour
-            total_disconnections_this_hour += 1
-            print(f"[SESSION] Logout: {userid}", flush=True)
-    return jsonify({"ok": True})
-
-
-@api.get("/auth/status")
-def auth_status():
-    """Check if the current session is valid."""
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token")
-             or request.args.get("session_token"))
-    if not token:
-        return jsonify({"authenticated": False})
-    valid, userid = user_auth.SessionManager.validate_session(token)
-    if valid:
-        u = user_auth.UserDatabase.get_user(userid) or {}
-        is_adm = True if userid == "admin" else bool(u.get("is_admin", False))
-        force_pw = bool(u.get("force_password_change", False))
-        from ..core.nvidia_llm import get_nvidia_api_status
-        return jsonify({"authenticated": True, "userid": userid,
-                        "is_admin": is_adm,
-                        "force_password_change": force_pw,
-                        "nvidia_status": get_nvidia_api_status()})
-    
-    # Check if this token was terminated due to inactivity
-    if token:
-        now = time.time()
-        for k in list(_inactive_tokens.keys()):
-            if now - _inactive_tokens[k] > 3600:
-                _inactive_tokens.pop(k, None)
-        if token in _inactive_tokens:
-            _inactive_tokens.pop(token, None)
-            return jsonify({"authenticated": False, "reason": "inactivity"})
-            
-    return jsonify({"authenticated": False})
-
-
-@api.post("/auth/change-password")
-def auth_change_password():
-    """Change user password enforcing policy & 2-time entry confirmation."""
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token")
-             or request.args.get("session_token"))
-    if not token:
-        return jsonify({"ok": False, "message": "Authentication required."}), 401
-    valid, userid = user_auth.SessionManager.validate_session(token)
-    if not valid or not userid:
-        return jsonify({"ok": False, "message": "Session invalid or expired."}), 401
-
-    body = request.get_json(force=True)
-    current_pw = body.get("current_password")
-    new_pw = body.get("new_password") or ""
-    confirm_pw = body.get("confirm_password") or ""
-
-    user = user_auth.UserDatabase.get_user(userid) or {}
-    is_forced = bool(user.get("force_password_change", False))
-
-    ok, msg = user_auth.change_user_password(
-        userid, current_pw, new_pw, confirm_pw, is_admin_override=is_forced
-    )
-    if not ok:
-        return jsonify({"ok": False, "message": msg}), 400
-
-    return jsonify({"ok": True, "message": msg})
-
-
-def start_cleanup_thread(app):
-    import threading
-    from ..core import user_auth as _ua
-
-    def _cleanup():
-        while True:
-            time.sleep(60)
-            try:
-                with app.app_context():
-                    _ua.SessionManager.cleanup_expired()
-
-                    now = time.time()
-                    extensions = app.extensions["nsedash"]
-                    services = extensions["services"]
-                    base_dir = extensions["base_dir"]
-                    hf_sync = extensions.get("hf_sync")
-                    from ..config import Paths
-                    import shutil
-
-                    expired_users = [
-                        userid for userid, last_active in list(_active_sessions.items())
-                        if userid not in ("admin",) and (now - last_active) > SESSION_TIMEOUT_SECONDS
-                    ]
-
-                    if expired_users:
-                        logging.info("[Cleanup] Expiring %d inactive session(s).", len(expired_users))
-
-                    for userid in expired_users:
-                        logging.info("[Cleanup] Purging session: %s", userid)
-                        print(f"[SESSION] Disconnected (Inactivity): {userid}", flush=True)
-                        global total_disconnections_this_hour
-                        total_disconnections_this_hour += 1
-
-                        for tok in _ua.SessionManager.get_userid_sessions(userid):
-                            _inactive_tokens[tok] = time.time()
-                            _ua.SessionManager.destroy_session(tok)
-
-                        if userid in services:
-                            del services[userid]
-                        _active_sessions.pop(userid, None)
-
-                        paths = Paths(base_dir=base_dir, session_token=userid)
-                        try:
-                            for f in (paths.state_file, paths.config_file, paths.portfolio_file):
-                                if f.exists():
-                                    f.unlink()
-                            if paths.watchlist_dir.exists():
-                                shutil.rmtree(paths.watchlist_dir)
-                        except Exception as exc:
-                            logging.error("[Cleanup] Error removing files for %s: %s", userid, exc)
-
-                        if hf_sync:
-                            hf_sync.delete_session_files(userid)
-
-            except Exception as ex:
-                logging.error("[Cleanup] Exception: %s", ex)
-
-    threading.Thread(target=_cleanup, daemon=True).start()
-
-
-def start_stats_thread(app):
-    import threading
-    import time
-
-    def _stats():
-        last_15m = time.time()
-        last_hour = time.time()
-        while True:
-            time.sleep(10)
-            now = time.time()
-            if now - last_15m >= 900:
-                last_15m = now
-                active_count = len(_active_sessions)
-                print(f"[STATS] {time.strftime('%Y-%m-%d %H:%M:%S')} - Active sessions: {active_count}", flush=True)
-                logging.info("[STATS] Active sessions: %d", active_count)
-            if now - last_hour >= 3600:
-                last_hour = now
-                global total_connections_this_hour, total_disconnections_this_hour
-                print(f"[STATS] Hourly: Logins={total_connections_this_hour}, Logouts={total_disconnections_this_hour}", flush=True)
-                logging.info("[STATS] Hourly: Logins=%d, Logouts=%d", total_connections_this_hour, total_disconnections_this_hour)
-                total_connections_this_hour = 0
-                total_disconnections_this_hour = 0
-
-    threading.Thread(target=_stats, daemon=True).start()
-
-
-def get_session_token() -> str:
-    """
-    Resolve the current request's session token to a userid.
-
-    The userid is the isolation key for DashboardService lookup.
-    Falls back to 'admin' for unauthenticated requests.
-    """
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token")
-             or request.args.get("session_token"))
-    if token:
-        valid, userid = user_auth.SessionManager.validate_session(token)
-        if valid and userid:
-            _active_sessions[userid] = time.time()
-            return userid
-    _active_sessions["admin"] = time.time()
-    return "admin"
-
-
-@api.after_request
-def backup_user_data_on_change(response):
-    """Automatically backs up user files encrypted to users.db on POST/PUT/DELETE."""
-    if request.method in ("POST", "PUT", "DELETE"):
-        # Ensure we don't back up auth or admin-action endpoints themselves
-        if not request.path.startswith(("/api/auth/", "/api/admin/")):
-            userid = get_session_token()
-            if userid and userid != "admin":
-                extensions = current_app.extensions["nsedash"]
-                base_dir = extensions["base_dir"]
-                user_auth.backup_session_to_db(userid, base_dir)
-    return response
-
-
-def _svc():
-    token = get_session_token()
-    _active_sessions[token] = time.time()  # Record last active timestamp
-    extensions = current_app.extensions["nsedash"]
-    services = extensions["services"]
-    if token not in services:
-        import shutil
-        from ..config import Paths
-        from ..core.dashboard_service import DashboardService
-        
-        base_dir = extensions["base_dir"]
-        paths = Paths(base_dir=base_dir, session_token=token)
-        paths.ensure_dirs()
-        
-        # Check if we can restore from encrypted users.db database
-        restored = user_auth.restore_session_from_db(token, base_dir)
-        
-        if not restored:
-            # Seed default state file if missing
-            default_state_file = base_dir / 'processed' / 'nse_dashboard_contents.json'
-            if paths.state_file != default_state_file and not paths.state_file.exists():
-                if default_state_file.exists():
-                    shutil.copy2(default_state_file, paths.state_file)
-                    
-            # Seed default config file if missing
-            default_config_file = base_dir / 'processed' / 'nse_dashboard_config.json'
-            if paths.config_file != default_config_file and not paths.config_file.exists():
-                if default_config_file.exists():
-                    shutil.copy2(default_config_file, paths.config_file)
-                    
-            # Seed default watchlists if slot directory is empty or missing slots
-            default_wl_dir = base_dir / 'processed' / 'watchlists'
-            if paths.watchlist_dir != default_wl_dir and default_wl_dir.exists() and not any(paths.watchlist_dir.iterdir()):
-                import re
-                for item in default_wl_dir.iterdir():
-                    if item.is_file():
-                        match = re.match(r"nse_dashboard_watchlist(\d+)\.json", item.name)
-                        if match:
-                            num = int(match.group(1))
-                            if 1 <= num <= WATCHLIST_SLOTS:
-                                if default_wl_dir.joinpath(item.name).exists():
-                                    shutil.copy2(item, paths.watchlist_dir / item.name)
-                                else:
-                                    # Create blank watchlist slot for other users
-                                    with open(paths.watchlist_dir / item.name, "w") as f:
-                                        json.dump({
-                                            "name": f"Watchlist {num}",
-                                            "symbols": [],
-                                            "last_updated": datetime.now().isoformat()
-                                        }, f, indent=2)
-                        else:
-                            # Copy non-slot files (like analysis cache)
-                            shutil.copy2(item, paths.watchlist_dir / item.name)
-
-            # Ensure all 1..20 watchlist slots exist in paths.watchlist_dir
-            from datetime import datetime
-            import json
-            for num in range(1, WATCHLIST_SLOTS + 1):
-                slot_file = paths.watchlist_dir / f"nse_dashboard_watchlist{num}.json"
-                if not slot_file.exists():
-                    src_file = default_wl_dir / f"nse_dashboard_watchlist{num}.json"
-                    if paths.watchlist_dir != default_wl_dir and src_file.exists():
-                        try:
-                            shutil.copy2(src_file, slot_file)
-                        except Exception:
-                            pass
-                    else:
-                        try:
-                            with open(slot_file, "w") as f:
-                                json.dump({
-                                    "name": f"Watchlist {num}",
-                                    "symbols": [],
-                                    "last_updated": datetime.now().isoformat()
-                                }, f, indent=2)
-                        except Exception:
-                            pass
-
-        services[token] = DashboardService(paths, hf_sync=extensions.get("hf_sync"))
-        
-    return services[token]
-
-
-
-def _crews():
-    return current_app.extensions["nsedash"]["crews"]
-
-
-def _jobs():
-    return current_app.extensions["nsedash"]["jobs"]
-
-
-def _scheduler():
-    return current_app.extensions["nsedash"]["scheduler"]
-
-
-def _scripts():
-    return current_app.extensions["nsedash"]["scripts"]
-
-
-# ---------------------------------------------------------------------------
-# Board snapshot
-# ---------------------------------------------------------------------------
-
-@api.get("/board")
-def board():
-    charts = request.args.get("charts", "1") != "0"
-    return jsonify(_svc().board_payload(include_charts=charts))
-
-
-# ---------------------------------------------------------------------------
-# Cell management
-# ---------------------------------------------------------------------------
-
-@api.post("/stocks")
-def add_stock():
-    body = request.get_json(force=True)
-    key = body.get("key")
-    svc = _svc()
-    cell = svc.add_stock(key)
-    if cell is None:
-        return jsonify({"ok": False,
-                        "message": f"Stock '{key}' not found in data."}), 404
-    return jsonify({"ok": True, "cell": svc._cell_payload(cell)})
-
-
-@api.delete("/stocks/<symbol>")
-def remove_stock(symbol):
-    ok = _svc().remove_stock(symbol)
-    return jsonify({"ok": ok})
-
-
-@api.delete("/stocks")
-def delete_all():
-    _svc().delete_all_stocks(mark_modified=True)
-    return jsonify({"ok": True})
-
-
-@api.post("/stocks/<symbol>/inputs")
-def cell_inputs(symbol):
-    body = request.get_json(force=True)
-    result = _svc().update_cell_inputs(
-        symbol, body.get("quantity", 0), body.get("price", 0)
-    )
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/stocks/<symbol>/select")
-def cell_select(symbol):
-    body = request.get_json(force=True)
-    _svc().toggle_selection(symbol, bool(body.get("selected")))
-    return jsonify({"ok": True})
-
-
-@api.post("/mc-link")
-def set_mc_link():
-    body = request.get_json(force=True)
-    svc = _svc()
-    try:
-        entry = svc.stock_data.set_mc_link(
-            body.get("symbol", ""), body.get("security", ""),
-            body.get("link", "")
-        )
-    except ValueError as e:
-        return jsonify({"ok": False, "message": str(e)}), 400
-    svc._mc_links_cache = None  # invalidate shared cache
-    return jsonify({"ok": True, "entry": entry,
-                    "message": f"Moneycontrol link saved for "
-                               f"{body.get('symbol')}."})
-
-
-# ---------------------------------------------------------------------------
-# Date / refresh / export / config
-# ---------------------------------------------------------------------------
-
-@api.post("/date")
-def change_date():
-    body = request.get_json(force=True)
-    result = _svc().change_date(body.get("date", ""))
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/refresh")
-def refresh():
-    svc, crews, jobs = _svc(), _crews(), _jobs()
-    if svc.real_time_active:
-        _scheduler().stop()
-
-    async def run():
-        crew = crews.refresh_crew(svc.selected_date)
-        result = svc.refresh_dashboard()
-        _ = crew  # crew retained for parity; refresh_dashboard drives reload
-        return result
-
-    job_id = jobs.submit("Refreshing dashboard data...", run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-@api.post("/export")
-def export_csv():
-    result = _svc().export_to_csv()
-    return jsonify(result), (200 if result.get("ok") else 500)
-
-
-@api.get("/export/download")
-def export_download():
-    path = request.args.get("file", "")
-    p = Path(path)
-    if not p.exists() or p.parent != _svc().paths.log_dir:
-        return jsonify({"ok": False, "message": "File not found."}), 404
-    return send_file(p, as_attachment=True)
-
-
-@api.post("/config")
-def set_config():
-    body = request.get_json(force=True)
-    svc = _svc()
-    if "volume_cutoff" in body:
-        try:
-            v = float(body["volume_cutoff"])
-            if v < 0:
-                raise ValueError
-            svc.volume_cutoff_threshold = v
-        except (TypeError, ValueError):
-            return jsonify({"ok": False,
-                            "message": "Volume cutoff must be a "
-                                       "non-negative number."}), 400
-    svc.persist_config()
-    return jsonify({"ok": True,
-                    "volume_cutoff": svc.volume_cutoff_threshold})
-
-
-# ---------------------------------------------------------------------------
-# Live data
-# ---------------------------------------------------------------------------
-
-@api.post("/live/start")
-def live_start():
-    body = request.get_json(force=True)
-    svc = _svc()
-    result = svc.start_live(body.get("source"), body.get("refresh_rate"))
-    if not result.get("ok"):
-        return jsonify(result), 400
-    if not svc.is_trading_hour():
-        result["warning"] = ("Outside NSE trading hours (09:15–15:30 IST, "
-                             "Mon–Fri). Prices may be stale.")
-    _scheduler().start(svc.live_refresh_rate)
-    return jsonify(result)
-
-
-@api.post("/live/stop")
-def live_stop():
-    _scheduler().stop()
-    return jsonify(_svc().stop_live())
-
-
-@api.post("/live/refresh")
-def live_refresh_now():
-    svc, jobs = _svc(), _jobs()
-
-    async def run():
-        import asyncio
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, svc.load_live_data)
-
-    job_id = jobs.submit("Fetching live data...", run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-# ---------------------------------------------------------------------------
-# Watchlists
-# ---------------------------------------------------------------------------
-
-@api.get("/watchlists")
-def watchlist_names():
-    return jsonify({"names": _svc().watchlist_names,
-                    "slots": WATCHLIST_SLOTS})
-
-
-@api.post("/watchlists/load/<int:num>")
-def watchlist_load(num):
-    result = _svc().load_watchlist(num)
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/watchlists/load-name")
-def watchlist_load_name():
-    body = request.get_json(force=True)
-    result = _svc().load_dashboard_state_by_name(body.get("name", ""))
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/watchlists/save")
-def watchlist_save():
-    result = _svc().save_dashboard()
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/watchlists/save-as")
-def watchlist_save_as():
-    body = request.get_json(force=True)
-    result = _svc().save_as_watchlist(body.get("slot"), body.get("name", ""))
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/watchlists/add-selected")
-def watchlist_add_selected():
-    body = request.get_json(force=True)
-    result = _svc().add_selected_to_watchlist(body.get("slot"))
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/watchlists/load-file")
-def watchlist_load_file():
-    """Arbitrary watchlist JSON: uploaded file or a known live-analysis name."""
-    svc = _svc()
-    known = {
-        "maxvol50": svc.paths.max_vol_watchlist_file,
-        "maxprice50": svc.paths.max_price_watchlist_file,
-        "maxfall50": svc.paths.max_fall_watchlist_file,
-        "maxscore50": svc.paths.max_score_watchlist_file,
-        "maxreturn1w50": svc.paths.max_return_files.get("1W"),
-        "maxreturn1m50": svc.paths.max_return_files.get("1M"),
-        "maxreturn3m50": svc.paths.max_return_files.get("3M"),
-    }
-    name = request.form.get("known") or (request.json or {}).get("known") \
-        if not request.files else None
-    if name:
-        path = known.get(name)
-        if not path:
-            return jsonify({"ok": False, "message": "Unknown analysis file."}), 400
-        result = svc.load_watchlist_from_file(path)
-        return jsonify(result), (200 if result.get("ok") else 400)
-    f = request.files.get("file")
-    if not f:
-        return jsonify({"ok": False, "message": "No file supplied."}), 400
-    dest = svc.paths.watchlist_dir / secure_filename(f.filename)
-    f.save(dest)
-    result = svc.load_watchlist_from_file(dest)
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-@api.post("/index/<name>")
-def load_index(name):
-    result = _svc().load_index_stocks(name.upper())
-    return jsonify(result), (200 if result.get("ok") else 400)
-
-
-# ---------------------------------------------------------------------------
-# Analysis (crew-driven, job-based)
-# ---------------------------------------------------------------------------
-
-def _analysis_job(description, crew_builder):
-    svc, jobs = _svc(), _jobs()
-
-    async def run():
-        ctx = await crew_builder().kickoff_async()
-        a = ctx["analysis"]
-        return svc.display_analysis_result(
-            a["symbols"], a["json_name"], a["cached"]
-        )
-
-    job_id = jobs.submit(description, run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-@api.post("/analysis/price")
-def analysis_price():
-    body = request.get_json(force=True)
-    duration = body.get("duration")
-    change_type = body.get("change_type")
-    try:
-        threshold = float(body.get("threshold"))
-    except (TypeError, ValueError):
-        return jsonify({"ok": False,
-                        "message": "Threshold must be a number."}), 400
-    svc, crews = _svc(), _crews()
-    return _analysis_job(
-        f"Calculating {duration} {change_type} price changes...",
-        lambda: crews.price_analysis_crew(
-            duration, change_type, threshold, svc.selected_date
-        ),
-    )
-
-
-@api.post("/analysis/volume")
-def analysis_volume():
-    body = request.get_json(force=True)
-    duration = body.get("duration")
-    svc, crews = _svc(), _crews()
-    return _analysis_job(
-        f"Calculating {duration} volume movers...",
-        lambda: crews.volume_analysis_crew(
-            duration, svc.volume_cutoff_threshold
-        ),
-    )
-
-
-@api.post("/analysis/return")
-def analysis_return():
-    body = request.get_json(force=True)
-    duration = body.get("duration")
-    svc, crews = _svc(), _crews()
-    return _analysis_job(
-        f"Calculating {duration} max returns...",
-        lambda: crews.return_analysis_crew(duration, svc.selected_date),
-    )
-
-
-@api.post("/analysis/top-score")
-def analysis_top_score():
-    crews = _crews()
-    return _analysis_job("Loading top Moneycontrol scores...",
-                         crews.top_score_crew)
-
-
-# ---------------------------------------------------------------------------
-# Portfolio / recommender
-# ---------------------------------------------------------------------------
-
-@api.post("/portfolio/import")
-def portfolio_import():
-    svc, crews, jobs = _svc(), _crews(), _jobs()
-
-    # Save the file from request context if uploaded
-    f = request.files.get("file")
-    filepath = None
-    if f:
-        session_token = get_session_token()
-        filename = secure_filename(f.filename)
-        dest = svc.paths.data_dir / f"uploaded_portfolio_{session_token}.csv"
-        f.save(dest)
-        filepath = str(dest)
-    else:
-        # Fallback to existing csv version of portfolio file if it exists
-        csv_path = Path(str(svc.paths.portfolio_file).replace('.parquet', '.csv'))
-        if csv_path.exists():
-            filepath = str(csv_path)
-
-    async def run():
-        ctx = await crews.portfolio_crew(filepath).kickoff_async()
-        p = ctx["portfolio"]
-        
-        result = svc.load_portfolio_holdings(p["holdings"], p["unmapped"], p["broker_name"])
-        
-        # Save the raw uploaded CSV to the parquet portfolio_file
-        if result.get("ok") and filepath:
-            try:
-                df_raw = pd.read_csv(filepath)
-                df_raw.to_parquet(svc.paths.portfolio_file, index=False)
-            except Exception as e:
-                logging.warning(f"Failed to save raw portfolio to parquet: {e}")
-            
-            # Trigger session backup to database/sync
-            svc.watchlists._backup()
-            
-            # Clean up the temporary uploaded CSV file
-            try:
-                Path(filepath).unlink(missing_ok=True)
-            except Exception as e:
-                logging.warning(f"Failed to delete temporary uploaded file {filepath}: {e}")
-                    
-        return result
-
-    job_id = jobs.submit("Importing portfolio...", run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-@api.post("/recommend")
-def recommend():
-    svc, crews, jobs = _svc(), _crews(), _jobs()
-    symbols = [c.symbol for c in svc.cells]
-    if not symbols:
-        return jsonify({"ok": False,
-                        "message": "No stocks on the board to analyse."}), 400
-
-    async def run():
-        ctx = await crews.recommendation_crew(symbols).kickoff_async()
-        recs = ctx["recommendations"]
-        by_symbol = {r["symbol"]: r for r in recs}
-        for cell in svc.cells:
-            cell.recommendation = by_symbol.get(cell.symbol)
-        return {"ok": True, "recommendations": recs}
-
-    job_id = jobs.submit("Running buy/sell recommender...", run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-@api.post("/train")
-def train():
-    crews, jobs = _crews(), _jobs()
-
-    async def run():
-        ctx = await crews.training_crew().kickoff_async()
-        return {"ok": True, **ctx["training"]}
-
-    job_id = jobs.submit("Training LSTM models for all symbols...", run)
-    return jsonify({"ok": True, "job_id": job_id})
-
-
-# ---------------------------------------------------------------------------
-# Jobs
-# ---------------------------------------------------------------------------
-
-@api.get("/jobs/<job_id>")
-def job_status(job_id):
-    job = _jobs().get(job_id)
-    if job is None:
-        return jsonify({"ok": False, "message": "Unknown job."}), 404
-    return jsonify(_jobs().to_payload(job))
-
-
-# ---------------------------------------------------------------------------
-# Utility scripts (interactive web terminal)
-# ---------------------------------------------------------------------------
-
-@api.get("/scripts")
-def scripts_list():
-    return jsonify({"scripts": [
-        {"label": label, "file": file} for label, file in UTILITY_SCRIPTS
-    ]})
-
-
-@api.post("/scripts/start")
-def scripts_start():
-    body = request.get_json(force=True)
-    file = body.get("file", "")
-    if file not in {f for _, f in UTILITY_SCRIPTS}:
-        return jsonify({"ok": False, "message": "Unknown script."}), 400
-    svc = _svc()
-    run = _scripts().start(file, str(svc.paths.base_dir))
-    logging.info(f"Started utility script {file} run={run.id}")
-    return jsonify({"ok": True, "run_id": run.id})
-
-
-@api.get("/scripts/<run_id>/output")
-def scripts_output(run_id):
-    run = _scripts().get(run_id)
-    if run is None:
-        return jsonify({"ok": False, "message": "Unknown run."}), 404
-    after = request.args.get("after", -1, type=int)
-    return jsonify({"ok": True, "lines": run.lines_since(after),
-                    "finished": run.finished,
-                    "returncode": run.returncode})
-
-
-@api.post("/scripts/<run_id>/input")
-def scripts_input(run_id):
-    run = _scripts().get(run_id)
-    if run is None:
-        return jsonify({"ok": False, "message": "Unknown run."}), 404
-    body = request.get_json(force=True)
-    ok = run.send_input(body.get("text", ""))
-    return jsonify({"ok": ok})
-
-
-@api.get("/status")
-def server_status():
-    svc = _svc()
-    exiting = getattr(svc, "is_exiting", False)
-    is_hf = "SPACE_ID" in os.environ
-    from ..core.nvidia_llm import get_nvidia_api_status
-    return jsonify({"ok": True, "exiting": exiting, "is_hf": is_hf,
-                    "nvidia_status": get_nvidia_api_status()})
-
-
-@api.post("/shutdown")
-def shutdown():
-    is_hf = "SPACE_ID" in os.environ
-    if is_hf:
-        logging.info("Hugging Face mode: /shutdown cleans up current user session.")
-        userid = get_session_token()
-        extensions = current_app.extensions["nsedash"]
-        services = extensions["services"]
-        base_dir = extensions["base_dir"]
-        hf_sync = extensions.get("hf_sync")
-        from ..config import Paths
-        import shutil
-
-        if userid and userid not in ("admin", "default"):
-            logging.info("Purging session '%s' assets on UI exit...", userid)
-            print(f"[SESSION] Disconnected (User Exit): {userid}", flush=True)
-            global total_disconnections_this_hour
-            total_disconnections_this_hour += 1
-
-            for tok in user_auth.SessionManager.get_userid_sessions(userid):
-                user_auth.SessionManager.destroy_session(tok)
-
-            if userid in services:
-                del services[userid]
-            _active_sessions.pop(userid, None)
-
-            paths = Paths(base_dir=base_dir, session_token=userid)
-            try:
-                for f in (paths.state_file, paths.config_file, paths.portfolio_file):
-                    if f.exists():
-                        f.unlink()
-                if paths.watchlist_dir.exists():
-                    shutil.rmtree(paths.watchlist_dir)
-            except Exception as exc:
-                logging.error("Error purging session '%s' assets: %s", userid, exc)
-
-            if hf_sync:
-                hf_sync.delete_session_files(userid)
-
-        return jsonify({"ok": True, "message": "UI session closed successfully."})
-
-    logging.info("Initiating graceful shutdown via API...")
-    
-    # Mark as exiting
-    try:
-        _svc().is_exiting = True
-    except Exception:
-        pass
-        
-    # Trigger SIGINT process termination (handled by register_shutdown_handler in nsedash_app.py)
-    import sys
-    is_testing = current_app.testing or current_app.config.get("TESTING") or "pytest" in sys.modules
-    if not is_testing:
-        def kill_self():
-            time.sleep(0.1)
-            logging.info("Triggering SIGINT signal for process shutdown.")
-            os.kill(os.getpid(), signal.SIGINT)
-        threading.Thread(target=kill_self, daemon=True).start()
-    else:
-        # Under tests, we manually run cleanup since we won't SIGINT the test process
-        logging.info("Testing mode: performing manual cleanup instead of SIGINT.")
-        try:
-            _scheduler().stop()
-        except Exception:
-            pass
-        try:
-            scripts = _scripts()
-            for run in list(scripts._runs.values()):
-                if run.process and run.process.poll() is None:
-                    run.process.terminate()
-                    try:
-                        run.process.wait(timeout=1.0)
-                    except Exception:
-                        run.process.kill()
-        except Exception:
-            pass
-        try:
-            runtime = current_app.extensions["nsedash"]["runtime"]
-            runtime.shutdown()
-        except Exception:
-            pass
-
-    return jsonify({"ok": True, "message": "Backend shutting down..."})
-
-
-# ---------------------------------------------------------------------------
-# Admin API endpoints
-# ---------------------------------------------------------------------------
-
-def is_admin_request() -> bool:
-    token = (request.headers.get("X-Session-Token")
-             or request.cookies.get("session_token")
-             or request.args.get("session_token"))
-    if not token:
-        return False
-    valid, userid = user_auth.SessionManager.validate_session(token)
-    if not valid:
-        return False
-    if userid == "admin":
-        return True
-    u = user_auth.UserDatabase.get_user(userid)
-    return bool(u and u.get("is_admin", False))
-
-
-@api.get("/admin/dashboard")
-def admin_dashboard():
-    if not is_admin_request():
-        return jsonify({"error": "Unauthorized"}), 403
-
-    all_users = user_auth.UserDatabase.get_all_users()
-    
-    # Calculate some metrics
-    extensions = current_app.extensions["nsedash"]
-    service = extensions.get("service") or _svc()
-    
-    from ..config import NIFTY_STOCKS, SENSEX_STOCKS
-    total_symbols = len(set(NIFTY_STOCKS + SENSEX_STOCKS))
-    
-    # Check yfinance connection status (parity with mlbench)
-    yfinance_ok = True
-    try:
-        import requests
-        r = requests.get(
-            "https://query1.finance.yahoo.com/v8/finance/chart/RELIANCE.NS?interval=1d&range=1d",
-            headers={"User-Agent": "Mozilla/5.0"},
-            timeout=1.5
-        )
-        yfinance_ok = (r.status_code == 200)
-    except Exception:
-        yfinance_ok = False
-
-    return jsonify({
-        "ok": True,
-        "total_users": len(all_users),
-        "total_symbols": total_symbols,
-        "yfinance_connected": yfinance_ok,
-        "users": all_users,
-    })
-
-
-@api.post("/admin/action")
-def admin_action():
-    if not is_admin_request():
-        return jsonify({"error": "Unauthorized"}), 403
-
-    body = request.get_json(force=True)
-    action = body.get("action")
-    target_uid = body.get("target_uid")
-    
-    admin_user = user_auth.SessionManager.validate_session(
-        request.headers.get("X-Session-Token") or request.cookies.get("session_token")
-    )[1]
-
-    if action == "add_user":
-        username = body.get("username", "").strip()
-        role = body.get("role", "user")
-        if not username:
-            return jsonify({"ok": False, "message": "Username is required."}), 400
-        is_adm = (role == "admin")
-        created = user_auth.UserDatabase.create_user(username, is_admin=is_adm)
-        if created:
-            user_auth.ActivityLogger.log_activity(admin_user, "add_user", "success", f"Created user {username} with role {role}")
-            return jsonify({"ok": True, "message": f"User '{username}' created successfully."})
-        return jsonify({"ok": False, "message": "Failed to create user. User may already exist."}), 400
-
-    elif action == "qr":
-        qr_uri = user_auth.AuthenticationService.get_totp_qr(target_uid)
-        user = user_auth.UserDatabase.get_user(target_uid) or {}
-        secret = user.get("totp_secret", "Not Set")
-        return jsonify({"ok": True, "qr": qr_uri, "secret": secret})
-
-    elif action == "metrics":
-        user = user_auth.UserDatabase.get_user(target_uid)
-        if not user:
-            return jsonify({"ok": False, "message": "User not found"}), 404
-        
-        # Load user's data to check symbols/watchlists
-        user_data = user_auth.get_user_data(target_uid)
-        state_data = user_data.get("state") or {}
-        symbols_count = len(state_data.get("symbols", []))
-        watchlists_count = len(user_data.get("watchlists", {}))
-        
-        return jsonify({
-            "ok": True,
-            "userid": target_uid,
-            "login_count": user.get("login_count", 0),
-            "last_login": user.get("last_login", "Never"),
-            "symbols_count": symbols_count,
-            "watchlists_count": watchlists_count,
-        })
-
-    elif action == "reset":
-        if target_uid == admin_user:
-            return jsonify({"ok": False, "message": "Cannot reset your own account while logged in."}), 400
-        res = user_auth.UserDatabase.reset_totp(target_uid)
-        if res:
-            user_auth.SessionManager.clear_sessions_for_user(target_uid)
-            user_auth.ActivityLogger.log_activity(admin_user, "reset_account", "success", f"Full reset for {target_uid}")
-            return jsonify({"ok": True, "message": f"Full account reset for {target_uid}. They have been logged out."})
-        return jsonify({"ok": False, "message": "Failed to reset account."}), 400
-
-    elif action == "lock":
-        if target_uid == admin_user:
-            return jsonify({"ok": False, "message": "Cannot lock your own admin account."}), 400
-        user = user_auth.UserDatabase.get_user(target_uid)
-        if user:
-            new_status = not user.get("active", True)
-            user_auth.UserDatabase.update_user(target_uid, {"active": new_status})
-            status_str = "unlocked" if new_status else "locked"
-            user_auth.ActivityLogger.log_activity(admin_user, "lock_user", "success", f"Toggled status of {target_uid} to {status_str}")
-            return jsonify({"ok": True, "message": f"User '{target_uid}' account status toggled to {status_str}."})
-        return jsonify({"ok": False, "message": "User not found."}), 404
-
-    elif action == "delete":
-        if target_uid == admin_user:
-            return jsonify({"ok": False, "message": "Cannot delete your own admin account."}), 400
-        res = user_auth.UserDatabase.delete_user(target_uid)
-        if res:
-            user_auth.ActivityLogger.log_activity(admin_user, "delete_user", "success", f"Permanently deleted user {target_uid}")
-            return jsonify({"ok": True, "message": f"User '{target_uid}' has been deleted."})
-        return jsonify({"ok": False, "message": "Failed to delete user."}), 400
-
-    elif action == "mirror":
-        logs = user_auth.ActivityLogger.get_activity_logs(target_uid, limit=20)
-        return jsonify({"ok": True, "logs": logs})
-
-    elif action == "chat":
-        # Port chat functionality: return simulated message log with the user
-        msgs = [
-            {"sender": "system", "text": f"Connected to {target_uid} session channel..."},
-            {"sender": "system", "text": "No live messages in queue. Showing connection log."}
-        ]
-        return jsonify({"ok": True, "messages": msgs})
-
-    return jsonify({"ok": False, "message": "Unknown action."}), 400
-
-
-@api.get("/stocks/<symbol>/ai-analysis")
-def get_ai_analysis(symbol: str):
-    """
-    Get or generate AI-driven stock analysis in markdown format using the 5-pipeline
-    agentic research analyst (nsedash_research.py). Non-blocking background mode.
-    """
-    token = get_session_token()
-    base_dir = current_app.extensions["nsedash"]["base_dir"]
-    force_refresh = request.args.get("force", "0") == "1"
-
-    # Extract user-selected model and timeout
-    req_json = request.get_json(silent=True) or {}
-    selected_model = request.args.get("model") or req_json.get("model") or "google/diffusiongemma-26b-a4b-it"
-    timeout_raw = request.args.get("timeout") or req_json.get("timeout") or 5
-    try:
-        timeout_mins = int(timeout_raw)
-        timeout_sec = timeout_mins * 60 if timeout_mins <= 60 else timeout_mins
-    except (ValueError, TypeError):
-        timeout_sec = 300
-
-    import sys
-    sys_path = str(Path(base_dir))
-    if sys_path not in sys.path:
-        sys.path.insert(0, sys_path)
-    from nsedash_research import NSEMarketResearchAnalyst
-
-    analyst = NSEMarketResearchAnalyst(base_dir=base_dir)
-    cache_file = analyst.aianalysis_dir / f"{symbol.upper()}_agentic.md"
-
-    # If cached and not force_refresh, return immediately!
-    if not force_refresh and cache_file.exists():
-        try:
-            mtime = cache_file.stat().st_mtime
-            if time.time() - mtime < 24 * 3600:
-                result = analyst.run_full_research_pipeline(symbol, force_refresh=False, selected_model=selected_model, timeout_seconds=timeout_sec)
-                return jsonify(result)
-        except Exception:
-            pass
-
-    # Submit background job to prevent blocking Flask worker / UI thread
-    jobs = _jobs()
-
-    async def run():
-        res = analyst.run_full_research_pipeline(symbol, force_refresh=True, selected_model=selected_model, timeout_seconds=timeout_sec)
-        # Sync with HF dataset if enabled
-        from ..config import Paths
-        paths = Paths(base_dir=base_dir, session_token=token)
-        file_path = paths.aianalysis_dir / f"{symbol}.md"
-        hf_sync = current_app.extensions["nsedash"].get("hf_sync")
-        if hf_sync and file_path.exists():
-            hf_sync.push_file_async(file_path)
-        return res
-
-    job_id = jobs.submit(f"AI Agentic Research for {symbol}", run)
-    return jsonify({"ok": True, "job_id": job_id, "symbol": symbol, "status": "running", "cached": False, "selected_model": selected_model, "timeout_seconds": timeout_sec})
-
-
-@api.post("/stocks/<symbol>/ai-analysis/start")
-def start_ai_analysis(symbol: str):
-    """Explicitly trigger fresh background AI agentic research."""
-    return get_ai_analysis(symbol)
-
-
-
-

nsedash/web/background.py

@@ -1,281 +0,0 @@
-"""
-nsedash.web.background
-======================
-Async runtime for the web application.
-
-- ``AsyncRuntime``: a dedicated thread running an asyncio event loop. All
-  crews and long operations execute here, so Flask request handlers never
-  block on analysis, scraping, or training (the web equivalent of v4.5's
-  ``_run_long_operation`` daemon threads + wait dialogs).
-- ``JobManager``: submit coroutine factories, poll status/result by job id.
-  The frontend polls ``/api/jobs/<id>`` and shows a spinner — the UX role of
-  the v4.5 ``wait_dialog_decorator``.
-- ``LiveRefreshScheduler``: asyncio task replicating the ``root.after``
-  polling loop (fetch immediately, then every N minutes while active).
-- ``ScriptRunner``: interactive subprocess execution with a scrollback
-  buffer and stdin injection — the web replacement for v4.5's
-  ``run_script_in_shell`` Tk shell window.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import itertools
-import logging
-import os
-import subprocess
-import sys
-import threading
-import time
-import uuid
-from dataclasses import dataclass, field
-from typing import Any, Callable, Coroutine
-
-
-class AsyncRuntime:
-    """Dedicated asyncio loop in a daemon thread."""
-
-    def __init__(self):
-        self.loop = asyncio.new_event_loop()
-        self._thread = threading.Thread(
-            target=self._run, name="nsedash-async", daemon=True
-        )
-        self._thread.start()
-
-    def _run(self):
-        asyncio.set_event_loop(self.loop)
-        self.loop.run_forever()
-
-    def submit(self, coro: Coroutine) -> "asyncio.Future":
-        return asyncio.run_coroutine_threadsafe(coro, self.loop)
-
-    def shutdown(self):
-        self.loop.call_soon_threadsafe(self.loop.stop)
-
-
-@dataclass
-class Job:
-    id: str
-    description: str
-    status: str = "running"          # running | done | error
-    result: Any = None
-    error: str | None = None
-    created_at: float = field(default_factory=time.time)
-    finished_at: float | None = None
-
-
-class JobManager:
-
-    def __init__(self, runtime: AsyncRuntime):
-        self.runtime = runtime
-        self._jobs: dict[str, Job] = {}
-        self._lock = threading.Lock()
-
-    def submit(self, description: str,
-               coro_factory: Callable[[], Coroutine]) -> str:
-        job = Job(id=uuid.uuid4().hex[:12], description=description)
-        with self._lock:
-            self._jobs[job.id] = job
-
-        async def runner():
-            try:
-                job.result = await coro_factory()
-                job.status = "done"
-            except Exception as e:
-                logging.error(f"Job '{description}' failed: {e}")
-                job.status = "error"
-                job.error = str(e)
-            finally:
-                job.finished_at = time.time()
-
-        self.runtime.submit(runner())
-        return job.id
-
-    def get(self, job_id: str) -> Job | None:
-        with self._lock:
-            return self._jobs.get(job_id)
-
-    def to_payload(self, job: Job) -> dict:
-        return {
-            "id": job.id, "description": job.description,
-            "status": job.status, "result": job.result, "error": job.error,
-        }
-
-
-class LiveRefreshScheduler:
-    """Replicates the v4.5 root.after() live polling loop with asyncio and triggers background nsemaxlive.py."""
-
-    def __init__(self, runtime: AsyncRuntime, refresh_callable: Callable[[], dict], base_dir: Any = None):
-        self.runtime = runtime
-        self.refresh_callable = refresh_callable
-        self.base_dir = str(base_dir) if base_dir else None
-        self._task: asyncio.Task | None = None
-        self._interval_min: int = 5
-
-    @property
-    def active(self) -> bool:
-        return self._task is not None and not self._task.done()
-
-    def _run_maxlive_script(self):
-        if not self.base_dir:
-            return
-        script_path = os.path.join(self.base_dir, "nsemaxlive.py")
-        if os.path.exists(script_path):
-            try:
-                logging.info("[LiveScheduler] Triggering background nsemaxlive.py execution...")
-                res = subprocess.run([sys.executable, "-u", script_path], cwd=self.base_dir, timeout=300, capture_output=True, text=True)
-                if res.returncode == 0:
-                    logging.info("[LiveScheduler] Background nsemaxlive.py finished successfully.")
-                else:
-                    logging.warning(f"[LiveScheduler] Background nsemaxlive.py exited with code {res.returncode}: {res.stderr[:200]}")
-            except Exception as exc:
-                logging.error(f"[LiveScheduler] Error running background nsemaxlive.py: {exc}")
-
-    def start(self, interval_min: int) -> None:
-        self.stop()
-        self._interval_min = max(1, int(interval_min))
-
-        async def poll_loop():
-            loop = asyncio.get_running_loop()
-            while True:
-                try:
-                    # Blocking scrape work runs in executor
-                    await loop.run_in_executor(None, self.refresh_callable)
-                except Exception as e:
-                    logging.error(f"Live refresh cycle failed: {e}")
-
-                try:
-                    # Run nsemaxlive.py in background executor
-                    await loop.run_in_executor(None, self._run_maxlive_script)
-                except Exception as e:
-                    logging.error(f"Live maxlive background script failed: {e}")
-
-                await asyncio.sleep(self._interval_min * 60)
-
-        def create():
-            self._task = self.runtime.loop.create_task(poll_loop())
-
-        self.runtime.loop.call_soon_threadsafe(create)
-
-    def stop(self) -> None:
-        task = self._task
-        self._task = None
-        if task and not task.done():
-            self.runtime.loop.call_soon_threadsafe(task.cancel)
-            logging.info("Periodic live data refresh stopped.")
-
-
-class ScriptRun:
-    """One interactive subprocess with buffered, tagged output lines."""
-
-    def __init__(self, script_name: str, script_dir: str):
-        self.id = uuid.uuid4().hex[:12]
-        self.script_name = script_name
-        self.script_dir = script_dir
-        self._lines: list[dict] = []
-        self._seq = itertools.count()
-        self._lock = threading.Lock()
-        self.process: subprocess.Popen | None = None
-        self.returncode: int | None = None
-        self._start()
-
-    def _log(self, message: str, tag: str = "stdout") -> None:
-        with self._lock:
-            self._lines.append(
-                {"n": next(self._seq), "text": message, "tag": tag}
-            )
-
-    def _start(self) -> None:
-        self._log(f"Starting script from directory: {self.script_dir}", "info")
-        self._log(f"Executing command: python -u {self.script_name}", "info")
-        self._log("-" * 80, "info")
-
-        def run():
-            try:
-                creationflags = 0
-                if os.name == 'nt':
-                    creationflags = subprocess.CREATE_NO_WINDOW
-                # AUDIT FIX M-11: use sys.executable, not bare 'python',
-                # which resolves differently (or not at all) per PATH.
-                self.process = subprocess.Popen(
-                    [sys.executable, '-u', self.script_name],
-                    cwd=self.script_dir,
-                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                    stdin=subprocess.PIPE, text=True,
-                    encoding='utf-8', errors='replace',
-                    bufsize=1, creationflags=creationflags,
-                )
-
-                def read_pipe(pipe, tag):
-                    for line in iter(pipe.readline, ''):
-                        self._log(line.rstrip("\n"), tag)
-                    pipe.close()
-
-                t_out = threading.Thread(
-                    target=read_pipe, args=(self.process.stdout, "stdout"),
-                    daemon=True)
-                t_err = threading.Thread(
-                    target=read_pipe, args=(self.process.stderr, "error"),
-                    daemon=True)
-                t_out.start(); t_err.start()
-                rc = self.process.wait()
-                t_out.join(); t_err.join()
-                self.returncode = rc
-                self._log("-" * 80, "info")
-                if rc == 0:
-                    self._log(
-                        f"Process '{self.script_name}' finished successfully "
-                        f"(Exit Code: {rc}).", "success")
-                else:
-                    self._log(
-                        f"Process '{self.script_name}' finished with error "
-                        f"(Exit Code: {rc}).", "error")
-            except FileNotFoundError:
-                self.returncode = -1
-                self._log(
-                    f"Error: Script '{self.script_name}' not found in "
-                    f"'{self.script_dir}'.", "error")
-            except Exception as e:
-                self.returncode = -1
-                self._log(
-                    f"Unexpected error running '{self.script_name}': {e}",
-                    "error")
-            finally:
-                self._log("Input disabled. Process has finished.", "info")
-
-        threading.Thread(target=run, daemon=True).start()
-
-    def send_input(self, text: str) -> bool:
-        text = (text or "").strip()
-        if not text or self.process is None or self.process.poll() is not None:
-            return False
-        try:
-            self._log(f"> {text}", "input")
-            self.process.stdin.write(text + '\n')
-            self.process.stdin.flush()
-            return True
-        except (IOError, BrokenPipeError) as e:
-            self._log(f"Error: Could not write to process. {e}", "error")
-            return False
-
-    def lines_since(self, after: int = -1) -> list[dict]:
-        with self._lock:
-            return [ln for ln in self._lines if ln["n"] > after]
-
-    @property
-    def finished(self) -> bool:
-        return self.returncode is not None
-
-
-class ScriptRunner:
-
-    def __init__(self):
-        self._runs: dict[str, ScriptRun] = {}
-
-    def start(self, script_name: str, script_dir: str) -> ScriptRun:
-        run = ScriptRun(script_name, script_dir)
-        self._runs[run.id] = run
-        return run
-
-    def get(self, run_id: str) -> ScriptRun | None:
-        return self._runs.get(run_id)

nsedash/web/routes.py

@@ -1,14 +0,0 @@
-import uuid
-from flask import Blueprint, render_template, request, make_response
-
-pages = Blueprint("pages", __name__)
-
-
-@pages.get("/")
-def index():
-    token = request.cookies.get("session_token")
-    response = make_response(render_template("index.html"))
-    if not token:
-        token = uuid.uuid4().hex[:12]
-        response.set_cookie("session_token", token, max_age=30*86400, httponly=True, samesite='Lax', secure=request.is_secure)
-    return response

nsedash/web/static/app.js

@@ -1,1548 +0,0 @@
-/* NSE Stock Dashboard — frontend controller.
- * Mirrors every v4.5 desktop interaction against the REST API. */
-
-"use strict";
-
-const $ = (sel) => document.querySelector(sel);
-const state = { board: null, charts: {}, livePoll: null };
-
-function esc(s) {
-  if (s === null || s === undefined) return "";
-  return String(s)
-    .replace(/&/g, "&")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;")
-    .replace(/"/g, "&quot;")
-    .replace(/'/g, "&#039;");
-}
-
-/* ---------------------------------------------------------------- utils */
-
-async function api(path, opts = {}) {
-  const token = localStorage.getItem("session_token") || "";
-  const headers = {
-    "Content-Type": "application/json",
-    "X-Session-Token": token,
-    ...(opts.headers || {})
-  };
-  const res = await fetch("/api" + path, {
-    ...opts,
-    headers,
-  });
-  let body = {};
-  try { body = await res.json(); } catch (e) { /* empty */ }
-  if (res.status === 401 && body.auth_required) {
-    showLoginBox();
-    if (body.reason === "inactivity") {
-      toast("Session terminated due to inactivity.", "err");
-      const loginError = $("#login-error");
-      if (loginError) {
-        loginError.textContent = "Session terminated due to inactivity.";
-        loginError.classList.remove("hidden");
-      }
-    }
-    return body;
-  }
-  if (!res.ok && body.message) toast(body.message, "err");
-  return body;
-}
-
-function toast(msg, kind = "") {
-  const el = document.createElement("div");
-  el.className = "toast " + kind;
-  el.textContent = msg;
-  $("#toasts").appendChild(el);
-  setTimeout(() => el.remove(), 6000);
-}
-
-function fmt(n, dp = 2) {
-  if (n === null || n === undefined || isNaN(n)) return "0.00";
-  return Number(n).toLocaleString("en-IN",
-    { minimumFractionDigits: dp, maximumFractionDigits: dp });
-}
-
-function arrow(v) { return v > 0 ? "↑" : v < 0 ? "↓" : "↔"; }
-function cls(v) { return v > 0 ? "gain" : v < 0 ? "loss" : "flat"; }
-
-function scoreClass(s) {
-  if (s >= 80) return "score-80";
-  if (s >= 60) return "score-60";
-  if (s >= 40) return "score-40";
-  if (s >= 20) return "score-20";
-  return "score-0";
-}
-
-/* Busy overlay + job polling (v4.5 wait dialog equivalent) */
-function busy(text) {
-  $("#busy-text").textContent = text || "Processing…";
-  $("#busy").classList.remove("hidden");
-}
-function unbusy() { $("#busy").classList.add("hidden"); }
-
-async function runJob(startPromise, description) {
-  busy(description);
-  try {
-    const start = await startPromise;
-    if (!start.job_id) { unbusy(); return start; }
-    while (true) {
-      await new Promise((r) => setTimeout(r, 700));
-      const job = await api("/jobs/" + start.job_id);
-      if (job.status === "done") { unbusy(); return job.result || {}; }
-      if (job.status === "error") {
-        unbusy();
-        toast(job.error || "Operation failed.", "err");
-        return { ok: false, message: job.error };
-      }
-    }
-  } catch (e) {
-    unbusy();
-    toast(String(e), "err");
-    return { ok: false };
-  }
-}
-
-/* ---------------------------------------------------------------- board */
-
-async function loadBoard() {
-  state.board = await api("/board");
-  render();
-}
-
-function render() {
-  const b = state.board;
-  if (!b) return;
-
-  // Banner
-  const banner = $("#banner");
-  banner.className = "banner " + (b.banner.mode === "live" ? "live" : "historic");
-  $("#banner-text").textContent = b.banner.mode === "live"
-    ? `Live Data — ${b.banner.source} (every ${b.banner.refresh_rate} min)`
-    : "Historic Data";
-  $("#banner-watchlist").textContent =
-    b.banner.watchlist + (b.watchlist_modified ? " *" : "");
-  $("#live-toggle").checked = b.banner.mode === "live";
-  $("#date-input").value = b.selected_date;
-  $("#date-input").max = new Date().toISOString().slice(0, 10);
-
-  // Search datalist
-  const dl = $("#stock-list");
-  dl.innerHTML = "";
-  for (const key of b.search_keys) {
-    const o = document.createElement("option");
-    o.value = key;
-    dl.appendChild(o);
-  }
-  $("#board-count").textContent = `${b.cells.length} stocks on board`;
-
-  // Live index badges (NIFTY & SENSEX)
-  if (b.indices) {
-    if (b.indices.NIFTY) {
-      const n = b.indices.NIFTY;
-      const isPos = n.change >= 0;
-      const sym = isPos ? "▲" : "▼";
-      const el = $("#nifty-val");
-      if (el) {
-        el.textContent = `${sym} ${n.formatted_price} (${n.formatted_change})`;
-        el.className = `idx-pill ${isPos ? "positive" : "negative"}`;
-      }
-    }
-    if (b.indices.SENSEX) {
-      const s = b.indices.SENSEX;
-      const isPos = s.change >= 0;
-      const sym = isPos ? "▲" : "▼";
-      const el = $("#sensex-val");
-      if (el) {
-        el.textContent = `${sym} ${s.formatted_price} (${s.formatted_change})`;
-        el.className = `idx-pill ${isPos ? "positive" : "negative"}`;
-      }
-    }
-  }
-
-  renderPortfolio(b.portfolio);
-  renderMenus(b);
-  renderGrid(b.cells);
-}
-
-/* RTC Clock in IST Timezone */
-function updateRTC() {
-  try {
-    const options = {
-      timeZone: "Asia/Kolkata",
-      year: "numeric",
-      month: "short",
-      day: "2-digit",
-      hour: "2-digit",
-      minute: "2-digit",
-      second: "2-digit",
-      hour12: true
-    };
-    const istString = new Date().toLocaleString("en-IN", options) + " IST";
-    const rtcEl = document.getElementById("rtc-time");
-    if (rtcEl) rtcEl.textContent = istString;
-  } catch (e) {
-    console.error("RTC clock error:", e);
-  }
-}
-setInterval(updateRTC, 1000);
-updateRTC();
-
-function renderPortfolio(p) {
-  if (!p) return;
-  $("#pf-invested").textContent = fmt(p.invested);
-  $("#pf-current").textContent = fmt(p.current_value);
-  const day = $("#pf-day");
-  day.textContent = `${fmt(p.day_gain_abs)} (${fmt(p.day_gain_pct)}%)`;
-  day.className = cls(p.day_gain_abs);
-  const abs = $("#pf-abs");
-  abs.textContent = `${fmt(p.abs_returns_abs)} (${fmt(p.abs_returns_pct)}%)`;
-  abs.className = cls(p.abs_returns_abs);
-}
-
-/* ---------------------------------------------------------------- menus */
-
-function menuButton(label, fn) {
-  const btn = document.createElement("button");
-  btn.textContent = label;
-  btn.setAttribute("role", "menuitem");
-  btn.addEventListener("click", () => { closeMenus(); fn(); });
-  return btn;
-}
-
-function renderMenus(b) {
-  // Watchlists: load 1..20, load from file, live analysis quick loads
-  const wl = $("#menu-watchlists");
-  wl.innerHTML = "";
-  b.watchlist_names.forEach((name, i) => {
-    const isPrivate = i < 5;
-    const label = isPrivate ? `🔒 ${i + 1}. ${name}` : `${i + 1}. ${name}`;
-    const btn = menuButton(label, () => loadWatchlist(i + 1));
-    if (isPrivate) {
-      btn.style.color = "#10b981";
-      btn.style.fontWeight = "bold";
-    }
-    wl.appendChild(btn);
-  });
-  wl.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-  wl.appendChild(menuButton("Load watchlist from file…", pickWatchlistFile));
-
-  // Max/Min price analysis
-  const mm = $("#menu-maxmin");
-  mm.innerHTML = "";
-  for (const d of ["1D", "1W", "1M", "3M"]) {
-    for (const t of ["Max", "Min"]) {
-      mm.appendChild(menuButton(`${d} ${t} Price Change`,
-        () => priceAnalysis(d, t)));
-    }
-  }
-
-  // Volume analysis
-  const vol = $("#menu-volume");
-  vol.innerHTML = "";
-  for (const d of ["1D", "1W", "1M", "3M"]) {
-    vol.appendChild(menuButton(`${d} Max Volume`, () => volumeAnalysis(d)));
-  }
-  vol.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-  vol.appendChild(menuButton(
-    `Set Volume Cutoff (now ${fmt(b.volume_cutoff, 0)})`, setVolumeCutoff));
-
-  // Live analysis file loads (top 3 menu items only)
-  const la = $("#menu-liveanalysis");
-  la.innerHTML = "";
-  la.appendChild(menuButton("Top 50 Volume Gainers (maxvol50)",
-    () => loadKnownFile("maxvol50")));
-  la.appendChild(menuButton("Top 50 Price Gainers (maxprice50)",
-    () => loadKnownFile("maxprice50")));
-  la.appendChild(menuButton("Top 50 Price Losers (maxfall50)",
-    () => loadKnownFile("maxfall50")));
-
-  // Simulation
-  const sim = $("#menu-simulation");
-  sim.innerHTML = "";
-  sim.appendChild(menuButton("Top 50 Score (maxscore50)",
-    () => loadKnownFile("maxscore50")));
-  sim.appendChild(menuButton("Top Score (Moneycontrol)", topScore));
-  for (const d of ["1W", "1M", "3M"]) {
-    sim.appendChild(menuButton(`${d} Max Return`, () => returnAnalysis(d)));
-  }
-  sim.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-  sim.appendChild(menuButton("Buy/Sell Recommender", recommend));
-
-  // Utilities
-  loadUtilitiesMenu();
-}
-
-async function loadUtilitiesMenu() {
-  const u = $("#menu-utilities");
-  if (!u) return;
-
-  // Immediately render static items so Admin Control Panel is instantly available
-  u.innerHTML = "";
-  u.appendChild(menuButton("Deep Learning: Train All Models", trainModels));
-  u.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-  u.appendChild(menuButton("Admin Control Panel", openAdminPanel));
-
-  try {
-    const res = await api("/scripts");
-    if (res && res.scripts) {
-      u.innerHTML = "";
-      (res.scripts || []).forEach((s) => {
-        u.appendChild(menuButton(s.label, () => startScript(s)));
-      });
-      u.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-      u.appendChild(menuButton("Deep Learning: Train All Models", trainModels));
-      u.appendChild(Object.assign(document.createElement("div"), { className: "sep" }));
-      u.appendChild(menuButton("Admin Control Panel", openAdminPanel));
-    }
-  } catch (err) {
-    console.error("Error loading utility scripts:", err);
-  }
-}
-
-function closeMenus() {
-  document.querySelectorAll(".menu.open")
-    .forEach((m) => {
-      m.classList.remove("open");
-      const btn = m.querySelector(".menu-btn");
-      if (btn) btn.setAttribute("aria-expanded", "false");
-    });
-}
-document.querySelectorAll(".menu-btn").forEach((btn) => {
-  btn.setAttribute("aria-haspopup", "true");
-  btn.setAttribute("aria-expanded", "false");
-  btn.addEventListener("click", (e) => {
-    if (btn.id === "btn-exit") {
-      e.stopPropagation();
-      triggerExit();
-      return;
-    }
-    const menu = e.target.closest(".menu");
-    const wasOpen = menu.classList.contains("open");
-    closeMenus();
-    if (!wasOpen) {
-      menu.classList.add("open");
-      btn.setAttribute("aria-expanded", "true");
-    }
-    e.stopPropagation();
-  });
-});
-document.addEventListener("click", closeMenus);
-
-/* ---------------------------------------------------------------- cards */
-
-function renderGrid(cells) {
-  // Destroy old charts to avoid canvas leaks
-  Object.values(state.charts).forEach((c) => c.destroy());
-  state.charts = {};
-  const grid = $("#grid");
-  grid.innerHTML = "";
-  for (const c of cells) grid.appendChild(cardEl(c));
-}
-
-/* ------------------------------------------------------------- AI Agentic Research */
-let currentAISymbol = null;
-let currentAIAnalysisText = "";
-
-async function openAIAnalysisModal(symbol, forceRefresh = false) {
-  currentAISymbol = symbol;
-  $("#modal-ai").classList.remove("hidden");
-  $("#ai-title").innerHTML = `🤖 Agentic AI Stock Research: <span style="color: #3b82f6; margin-left: 6px;">${esc(symbol)}</span>`;
-  
-  // Show spinner initially
-  $("#ai-content").innerHTML = `<div class="loading-spinner" style="display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 40px; color: var(--muted);">
-    <div class="spinner" style="border: 3px solid rgba(255,255,255,0.1); border-top: 3px solid #3b82f6; border-radius: 50%; width: 30px; height: 30px; animation: spin 1s linear infinite; margin-bottom: 12px;"></div>
-    <span>Connecting to 5-Pipeline AI Agentic Research for ${esc(symbol)}...</span>
-  </div>`;
-  
-  const selectedModel = $("#select-primary-llm") ? $("#select-primary-llm").value : "google/diffusiongemma-26b-a4b-it";
-  const selectedTimeout = $("#select-llm-timeout") ? $("#select-llm-timeout").value : "5";
-
-  $("#ai-meta-model").innerHTML = `🤖 LLM: <b style="color: #3b82f6;">${esc(selectedModel)}</b>`;
-  $("#ai-meta-time").innerHTML = `⏱️ Time: <b style="color: #10b981;">--s (Max ${selectedTimeout}m)</b>`;
-  $("#ai-meta-tokens").innerHTML = `📊 Tokens: <b style="color: #cbd5e1;">In: -- | Out: -- | Total: --</b>`;
-
-  try {
-    const url = `/stocks/${encodeURIComponent(symbol)}/ai-analysis?model=${encodeURIComponent(selectedModel)}&timeout=${encodeURIComponent(selectedTimeout)}${forceRefresh ? '&force=1' : ''}`;
-    const payload = await api(url);
-    
-    if (payload && payload.ok) {
-      if (payload.analysis) {
-        // Immediate cached / completed display!
-        renderAIAnalysisReport(payload);
-      } else if (payload.job_id && payload.status === "running") {
-        // Background job submitted!
-        $("#ai-content").innerHTML = `<div style="padding: 30px; text-align: center; color: #cbd5e1;">
-          <div class="spinner" style="border: 3px solid rgba(255,255,255,0.1); border-top: 3px solid #3b82f6; border-radius: 50%; width: 36px; height: 36px; animation: spin 1s linear infinite; margin: 0 auto 16px auto;"></div>
-          <h4 style="margin: 0 0 8px 0; color: #60a5fa; font-size: 16px;">5-Pipeline AI Agentic Research in Progress</h4>
-          <p style="font-size: 13px; color: #94a3b8; max-width: 500px; margin: 0 auto;">Running 5 research pipelines for <strong>${esc(symbol)}</strong> in background using <strong>${esc(selectedModel)}</strong> (Timeout: ${selectedTimeout} mins).<br/>You can close this modal and continue navigating NSEDASH. Live status is shown at bottom right.</p>
-        </div>`;
-        pollAIAgentBackground(symbol, payload.job_id);
-      }
-    } else {
-      $("#ai-content").innerHTML = `<span class="loss">Error: ${esc((payload && payload.message) || "Failed to start AI analysis")}</span>`;
-    }
-  } catch (err) {
-    $("#ai-content").innerHTML = `<span class="loss">Error: ${esc(err.message)}</span>`;
-  }
-}
-
-function renderAIAnalysisReport(payload) {
-  currentAIAnalysisText = payload.analysis || "";
-  $("#ai-content").innerHTML = marked.parse(currentAIAnalysisText);
-  
-  const modelName = payload.llm_model || "z-ai/glm-5.2";
-  const elapsed = payload.elapsed_seconds !== undefined ? `${payload.elapsed_seconds}s` : "--s";
-  const pt = payload.prompt_tokens || 0;
-  const ct = payload.completion_tokens || 0;
-  const tt = payload.total_tokens || (pt + ct);
-
-  $("#ai-meta-model").innerHTML = `🤖 LLM: <b style="color: #3b82f6;">${esc(modelName)}</b>`;
-  $("#ai-meta-time").innerHTML = `⏱️ Time: <b style="color: #10b981;">${elapsed}</b>`;
-  $("#ai-meta-tokens").innerHTML = `📊 Tokens: <b style="color: #cbd5e1;">In: ${fmt(pt, 0)} | Out: ${fmt(ct, 0)} | Total: ${fmt(tt, 0)}</b>`;
-}
-
-function pollAIAgentBackground(symbol, jobId) {
-  const widget = $("#ai-bg-widget");
-  const spinner = $("#ai-bg-spinner");
-  const text = $("#ai-bg-text");
-  const viewBtn = $("#ai-bg-view-btn");
-
-  if (!widget) return;
-  widget.classList.remove("hidden");
-  spinner.style.display = "block";
-  text.textContent = `🤖 AI Agentic Research Running for ${symbol}...`;
-  viewBtn.style.display = "none";
-
-  const startTime = Date.now();
-  const pollInterval = setInterval(async () => {
-    // Timeout safeguard after 310s
-    if (Date.now() - startTime > 310000) {
-      clearInterval(pollInterval);
-      text.textContent = `❌ AI Research for ${symbol} timed out`;
-      spinner.style.display = "none";
-      setTimeout(() => widget.classList.add("hidden"), 8000);
-      return;
-    }
-
-    try {
-      const job = await api("/jobs/" + jobId);
-      if (!job) return;
-
-      if (job.status === "done") {
-        clearInterval(pollInterval);
-        const res = job.result || {};
-        text.textContent = `✔ AI Agentic Research Completed for ${symbol}!`;
-        spinner.style.display = "none";
-        viewBtn.style.display = "inline-block";
-        viewBtn.onclick = () => {
-          openAIAnalysisModal(symbol, false);
-          widget.classList.add("hidden");
-        };
-
-        toast(`AI Agentic Research report ready for ${symbol}`, "ok");
-
-        // If modal is currently open for this symbol, render immediately!
-        if (currentAISymbol === symbol && !$("#modal-ai").classList.contains("hidden")) {
-          renderAIAnalysisReport(res);
-        }
-      } else if (job.status === "error") {
-        clearInterval(pollInterval);
-        text.textContent = `❌ AI Research failed for ${symbol}`;
-        spinner.style.display = "none";
-        toast(job.error || `AI Research failed for ${symbol}`, "err");
-        setTimeout(() => widget.classList.add("hidden"), 8000);
-      }
-    } catch (e) {
-      console.error("Error polling AI job:", e);
-    }
-  }, 1500);
-}
-
-function cardEl(c) {
-  const card = document.createElement("div");
-  card.className = "card";
-  const ch = c.changes || {};
-  const mc = c.mc || {};
-  const pl = (c.quantity * c.current_cmp) - (c.quantity * c.price);
-  const plPct = c.quantity * c.price
-    ? (pl / (c.quantity * c.price)) * 100 : 0;
-
-  card.innerHTML = `
-    <button class="rm" title="Remove">×</button>
-    <h4>${esc(c.symbol)}</h4>
-    <div class="sec">${esc(c.security || "")} — ${esc(c.tran_date || "")}</div>
-    <div class="row"><span class="lbl">Prev / Curr</span>
-      <b class="num">${fmt(c.prev_close)} / ${fmt(c.close_price)}</b></div>
-    <div class="row"><span class="lbl">MCap / FV / Score</span>
-      <b class="num">${fmt(mc.mcap)} / ${fmt(mc.fv, 0)} /
-        <span class="${scoreClass(mc.score)}">${fmt(mc.score, 0)}</span></b></div>
-    <div class="row"><span class="lbl">52wk Hi (Δ)</span>
-      <b class="num">${fmt(c.hi_52wk)} <span class="loss">(${fmt(c.hi_diff)}%)</span></b></div>
-    <div class="row"><span class="lbl">52wk Lo (Δ)</span>
-      <b class="num">${fmt(c.lo_52wk)} <span class="gain">(${fmt(c.lo_diff)}%)</span></b></div>
-    <div class="row"><span class="lbl">ATH / ATL</span>
-      <b class="num">${fmt(mc.ath)} / ${fmt(mc.atl)}</b></div>
-    <div class="changes">
-      ${["1D", "1W", "1M", "3M"].map((k) =>
-        `<span class="chip ${cls(ch[k])}">${k} ${arrow(ch[k])} ${fmt(ch[k])}%</span>`
-      ).join("")}
-    </div>
-    <div class="volchips">
-      ${(c.volume_changes || []).map((v, i) =>
-        `<span class="chip ${cls(v)}" title="Volume change D-${i + 1}">V${i + 1} ${fmt(v)}%</span>`
-      ).join("")}
-    </div>
-    <canvas></canvas>
-    ${c.live ? `<div class="live-row">
-        LIVE <b class="num ${cls(c.live.pct_change)}">${fmt(c.live.price)}
-        (${arrow(c.live.pct_change)} ${fmt(c.live.pct_change)}%)</b>
-        &nbsp;H ${fmt(c.live.day_high)} L ${fmt(c.live.day_low)}
-        &nbsp;Vol ${arrow(c.live.vol_change)} ${fmt(c.live.vol_change)}%
-        <div class="muted">${esc(c.live.updated_at)}</div>
-      </div>` : ""}
-    <div class="inputs">
-      <label>Quantity
-        <input type="number" class="in-qty" value="${c.quantity}" step="any"></label>
-      <label>Price
-        <input type="number" class="in-price" value="${c.price}" step="any"></label>
-    </div>
-    <div class="pl-row num">
-      Cost <b>${fmt(c.quantity * c.price)}</b> ·
-      CMP <b>${fmt(c.quantity * c.current_cmp)}</b> ·
-      P&amp;L <b class="${cls(pl)}">${fmt(pl)} (${fmt(plPct)}%)</b>
-    </div>
-    <div class="card-foot">
-      <label><input type="checkbox" class="sel" ${c.selected ? "checked" : ""}>
-        Select</label>
-      <span class="reco">${c.recommendation
-        ? `${c.recommendation.action}
-           <span class="muted">(${fmt(c.recommendation.confidence)})</span>`
-        : ""}</span>
-      <div style="display: flex; gap: 4px;">
-        <button class="ai-btn" aria-label="Generate AI analysis for ${c.symbol}">AI ✦</button>
-        <button class="mc-btn">${mc.link ? "MC ↗" : "Set MC link"}</button>
-      </div>
-    </div>`;
-
-  card.querySelector(".rm").addEventListener("click", async () => {
-    await api("/stocks/" + encodeURIComponent(c.symbol), { method: "DELETE" });
-    loadBoard();
-  });
-  card.querySelector(".sel").addEventListener("change", (e) => {
-    api("/stocks/" + encodeURIComponent(c.symbol) + "/select", {
-      method: "POST",
-      body: JSON.stringify({ selected: e.target.checked }),
-    });
-  });
-  const pushInputs = async () => {
-    const r = await api("/stocks/" + encodeURIComponent(c.symbol) + "/inputs", {
-      method: "POST",
-      body: JSON.stringify({
-        quantity: card.querySelector(".in-qty").value || 0,
-        price: card.querySelector(".in-price").value || 0,
-      }),
-    });
-    if (r.ok) { renderPortfolio(r.portfolio); loadBoard(); }
-  };
-  card.querySelectorAll(".in-qty, .in-price").forEach((inp) => {
-    inp.addEventListener("change", pushInputs);
-  });
-  card.querySelector(".ai-btn").addEventListener("click", () => {
-    openAIAnalysisModal(c.symbol, false);
-  });
-
-  card.querySelector(".mc-btn").addEventListener("click", async () => {
-    if (mc.link) { window.open(mc.link, "_blank"); return; }
-    const link = prompt(
-      `Enter the Moneycontrol URL for ${c.symbol} (${c.security || ""}):`);
-    if (link === null) return;
-    const r = await api("/mc-link", {
-      method: "POST",
-      body: JSON.stringify({ symbol: c.symbol, security: c.security, link }),
-    });
-    if (r.ok) { toast(r.message, "ok"); loadBoard(); }
-  });
-
-  if (c.chart) drawChart(card.querySelector("canvas"), c);
-  return card;
-}
-
-function drawChart(canvas, c) {
-  const ch = c.chart;
-  const datasets = [{
-    label: "Close", data: ch.prices, borderColor: "#3b82c4",
-    borderWidth: 1.6, pointRadius: 0, tension: 0.1, yAxisID: "y",
-  }];
-  const dmaColors = { 9: "#8e44ad", 20: "#d29922", 50: "#16a085", 200: "#e07b70" };
-  for (const w of [9, 20, 50, 200]) {
-    const d = ch["dma_" + w];
-    if (d) datasets.push({
-      label: w + "DMA", data: d, borderColor: dmaColors[w],
-      borderWidth: 1, pointRadius: 0, tension: 0.1, yAxisID: "y",
-    });
-  }
-  if (ch.volumes && ch.volumes.length) {
-    datasets.push({
-      label: "Volume", data: ch.volumes, type: "bar",
-      backgroundColor: "rgba(125,136,148,.35)", yAxisID: "vol",
-    });
-  }
-  state.charts[c.symbol] = new Chart(canvas, {
-    type: "line",
-    data: { labels: ch.dates, datasets },
-    options: {
-      responsive: true, maintainAspectRatio: true, aspectRatio: 2.2, animation: false,
-      plugins: { legend: { display: false } },
-      scales: {
-        x: { ticks: { maxTicksLimit: 5, color: "#7d8894" }, grid: { display: false } },
-        y: { ticks: { color: "#7d8894" }, grid: { color: "#2a333f" } },
-        vol: { position: "right", display: false },
-      },
-    },
-  });
-}
-
-/* --------------------------------------------------------------- actions */
-
-async function loadWatchlist(n) {
-  const r = await api("/watchlists/load/" + n, { method: "POST" });
-  if (r.ok) toast(`Loaded ${r.loaded} stocks from '${r.name}'.`, "ok");
-  loadBoard();
-}
-
-function pickWatchlistFile() {
-  const inp = document.createElement("input");
-  inp.type = "file";
-  inp.accept = ".json";
-  inp.addEventListener("change", async () => {
-    if (!inp.files.length) return;
-    const fd = new FormData();
-    fd.append("file", inp.files[0]);
-    busy("Loading watchlist file…");
-    const res = await fetch("/api/watchlists/load-file",
-      { method: "POST", body: fd });
-    const body = await res.json();
-    unbusy();
-    toast(body.message || "Done", body.ok ? "ok" : "err");
-    loadBoard();
-  });
-  inp.click();
-}
-
-async function loadKnownFile(name) {
-  busy("Loading live analysis file…");
-  try {
-    const body = await api("/watchlists/load-file", {
-      method: "POST",
-      body: JSON.stringify({ known: name }),
-    });
-    unbusy();
-    if (body) {
-      toast(body.message || "Done", body.ok ? "ok" : "err");
-    }
-    loadBoard();
-  } catch (e) {
-    unbusy();
-    toast(String(e), "err");
-  }
-}
-
-async function priceAnalysis(duration, changeType) {
-  const t = prompt(
-    `Enter the minimum % change threshold for ${duration} ${changeType}:`,
-    changeType === "Max" ? "5" : "-5");
-  if (t === null) return;
-  const r = await runJob(api("/analysis/price", {
-    method: "POST",
-    body: JSON.stringify({ duration, change_type: changeType, threshold: t }),
-  }), `Calculating ${duration} ${changeType} price changes…`);
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-}
-
-async function volumeAnalysis(duration) {
-  const r = await runJob(api("/analysis/volume", {
-    method: "POST", body: JSON.stringify({ duration }),
-  }), `Calculating ${duration} volume movers…`);
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-}
-
-async function returnAnalysis(duration) {
-  const r = await runJob(api("/analysis/return", {
-    method: "POST", body: JSON.stringify({ duration }),
-  }), `Calculating ${duration} max returns…`);
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-}
-
-async function topScore() {
-  const r = await runJob(api("/analysis/top-score", { method: "POST" }),
-    "Loading top Moneycontrol scores…");
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-}
-
-async function recommend() {
-  const r = await runJob(api("/recommend", { method: "POST" }),
-    "Running buy/sell recommender…");
-  if (r.recommendations) {
-    const top = r.recommendations.slice(0, 10)
-      .map((x) => `${x.symbol}: ${x.action} (${fmt(x.confidence)})`)
-      .join("\n");
-    toast("Recommendations updated.\n" + top, "ok");
-  }
-  loadBoard();
-}
-
-async function trainModels() {
-  if (!confirm("Train LSTM models for ALL symbols? This can take a long time."))
-    return;
-  const r = await runJob(api("/train", { method: "POST" }),
-    "Training LSTM models…");
-  if (r.ok) toast(`Training complete for ${r.trained_symbols} symbols.`, "ok");
-  loadBoard();
-}
-
-async function setVolumeCutoff() {
-  const v = prompt("Minimum average volume for volume analysis:",
-    state.board ? state.board.volume_cutoff : 10000);
-  if (v === null) return;
-  const r = await api("/config", {
-    method: "POST", body: JSON.stringify({ volume_cutoff: v }),
-  });
-  if (r.ok) { toast(`Volume cutoff set to ${fmt(r.volume_cutoff, 0)}.`, "ok"); loadBoard(); }
-}
-
-/* Scripts terminal */
-let termRun = null, termTimer = null, termLast = -1;
-
-async function startScript(s) {
-  const r = await api("/scripts/start", {
-    method: "POST", body: JSON.stringify({ file: s.file }),
-  });
-  if (!r.ok) return;
-  termRun = r.run_id;
-  termLast = -1;
-  $("#terminal-title").textContent = s.label + " — " + s.file;
-  $("#terminal-out").innerHTML = "";
-  $("#modal-terminal").classList.remove("hidden");
-  termTimer = setInterval(pollTerminal, 800);
-}
-
-async function pollTerminal() {
-  if (!termRun) return;
-  const r = await api(`/scripts/${termRun}/output?after=${termLast}`);
-  if (!r.ok) return;
-  const out = $("#terminal-out");
-  for (const ln of r.lines) {
-    termLast = ln.n;
-    const span = document.createElement("span");
-    span.className = ln.tag;
-    span.textContent = ln.text + "\n";
-    out.appendChild(span);
-  }
-  if (r.lines.length) out.scrollTop = out.scrollHeight;
-  if (r.finished && termTimer) { clearInterval(termTimer); termTimer = null; }
-}
-
-$("#terminal-send").addEventListener("click", sendTerminal);
-$("#terminal-in").addEventListener("keydown", (e) => {
-  if (e.key === "Enter") sendTerminal();
-});
-async function sendTerminal() {
-  const text = $("#terminal-in").value;
-  if (!text.trim() || !termRun) return;
-  await api(`/scripts/${termRun}/input`, {
-    method: "POST", body: JSON.stringify({ text }),
-  });
-  $("#terminal-in").value = "";
-}
-$("#terminal-close").addEventListener("click", () => {
-  $("#modal-terminal").classList.add("hidden");
-  if (termTimer) clearInterval(termTimer);
-  termTimer = null;
-});
-
-/* ------------------------------------------------------------- toolbar */
-
-$("#btn-add").addEventListener("click", async () => {
-  const key = $("#stock-search").value.trim();
-  if (!key) return;
-  const r = await api("/stocks", {
-    method: "POST", body: JSON.stringify({ key }),
-  });
-  if (r.ok) { $("#stock-search").value = ""; loadBoard(); }
-});
-$("#stock-search").addEventListener("keydown", (e) => {
-  if (e.key === "Enter") $("#btn-add").click();
-});
-
-$("#btn-sensex").addEventListener("click", async () => {
-  await api("/index/SENSEX", { method: "POST" }); loadBoard();
-});
-$("#btn-nifty").addEventListener("click", async () => {
-  await api("/index/NIFTY", { method: "POST" }); loadBoard();
-});
-
-$("#btn-portfolio").addEventListener("click", () => {
-  const inp = document.createElement("input");
-  inp.type = "file";
-  inp.accept = ".csv";
-  inp.addEventListener("change", async () => {
-    if (!inp.files.length) return;
-    const fd = new FormData();
-    fd.append("file", inp.files[0]);
-    const token = localStorage.getItem("session_token") || "";
-    const startPromise = fetch("/api/portfolio/import", {
-      method: "POST",
-      headers: { "X-Session-Token": token },
-      body: fd
-    }).then(res => res.json());
-
-    const r = await runJob(startPromise, "Importing portfolio…");
-    if (r.message) toast(r.message, r.ok ? "ok" : "err");
-    loadBoard();
-  });
-  inp.click();
-});
-
-$("#btn-export").addEventListener("click", async () => {
-  const r = await api("/export", { method: "POST" });
-  if (r.ok) {
-    toast(r.message, "ok");
-    window.location = "/api/export/download?file=" + encodeURIComponent(r.file);
-  }
-});
-
-$("#date-input").addEventListener("change", async (e) => {
-  const r = await api("/date", {
-    method: "POST", body: JSON.stringify({ date: e.target.value }),
-  });
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-});
-
-/* Live toggle + modal */
-$("#live-toggle").addEventListener("change", (e) => {
-  if (e.target.checked) {
-    $("#modal-live").classList.remove("hidden");
-  } else {
-    stopLive();
-  }
-});
-$("#live-cancel").addEventListener("click", () => {
-  $("#modal-live").classList.add("hidden");
-  $("#live-toggle").checked = false;
-});
-async function fetchLiveDataInBackground() {
-  toast("Fetching live data in background…", "info");
-  try {
-    const start = await api("/live/refresh", { method: "POST" });
-    if (!start || !start.job_id) {
-      if (start && start.message) toast(start.message, "err");
-      return;
-    }
-    (async () => {
-      const startTime = Date.now();
-      while (Date.now() - startTime < 60000) {
-        await new Promise((r) => setTimeout(r, 1000));
-        const job = await api("/jobs/" + start.job_id);
-        if (job.status === "done") {
-          const res = job.result || {};
-          toast(res.message || `Live data updated (${res.updated || 0} stocks).`, "ok");
-          loadBoard();
-          break;
-        }
-        if (job.status === "error") {
-          toast(job.error || "Live data fetch failed.", "err");
-          break;
-        }
-      }
-    })();
-  } catch (e) {
-    toast(String(e), "err");
-  }
-}
-
-$("#live-ok").addEventListener("click", async () => {
-  $("#modal-live").classList.add("hidden");
-  const r = await api("/live/start", {
-    method: "POST",
-    body: JSON.stringify({
-      source: $("#live-source").value,
-      refresh_rate: $("#live-rate").value,
-    }),
-  });
-  if (!r.ok) { $("#live-toggle").checked = false; return; }
-  if (r.warning) toast(r.warning);
-  fetchLiveDataInBackground();
-  // Poll the board while live so scheduler updates reach the UI.
-  if (state.livePoll) clearInterval(state.livePoll);
-  state.livePoll = setInterval(loadBoard, 10000);
-});
-
-async function stopLive() {
-  if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; }
-  await api("/live/stop", { method: "POST" });
-  loadBoard();
-}
-
-/* Bottom controls */
-$("#btn-refresh").addEventListener("click", async () => {
-  const r = await runJob(api("/refresh", { method: "POST" }),
-    "Refreshing dashboard data…");
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-});
-
-$("#btn-save").addEventListener("click", async () => {
-  const r = await api("/watchlists/save", { method: "POST" });
-  if (r.needs_save_as) { openSaveAs(); return; }
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-});
-
-function openSaveAs() {
-  $("#saveas-name").value = state.board
-    ? state.board.banner.watchlist.replace(/\s*\*$/, "") : "";
-  $("#modal-saveas").classList.remove("hidden");
-}
-$("#btn-saveas").addEventListener("click", openSaveAs);
-$("#saveas-cancel").addEventListener("click", () =>
-  $("#modal-saveas").classList.add("hidden"));
-$("#saveas-ok").addEventListener("click", async () => {
-  $("#modal-saveas").classList.add("hidden");
-  const r = await api("/watchlists/save-as", {
-    method: "POST",
-    body: JSON.stringify({
-      slot: Number($("#saveas-slot").value),
-      name: $("#saveas-name").value,
-    }),
-  });
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-});
-
-$("#btn-add-selected").addEventListener("click", async () => {
-  const slot = prompt("Add selected stocks to watchlist slot (1–20):", "1");
-  if (slot === null) return;
-  const r = await api("/watchlists/add-selected", {
-    method: "POST", body: JSON.stringify({ slot: Number(slot) }),
-  });
-  if (r.message) toast(r.message, r.ok ? "ok" : "err");
-  loadBoard();
-});
-
-$("#btn-delete-all").addEventListener("click", async () => {
-  if (!confirm("Remove all stocks from the dashboard?")) return;
-  await api("/stocks", { method: "DELETE" });
-  loadBoard();
-});
-
-function showShutdownScreen() {
-  if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; }
-  if (state.heartbeat) { clearInterval(state.heartbeat); state.heartbeat = null; }
-  const screen = $("#shutdown-screen");
-  if (screen) {
-    screen.classList.add("visible");
-  }
-}
-
-function updateEnvPill(isHf) {
-  const pill = $("#env-pill");
-  if (!pill) return;
-  if (isHf) {
-    pill.textContent = "HuggingFace Spaces";
-    pill.className = "env-pill hf";
-  } else {
-    pill.textContent = "Local";
-    pill.className = "env-pill local";
-  }
-}
-
-function startHeartbeat() {
-  state.consecutiveFailures = 0;
-  state.heartbeat = setInterval(async () => {
-    try {
-      const res = await fetch("/api/status");
-      if (!res.ok) {
-        state.consecutiveFailures++;
-        if (state.consecutiveFailures >= 3) {
-          showShutdownScreen();
-        }
-        return;
-      }
-      const body = await res.json().catch(() => ({}));
-      if (body.exiting) {
-        showShutdownScreen();
-      }
-      updateEnvPill(body.is_hf);
-      state.consecutiveFailures = 0;
-    } catch (err) {
-      state.consecutiveFailures++;
-      if (state.consecutiveFailures >= 3) {
-        showShutdownScreen();
-      }
-    }
-  }, 1000);
-}
-
-async function triggerExit() {
-  if (!confirm("Are you sure you want to exit? This will shut down the server and stop the application.")) {
-    return;
-  }
-  showShutdownScreen();
-  try {
-    await fetch("/api/shutdown", { method: "POST" });
-  } catch (err) {
-    console.error("Shutdown initiated (fetch failed/aborted as server terminated).", err);
-  }
-}
-
-/* Login, TOTP, and Admin Control Panel helper functions */
-
-function showLoginBox() {
-  const splash = $("#splash-screen");
-  if (splash) {
-    splash.classList.remove("hidden");
-  }
-  const splashStatus = $("#splash-status");
-  const loader = $(".splash-loader-container");
-  const loginBox = $("#login-box");
-  const totpBox = $("#totp-box");
-
-  if (loader) loader.classList.add("hidden");
-  if (splashStatus) splashStatus.classList.add("hidden");
-  if (totpBox) totpBox.classList.add("hidden");
-  if (loginBox) loginBox.classList.remove("hidden");
-}
-
-function showTotpBox(userid, qrUrl = null, secretKey = null) {
-  const loginBox = $("#login-box");
-  const totpBox = $("#totp-box");
-  const qrContainer = $("#totp-qr-container");
-  const qrImage = $("#totp-qr-image");
-  const secretKeyText = $("#totp-secret-key");
-
-  if (loginBox) loginBox.classList.add("hidden");
-  if (totpBox) totpBox.classList.remove("hidden");
-
-  if (secretKey && secretKeyText) {
-    secretKeyText.textContent = secretKey;
-    if (qrContainer) qrContainer.classList.remove("hidden");
-    
-    if (qrUrl && qrImage) {
-      qrImage.src = qrUrl;
-      qrImage.style.display = "block";
-    } else if (qrImage) {
-      qrImage.style.display = "none";
-    }
-  } else if (qrContainer) {
-    qrContainer.classList.add("hidden");
-  }
-}
-
-let currentAdminActionUser = null;
-
-async function openAdminPanel() {
-  $("#modal-admin").classList.remove("hidden");
-  await refreshAdminDashboard();
-}
-
-async function refreshAdminDashboard() {
-  const res = await api("/admin/dashboard");
-  if (!res.ok) return;
-
-  $("#stat-users").textContent = res.total_users;
-  $("#stat-symbols").textContent = res.total_symbols;
-  $("#stat-yfinance").textContent = res.yfinance_connected ? "Online" : "Offline";
-  $("#stat-yfinance").style.color = res.yfinance_connected ? "var(--gain)" : "var(--loss)";
-
-  const userList = $("#admin-user-list");
-  userList.innerHTML = "";
-
-  Object.values(res.users).forEach((user) => {
-    const isOnline = user.login_status === "online";
-    const isActive = user.active;
-    const isUserAdmin = user.is_admin;
-
-    const tr = document.createElement("tr");
-    tr.style.borderBottom = "1px solid rgba(255, 255, 255, 0.08)";
-    tr.innerHTML = `
-      <td style="padding: 6px; font-weight: bold; color: #fff;">${esc(user.userid)}</td>
-      <td style="padding: 6px;"><span class="chip" style="background: ${isOnline ? "rgba(39, 174, 96, 0.2)" : "rgba(127, 140, 141, 0.2)"}; border-color: ${isOnline ? "var(--gain)" : "var(--border)"}; color: ${isOnline ? "var(--gain)" : "var(--muted)"}; padding: 1px 6px; font-size: 10px;">${isOnline ? "Online" : "Offline"}</span></td>
-      <td style="padding: 6px; color: ${isUserAdmin ? "#a78bfa" : "#94a3b8"};">${isUserAdmin ? "Admin" : "User"}</td>
-      <td style="padding: 6px;">${user.login_count}</td>
-      <td style="padding: 6px; color: #64748b; font-size: 11px;">${user.last_login ? user.last_login.substring(0, 16).replace("T", " ") : "Never"}</td>
-      <td style="padding: 6px;"><span class="chip" style="background: ${isActive ? "rgba(39, 174, 96, 0.1)" : "rgba(192, 57, 43, 0.15)"}; color: ${isActive ? "var(--gain)" : "var(--loss)"}; border-color: ${isActive ? "rgba(39, 174, 96, 0.3)" : "rgba(192, 57, 43, 0.3)"}; padding: 1px 6px; font-size: 10px;">${isActive ? "Active" : "Locked"}</span></td>
-      <td style="padding: 6px; text-align: right; white-space: nowrap;">
-        <button class="btn-action-qr primary btn-xs" title="Show Authenticator QR" style="padding: 2px 6px; font-size: 11px; margin-right: 3px;">📱 QR</button>
-        <button class="btn-action-metrics primary btn-xs" title="View Metrics" style="padding: 2px 6px; font-size: 11px; margin-right: 3px;">📊 Stats</button>
-        <button class="btn-action-reset primary btn-xs" title="Full Account Reset" style="padding: 2px 6px; font-size: 11px; margin-right: 3px; background: var(--loss); border-color: var(--loss);">🔄 Full Reset</button>
-        <button class="btn-action-lock primary btn-xs" title="Lock/Unlock User" style="padding: 2px 6px; font-size: 11px; margin-right: 3px; background: ${isActive ? "var(--loss)" : "var(--gain)"}; border-color: ${isActive ? "var(--loss)" : "var(--gain)"};">${isActive ? "🔒 Lock" : "🔓 Unlock"}</button>
-        <button class="btn-action-mirror primary btn-xs" title="Mirror Activity Logs" style="padding: 2px 6px; font-size: 11px; margin-right: 3px;">🔍 Logs</button>
-        <button class="btn-action-chat primary btn-xs" style="padding: 2px 6px; font-size: 11px; margin-right: 3px;">💬 Chat</button>
-        <button class="btn-action-delete danger btn-xs" title="Delete User" style="padding: 2px 6px; font-size: 11px;">🗑️ Delete</button>
-      </td>
-    `;
-
-    // Wire actions
-    tr.querySelector(".btn-action-qr").onclick = () => runAdminAction("qr", user.userid);
-    tr.querySelector(".btn-action-metrics").onclick = () => runAdminAction("metrics", user.userid);
-    tr.querySelector(".btn-action-reset").onclick = () => {
-      if (confirm(`Are you sure you want to perform a full account reset (password + TOTP) for ${user.userid}? This will log them out immediately.`)) {
-        runAdminAction("reset", user.userid);
-      }
-    };
-    tr.querySelector(".btn-action-lock").onclick = () => runAdminAction("lock", user.userid);
-    tr.querySelector(".btn-action-mirror").onclick = () => runAdminAction("mirror", user.userid);
-    tr.querySelector(".btn-action-chat").onclick = () => runAdminAction("chat", user.userid);
-    tr.querySelector(".btn-action-delete").onclick = () => {
-      if (confirm(`Are you sure you want to PERMANENTLY delete user ${user.userid}?`)) {
-        runAdminAction("delete", user.userid);
-      }
-    };
-
-    userList.appendChild(tr);
-  });
-}
-
-function hideAllAdminSidebarPanels() {
-  document.querySelectorAll(".sidebar-panel").forEach((p) => p.classList.add("hidden"));
-  const grid = document.querySelector(".admin-main-grid");
-  if (grid) grid.classList.remove("has-sidebar");
-}
-
-function showAdminSidebarPanel(panelId) {
-  hideAllAdminSidebarPanels();
-  const p = document.getElementById(panelId);
-  if (p) p.classList.remove("hidden");
-  const grid = document.querySelector(".admin-main-grid");
-  if (grid) grid.classList.add("has-sidebar");
-}
-
-async function runAdminAction(action, target_uid) {
-  currentAdminActionUser = target_uid;
-  hideAllAdminSidebarPanels();
-  
-  if (action === "qr") {
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action, target_uid })
-    });
-    if (res.ok) {
-      if (res.qr) {
-        $("#admin-qr-display").innerHTML = `<img src="${res.qr}" style="max-width: 140px;"/>`;
-      } else {
-        $("#admin-qr-display").innerHTML = `<div style="padding: 10px; color: #94a3b8; font-size: 12px; border: 1px dashed #475569; border-radius: 4px; text-align: center;">Pending first login by user.<br/>(QR code will be provisioned then)</div>`;
-      }
-      $("#admin-secret-key").textContent = res.secret;
-      showAdminSidebarPanel("panel-qr");
-    }
-  } else if (action === "metrics") {
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action, target_uid })
-    });
-    if (res.ok) {
-      $("#admin-metrics-display").innerHTML = `
-        <div style="margin-bottom: 4px;"><strong>User ID:</strong> ${res.userid}</div>
-        <div style="margin-bottom: 4px;"><strong>Login Count:</strong> ${res.login_count}</div>
-        <div style="margin-bottom: 4px;"><strong>Last Login:</strong> ${res.last_login ? res.last_login.substring(0, 16).replace("T", " ") : "Never"}</div>
-        <div style="margin-bottom: 4px;"><strong>Symbols in State:</strong> ${res.symbols_count}</div>
-        <div style="margin-bottom: 4px;"><strong>Watchlists Created:</strong> ${res.watchlists_count}</div>
-      `;
-      showAdminSidebarPanel("panel-metrics");
-    }
-  } else if (action === "reset" || action === "lock" || action === "delete") {
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action, target_uid })
-    });
-    if (res.ok) {
-      toast(res.message, "ok");
-      await refreshAdminDashboard();
-    }
-  } else if (action === "mirror") {
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action, target_uid })
-    });
-    if (res.ok) {
-      const logs = res.logs || [];
-      const logText = logs.map((l) => `[${l.timestamp.substring(11, 19)}] ${l.action.toUpperCase()}: ${l.status} (${l.details || ""})`).join("\n");
-      $("#admin-mirror-display").textContent = logText || "No activities logged yet.";
-      showAdminSidebarPanel("panel-mirror");
-    }
-  } else if (action === "chat") {
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action, target_uid })
-    });
-    if (res.ok) {
-      $("#chat-user-title").textContent = `Chat with ${target_uid}`;
-      const chatDisplay = $("#admin-chat-display");
-      chatDisplay.innerHTML = "";
-      res.messages.forEach((m) => {
-        const div = document.createElement("div");
-        div.className = "chat-msg " + m.sender;
-        div.innerHTML = `<span>${esc(m.text)}</span>`;
-        chatDisplay.appendChild(div);
-      });
-      showAdminSidebarPanel("panel-chat");
-    }
-  }
-}
-
-/* boot */
-document.addEventListener("DOMContentLoaded", async () => {
-  const btnLoginSubmit = $("#btn-login-submit");
-  const btnTotpSubmit = $("#btn-totp-submit");
-  const loginUsernameIn = $("#login-username");
-  const loginPasswordIn = $("#login-password");
-  const totpIn = $("#totp-input");
-  const loginError = $("#login-error");
-  const totpError = $("#totp-error");
-  const splash = $("#splash-screen");
-
-  let authUsername = null;
-
-  // Handle Login submission
-  if (btnLoginSubmit) {
-    btnLoginSubmit.addEventListener("click", async () => {
-      loginError.classList.add("hidden");
-      const userid = loginUsernameIn.value.trim();
-      const password = loginPasswordIn.value;
-      if (!userid || !password) return;
-
-      busy("Verifying credentials...");
-      try {
-        const res = await fetch("/api/auth/login", {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ userid, password })
-        });
-        const data = await res.json();
-        unbusy();
-
-        if (res.ok && data.ok) {
-          authUsername = data.userid;
-          if (data.totp_required) {
-            if (data.message === "first_login" || data.message === "totp_setup") {
-              showTotpBox(data.userid, data.qr, data.secret);
-            } else {
-              showTotpBox(data.userid);
-            }
-          }
-        } else {
-          loginError.textContent = data.message || "Invalid credentials.";
-          loginError.classList.remove("hidden");
-        }
-      } catch (err) {
-        unbusy();
-        console.error("Login failed:", err);
-        loginError.textContent = "Login request failed: " + err.message;
-        loginError.classList.remove("hidden");
-      }
-    });
-  }
-
-  // Handle TOTP submission
-  if (btnTotpSubmit) {
-    btnTotpSubmit.addEventListener("click", async () => {
-      totpError.classList.add("hidden");
-      const code = totpIn.value.trim();
-      if (!code) return;
-
-      busy("Verifying TOTP code...");
-      try {
-        const res = await fetch("/api/auth/totp", {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ userid: authUsername, code: code })
-        });
-        const data = await res.json();
-        unbusy();
-
-        if (res.ok && data.ok) {
-          localStorage.setItem("session_token", data.session_token);
-          
-          const splashStatus = $("#splash-status");
-          if (splashStatus) {
-            splashStatus.classList.remove("hidden");
-            splashStatus.textContent = "Loading market data...";
-          }
-          const loader = $(".splash-loader-container");
-          if (loader) loader.classList.remove("hidden");
-          $("#totp-box").classList.add("hidden");
-
-          // Successfully authenticated! Run check
-          await runBootSequence();
-        } else {
-          totpError.textContent = data.message || "Incorrect code.";
-          totpError.classList.remove("hidden");
-          totpIn.value = "";
-        }
-      } catch (err) {
-        unbusy();
-        console.error("TOTP verification failed:", err);
-      }
-    });
-  }
-
-  // Keyboard support for Enter key
-  if (loginUsernameIn) {
-    [loginUsernameIn, loginPasswordIn].forEach((el) => {
-      el.addEventListener("keydown", (e) => {
-        if (e.key === "Enter") btnLoginSubmit.click();
-      });
-    });
-  }
-  if (totpIn) {
-    totpIn.addEventListener("keydown", (e) => {
-      if (e.key === "Enter") btnTotpSubmit.click();
-    });
-  }
-
-  // Admin Panel Event Handlers
-  $("#admin-close").onclick = () => $("#modal-admin").classList.add("hidden");
-  $("#ai-close").onclick = () => $("#modal-ai").classList.add("hidden");
-  
-  const aiRefreshBtn = $("#ai-refresh-btn");
-  if (aiRefreshBtn) {
-    aiRefreshBtn.onclick = () => {
-      if (currentAISymbol) {
-        openAIAnalysisModal(currentAISymbol, true);
-      }
-    };
-  }
-
-  const aiCopyBtn = $("#ai-copy-btn");
-  if (aiCopyBtn) {
-    aiCopyBtn.onclick = () => {
-      if (!currentAIAnalysisText) {
-        toast("No report content to copy", "err");
-        return;
-      }
-      navigator.clipboard.writeText(currentAIAnalysisText).then(() => {
-        toast("AI Research Report copied to clipboard! 📋", "ok");
-      }).catch((err) => {
-        toast("Failed to copy report: " + err, "err");
-      });
-    };
-  }
-  $("#admin-btn-show-add").onclick = () => {
-    showAdminSidebarPanel("panel-add-user");
-  };
-  $("#admin-btn-create-submit").onclick = async () => {
-    const username = $("#add-username").value.trim();
-    const role = $("#add-role").value;
-    if (!username) return;
-
-    const res = await api("/admin/action", {
-      method: "POST",
-      body: JSON.stringify({ action: "add_user", username, role })
-    });
-    if (res.ok) {
-      toast(res.message, "ok");
-      $("#add-username").value = "";
-      hideAllAdminSidebarPanels();
-      await refreshAdminDashboard();
-    }
-  };
-
-  // Fake chat send button support
-  $("#chat-send-btn").onclick = () => {
-    const chatInput = $("#chat-input");
-    const text = chatInput.value.trim();
-    if (!text || !currentAdminActionUser) return;
-
-    const chatDisplay = $("#admin-chat-display");
-    const div = document.createElement("div");
-    div.className = "chat-msg admin";
-    div.innerHTML = `<span><strong>Admin:</strong> ${esc(text)}</span>`;
-    chatDisplay.appendChild(div);
-    chatInput.value = "";
-    chatDisplay.scrollTop = chatDisplay.scrollHeight;
-
-    // Simulated reply
-    setTimeout(() => {
-      const reply = document.createElement("div");
-      reply.className = "chat-msg user";
-      reply.innerHTML = `<span><strong>${esc(currentAdminActionUser)}</strong>: Thanks admin! Got it.</span>`;
-      chatDisplay.appendChild(reply);
-      chatDisplay.scrollTop = chatDisplay.scrollHeight;
-    }, 1000);
-  };
-
-  $("#chat-input").addEventListener("keydown", (e) => {
-    if (e.key === "Enter") $("#chat-send-btn").click();
-  });
-
-  // Settings -> Change Password Event Handlers
-  const btnSettingsChangePassword = $("#btn-settings-change-password");
-  if (btnSettingsChangePassword) {
-    btnSettingsChangePassword.addEventListener("click", () => {
-      closeMenus();
-      $("#cp-current").value = "";
-      $("#cp-new").value = "";
-      $("#cp-confirm").value = "";
-      $("#cp-error").classList.add("hidden");
-      $("#modal-change-password").classList.remove("hidden");
-    });
-  }
-
-  $("#cp-cancel").addEventListener("click", () => {
-    $("#modal-change-password").classList.add("hidden");
-  });
-
-  $("#cp-submit").addEventListener("click", async () => {
-    const current_password = $("#cp-current").value;
-    const new_password = $("#cp-new").value;
-    const confirm_password = $("#cp-confirm").value;
-    const cpError = $("#cp-error");
-
-    if (new_password !== confirm_password) {
-      cpError.textContent = "New password and confirm password do not match.";
-      cpError.classList.remove("hidden");
-      return;
-    }
-
-    try {
-      const res = await api("/auth/change-password", {
-        method: "POST",
-        body: JSON.stringify({ current_password, new_password, confirm_password })
-      });
-      if (res.ok) {
-        toast(res.message || "Password updated successfully!", "ok");
-        $("#modal-change-password").classList.add("hidden");
-      } else {
-        cpError.textContent = res.message || "Failed to update password.";
-        cpError.classList.remove("hidden");
-      }
-    } catch (err) {
-      cpError.textContent = "Error changing password: " + err;
-      cpError.classList.remove("hidden");
-    }
-  });
-
-  // Mandatory Policy Password Change Submission
-  $("#force-cp-submit").addEventListener("click", async () => {
-    const new_password = $("#force-cp-new").value;
-    const confirm_password = $("#force-cp-confirm").value;
-    const forceCpError = $("#force-cp-error");
-
-    if (new_password !== confirm_password) {
-      forceCpError.textContent = "New password and confirm password do not match.";
-      forceCpError.classList.remove("hidden");
-      return;
-    }
-
-    try {
-      const res = await api("/auth/change-password", {
-        method: "POST",
-        body: JSON.stringify({ new_password, confirm_password })
-      });
-      if (res.ok) {
-        toast("Compliant password saved successfully!", "ok");
-        $("#modal-force-password-change").classList.add("hidden");
-        await runBootSequence();
-      } else {
-        forceCpError.textContent = res.message || "Failed to update password.";
-        forceCpError.classList.remove("hidden");
-      }
-    } catch (err) {
-      forceCpError.textContent = "Error saving password: " + err;
-      forceCpError.classList.remove("hidden");
-    }
-  });
-
-  async function runBootSequence() {
-    const res = await fetch("/api/auth/status", {
-      headers: { "X-Session-Token": localStorage.getItem("session_token") || "" }
-    });
-    const status = await res.json();
-
-    const splashStatus = $("#splash-status");
-    if (status.nvidia_status) {
-      if (status.nvidia_status.success) {
-        if (splashStatus) splashStatus.textContent = "System Initialized — NVIDIA API connection tested and successful ✔";
-        toast(`NVIDIA API connection tested & successful (${status.nvidia_status.source || 'Keyring'})`, "ok");
-      } else {
-        if (splashStatus) splashStatus.textContent = "System Initialized — " + (status.nvidia_status.message || "NVIDIA API Key warning");
-      }
-    }
-
-    if (status.authenticated) {
-      $("#display-username").textContent = status.userid;
-      if (status.is_admin) {
-        $("#menu-utilities-container").classList.remove("hidden");
-        loadUtilitiesMenu();
-      } else {
-        $("#menu-utilities-container").classList.add("hidden");
-      }
-
-      if (status.force_password_change) {
-        $("#modal-force-password-change").classList.remove("hidden");
-        return;
-      }
-
-      await loadBoard();
-      startHeartbeat();
-
-      if (splash) {
-        splash.classList.add("fade-out");
-        setTimeout(() => {
-          splash.classList.add("hidden");
-          splash.classList.remove("fade-out");
-        }, 600);
-      }
-    } else {
-      showLoginBox();
-      if (status.reason === "inactivity") {
-        toast("Session terminated due to inactivity.", "err");
-        const loginError = $("#login-error");
-        if (loginError) {
-          loginError.textContent = "Session terminated due to inactivity.";
-          loginError.classList.remove("hidden");
-        }
-      }
-    }
-  }
-
-  // Handle Logout button click
-  const btnLogout = $("#btn-logout");
-  if (btnLogout) {
-    btnLogout.addEventListener("click", async () => {
-      busy("Logging out...");
-      try {
-        await fetch("/api/auth/logout", {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "X-Session-Token": localStorage.getItem("session_token") || ""
-          }
-        });
-      } catch (err) {
-        console.error("Logout request failed:", err);
-      }
-      localStorage.removeItem("session_token");
-      
-      // Stop live refresh heartbeat if active
-      if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; }
-      if (state.heartbeat) { clearInterval(state.heartbeat); state.heartbeat = null; }
-      
-      authUsername = null;
-      $("#login-username").value = "";
-      $("#login-password").value = "";
-      $("#totp-input").value = "";
-      unbusy();
-      showLoginBox();
-    });
-  }
-
-  // Start checking auth status on page load
-  runBootSequence();
-});
-
-

nsedash/web/static/style.css

@@ -1,924 +0,0 @@
-/* NSE Stock Dashboard — market-terminal styling.
-   Numeric-heavy layout: tabular figures, market-convention colours. */
-
-:root {
-  --bg: #0e1116;
-  --panel: #161b22;
-  --panel-2: #1c232d;
-  --border: #2a333f;
-  --text: #dbe2ea;
-  --muted: #94a3b8;
-  --gain: #4ade80;
-  --loss: #ff7675;
-  --flat: #a1a1aa;
-  --accent: #60a5fa;
-  --amber: #f59e0b;
-  --live: #1f7a3d;
-  --historic: #8a2f2a;
-}
-
-* { box-sizing: border-box; }
-html, body { margin: 0; }
-body {
-  background: var(--bg);
-  color: var(--text);
-  font: 13px/1.45 "Segoe UI", system-ui, -apple-system, sans-serif;
-}
-b, .num { font-variant-numeric: tabular-nums; }
-button {
-  background: var(--panel-2);
-  color: var(--text);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  padding: 5px 10px;
-  cursor: pointer;
-}
-button:hover { border-color: var(--accent); }
-button:focus-visible, input:focus-visible, select:focus-visible {
-  outline: 2px solid var(--accent);
-  outline-offset: 1px;
-}
-button.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
-button.danger { border-color: var(--loss); color: #e07b70; }
-input, select {
-  background: var(--panel);
-  color: var(--text);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  padding: 5px 8px;
-}
-.muted { color: var(--muted); }
-.hidden { display: none !important; }
-
-/* Banner */
-.banner {
-  display: flex;
-  justify-content: space-between;
-  padding: 6px 14px;
-  font-weight: 600;
-  color: #fff;
-  letter-spacing: .02em;
-}
-.banner.live { background: var(--live); }
-.banner.historic { background: var(--historic); }
-
-.env-pill {
-  display: inline-block;
-  padding: 2px 6px;
-  font-size: 10px;
-  font-weight: 700;
-  border-radius: 12px;
-  text-transform: uppercase;
-  letter-spacing: 0.05em;
-  line-height: 1;
-}
-.env-pill.local {
-  background: rgba(59, 130, 196, 0.25);
-  color: #5dade2;
-  border: 1px solid rgba(59, 130, 196, 0.4);
-}
-.env-pill.hf {
-  background: rgba(241, 196, 15, 0.25);
-  color: #f1c40f;
-  border: 1px solid rgba(241, 196, 15, 0.4);
-}
-
-/* Menu bar */
-.menubar {
-  display: flex;
-  gap: 2px;
-  background: var(--panel);
-  border-bottom: 1px solid var(--border);
-  padding: 0 8px;
-}
-.menu { position: relative; }
-.menu-btn {
-  border: none;
-  background: none;
-  border-radius: 0;
-  padding: 8px 12px;
-}
-.menu-btn:hover { background: var(--panel-2); }
-.menu-drop {
-  position: absolute;
-  top: 100%;
-  left: 0;
-  min-width: 230px;
-  max-height: 60vh;
-  overflow: auto;
-  background: var(--panel-2);
-  border: 1px solid var(--border);
-  border-radius: 0 0 6px 6px;
-  display: none;
-  z-index: 40;
-}
-.menu.open .menu-drop { display: block; }
-.menu-drop button {
-  display: block;
-  width: 100%;
-  text-align: left;
-  border: none;
-  border-radius: 0;
-  padding: 7px 12px;
-}
-.menu-drop button:hover { background: var(--accent); color: #fff; }
-.menu-drop .sep { border-top: 1px solid var(--border); margin: 4px 0; }
-.menu-drop .head {
-  padding: 6px 12px 2px;
-  font-size: 11px;
-  text-transform: uppercase;
-  color: var(--muted);
-}
-
-/* Toolbar */
-.toolbar {
-  display: flex;
-  align-items: center;
-  flex-wrap: wrap;
-  gap: 10px;
-  padding: 8px 12px;
-  background: var(--panel);
-  border-bottom: 1px solid var(--border);
-}
-.toggle { display: flex; align-items: center; gap: 6px; }
-.pf-summary {
-  display: flex;
-  gap: 16px;
-  padding: 4px 12px;
-  background: var(--panel-2);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  flex-wrap: wrap;
-}
-.pf-summary span { color: var(--muted); }
-.pf-summary b { color: var(--text); margin-left: 4px; }
-.pf-summary b.gain { color: var(--gain); }
-.pf-summary b.loss { color: var(--loss); }
-
-/* Live Index Tickers & RTC Clock */
-.index-tickers {
-  display: flex;
-  align-items: center;
-  gap: 8px;
-}
-.index-badge {
-  display: flex;
-  align-items: center;
-  gap: 6px;
-  background: var(--panel-2);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  padding: 3px 8px;
-  cursor: pointer;
-  transition: all 0.2s ease;
-}
-.index-badge:hover {
-  border-color: var(--accent);
-  background: rgba(59, 130, 196, 0.15);
-}
-.idx-name {
-  font-size: 11px;
-  font-weight: 700;
-  color: #94a3b8;
-  letter-spacing: 0.04em;
-}
-.idx-pill {
-  font-size: 11px;
-  font-weight: 700;
-  font-variant-numeric: tabular-nums;
-  padding: 2px 6px;
-  border-radius: 3px;
-  background: rgba(255, 255, 255, 0.08);
-}
-.idx-pill.positive {
-  color: var(--gain);
-  background: rgba(39, 174, 96, 0.18);
-  border: 1px solid rgba(39, 174, 96, 0.3);
-}
-.idx-pill.negative {
-  color: var(--loss);
-  background: rgba(224, 123, 112, 0.18);
-  border: 1px solid rgba(224, 123, 112, 0.3);
-}
-
-.date-rtc-wrap {
-  margin-left: auto;
-  display: flex;
-  align-items: center;
-  gap: 14px;
-}
-.rtc-clock {
-  display: flex;
-  align-items: center;
-  gap: 6px;
-  background: rgba(15, 23, 42, 0.7);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  padding: 4px 10px;
-  font-family: "JetBrains Mono", "Consolas", monospace;
-  font-size: 11px;
-  font-weight: 600;
-  color: #38bdf8;
-  box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.3);
-}
-.rtc-icon {
-  font-size: 12px;
-}
-
-.datewrap { margin-left: 0; display: flex; align-items: center; gap: 6px; }
-
-/* Add row */
-.addrow {
-  display: flex;
-  gap: 8px;
-  align-items: center;
-  padding: 8px 12px;
-}
-#stock-search { flex: 0 1 420px; }
-
-/* Grid */
-.grid {
-  display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(320px, 420px));
-  gap: 12px;
-  padding: 0 12px 60px;
-  justify-content: center;
-  max-width: 1680px;
-  margin: 0 auto;
-}
-
-/* Card */
-.card {
-  background: var(--panel);
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  padding: 10px 12px;
-  position: relative;
-}
-.card h4 { margin: 0; font-size: 14px; }
-.card .sec { font-size: 11px; color: var(--muted); }
-.card .rm {
-  position: absolute;
-  top: 8px;
-  right: 8px;
-  padding: 0 7px;
-  line-height: 20px;
-}
-.card .row {
-  display: flex;
-  justify-content: space-between;
-  gap: 8px;
-  margin: 2px 0;
-}
-.card .lbl { color: var(--muted); }
-.gain { color: var(--gain); }
-.loss { color: var(--loss); }
-.flat { color: var(--flat); }
-.amber { color: var(--amber); }
-.score-80 { color: #2ecc71; }
-.score-60 { color: #a3d977; }
-.score-40 { color: var(--amber); }
-.score-20 { color: #e67e22; }
-.score-0  { color: var(--loss); }
-.changes, .volchips {
-  display: flex;
-  gap: 6px;
-  flex-wrap: wrap;
-  margin: 4px 0;
-}
-.chip {
-  background: var(--panel-2);
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  padding: 2px 6px;
-  font-size: 11px;
-}
-.card canvas { width: 100%; height: auto; margin-top: 6px; }
-.live-row {
-  margin-top: 6px;
-  padding: 5px 8px;
-  background: var(--panel-2);
-  border-left: 3px solid var(--live);
-  border-radius: 4px;
-  font-size: 12px;
-}
-.inputs {
-  display: grid;
-  grid-template-columns: 1fr 1fr;
-  gap: 6px;
-  margin-top: 8px;
-}
-.inputs label { font-size: 11px; color: var(--muted); }
-.inputs input { width: 100%; }
-.pl-row { margin-top: 6px; font-size: 12px; }
-.card-foot {
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  margin-top: 8px;
-}
-.reco { font-size: 12px; }
-
-/* Bottom controls */
-.controls {
-  position: fixed;
-  bottom: 0;
-  left: 0;
-  right: 0;
-  display: flex;
-  gap: 8px;
-  padding: 8px 12px;
-  background: var(--panel);
-  border-top: 1px solid var(--border);
-  z-index: 30;
-}
-
-/* Overlays / modals */
-.overlay {
-  position: fixed;
-  inset: 0;
-  background: rgba(0,0,0,.55);
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  z-index: 60;
-}
-.modal {
-  background: var(--panel);
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  padding: 18px;
-  min-width: 320px;
-  display: flex;
-  flex-direction: column;
-  gap: 10px;
-}
-.modal h3 { margin: 0 0 4px; }
-.modal label { display: flex; flex-direction: column; gap: 4px; }
-.modal-actions { display: flex; justify-content: flex-end; gap: 8px; }
-.busy-box {
-  background: var(--panel);
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  padding: 22px 30px;
-  display: flex;
-  align-items: center;
-  gap: 14px;
-}
-.spinner {
-  width: 22px;
-  height: 22px;
-  border: 3px solid var(--border);
-  border-top-color: var(--accent);
-  border-radius: 50%;
-  animation: spin 0.9s linear infinite;
-}
-@keyframes spin { to { transform: rotate(360deg); } }
-@media (prefers-reduced-motion: reduce) {
-  .spinner { animation-duration: 2.5s; }
-}
-
-/* Terminal */
-.modal.terminal { width: min(860px, 92vw); }
-#terminal-out {
-  background: #0a0d11;
-  border: 1px solid var(--border);
-  border-radius: 4px;
-  height: 50vh;
-  overflow: auto;
-  padding: 10px;
-  margin: 0;
-  font: 12px/1.5 "Consolas", "SFMono-Regular", monospace;
-  white-space: pre-wrap;
-}
-#terminal-out .info { color: var(--muted); }
-#terminal-out .error { color: #e07b70; }
-#terminal-out .success { color: var(--gain); }
-#terminal-out .input { color: var(--accent); }
-.terminal-input { display: flex; gap: 8px; }
-.terminal-input input { flex: 1; }
-
-/* Toasts */
-#toasts {
-  position: fixed;
-  right: 14px;
-  bottom: 56px;
-  display: flex;
-  flex-direction: column;
-  gap: 8px;
-  z-index: 80;
-}
-.toast {
-  background: var(--panel-2);
-  border: 1px solid var(--border);
-  border-left: 3px solid var(--accent);
-  border-radius: 6px;
-  padding: 10px 14px;
-  max-width: 420px;
-  box-shadow: 0 4px 14px rgba(0,0,0,.4);
-}
-.toast.err { border-left-color: var(--loss); }
-.toast.ok { border-left-color: var(--gain); }
-
-@media (max-width: 720px) {
-  .pf-summary { order: 5; width: 100%; }
-  .datewrap { margin-left: 0; }
-}
-
-/* Splash Screen Styles */
-.splash-screen {
-  position: fixed;
-  top: 0;
-  left: 0;
-  width: 100vw;
-  height: 100vh;
-  background: radial-gradient(circle at center, #1b2230 0%, #0e1116 100%);
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  z-index: 9999;
-  opacity: 1;
-  visibility: visible;
-  transition: opacity 0.6s cubic-bezier(0.4, 0, 0.2, 1), visibility 0.6s cubic-bezier(0.4, 0, 0.2, 1);
-}
-
-.splash-screen.fade-out {
-  opacity: 0;
-  visibility: hidden;
-  pointer-events: none;
-}
-
-.splash-content {
-  text-align: center;
-  animation: splashFadeIn 1s cubic-bezier(0.16, 1, 0.3, 1);
-  padding: 20px;
-}
-
-.splash-logo {
-  font-size: 42px;
-  font-weight: 800;
-  letter-spacing: -0.5px;
-  margin-bottom: 8px;
-  color: #ffffff;
-}
-
-.splash-brand-nse {
-  background: linear-gradient(135deg, #3b82c4 0%, #27ae60 100%);
-  -webkit-background-clip: text;
-  -webkit-text-fill-color: transparent;
-  padding-right: 2px;
-}
-
-.splash-brand-dash {
-  color: #ffffff;
-  text-shadow: 0 0 20px rgba(59, 130, 196, 0.3);
-}
-
-.splash-subtitle {
-  font-size: 13px;
-  color: var(--muted);
-  letter-spacing: 2px;
-  text-transform: uppercase;
-  margin-bottom: 30px;
-}
-
-.splash-loader-container {
-  width: 220px;
-  height: 4px;
-  background: var(--border);
-  border-radius: 2px;
-  margin: 0 auto 12px;
-  overflow: hidden;
-  position: relative;
-}
-
-.splash-loader-bar {
-  width: 100%;
-  height: 100%;
-  background: linear-gradient(90deg, #3b82c4, #27ae60, #3b82c4);
-  background-size: 200% 100%;
-  position: absolute;
-  left: -100%;
-  animation: loadingProgress 2s infinite linear, gradientMove 3s infinite linear;
-}
-
-.splash-status {
-  font-size: 11px;
-  color: var(--muted);
-  font-family: monospace;
-}
-
-@keyframes splashFadeIn {
-  from {
-    opacity: 0;
-    transform: translateY(20px);
-  }
-  to {
-    opacity: 1;
-    transform: translateY(0);
-  }
-}
-
-@keyframes loadingProgress {
-  0% { left: -100%; }
-  50% { left: 0%; }
-  100% { left: 100%; }
-}
-
-@keyframes gradientMove {
-  0% { background-position: 0% 50%; }
-  100% { background-position: 200% 50%; }
-}
-
-.captcha-box {
-  margin-top: 24px;
-  padding: 20px;
-  border-radius: 12px;
-  background: rgba(255, 255, 255, 0.03);
-  border: 1px solid rgba(255, 255, 255, 0.08);
-  box-shadow: 0 4px 30px rgba(0, 0, 0, 0.2);
-  backdrop-filter: blur(5px);
-  -webkit-backdrop-filter: blur(5px);
-  animation: splashFadeIn 0.5s ease-out;
-}
-.captcha-box p {
-  margin: 0 0 14px 0;
-  font-size: 14px;
-  color: #ccc;
-  font-family: inherit;
-}
-.captcha-box .captcha-row {
-  display: flex;
-  gap: 12px;
-  align-items: center;
-  justify-content: center;
-}
-.captcha-box input {
-  width: 90px;
-  padding: 8px;
-  border-radius: 6px;
-  border: 1px solid rgba(255, 255, 255, 0.15);
-  background: rgba(255, 255, 255, 0.07);
-  color: #ffffff;
-  font-size: 16px;
-  text-align: center;
-  outline: none;
-  transition: border-color 0.3s;
-}
-.captcha-box input:focus {
-  border-color: #3b82c4;
-}
-.captcha-box button {
-  padding: 8px 18px;
-  border-radius: 6px;
-  background: var(--live);
-  color: #ffffff;
-  border: none;
-  cursor: pointer;
-  font-weight: 700;
-  transition: filter 0.3s;
-}
-.captcha-box button:hover {
-  filter: brightness(1.15);
-}
-.captcha-box .captcha-err {
-  color: var(--loss);
-  font-size: 12px;
-  margin: 10px 0 0 0;
-  font-weight: bold;
-}
-
-/* Shutdown Screen Styles */
-.shutdown-screen {
-  position: fixed;
-  top: 0;
-  left: 0;
-  width: 100vw;
-  height: 100vh;
-  background: radial-gradient(circle at center, #2c1a1d 0%, #0e0b0c 100%);
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  z-index: 10000;
-  opacity: 0;
-  visibility: hidden;
-  transition: opacity 0.5s ease, visibility 0.5s ease;
-}
-
-.shutdown-screen.visible {
-  opacity: 1;
-  visibility: visible;
-}
-
-
-/* Login and TOTP styling */
-.login-box, .totp-box {
-  margin-top: 24px;
-  padding: 20px;
-  border-radius: 12px;
-  background: rgba(255, 255, 255, 0.03);
-  border: 1px solid rgba(255, 255, 255, 0.08);
-  box-shadow: 0 4px 30px rgba(0, 0, 0, 0.2);
-  backdrop-filter: blur(5px);
-  -webkit-backdrop-filter: blur(5px);
-  animation: splashFadeIn 0.5s ease-out;
-  min-width: 280px;
-}
-.login-box h3, .totp-box h3 {
-  margin-top: 0;
-  margin-bottom: 12px;
-  color: #fff;
-  font-size: 18px;
-}
-.login-row {
-  margin-bottom: 10px;
-}
-.login-row input {
-  width: 100%;
-  padding: 8px;
-  border-radius: 6px;
-  border: 1px solid rgba(255, 255, 255, 0.15);
-  background: rgba(255, 255, 255, 0.07);
-  color: #ffffff;
-  font-size: 14px;
-  outline: none;
-}
-.login-row input:focus {
-  border-color: #3b82c4;
-}
-.login-box button, .totp-box button {
-  width: 100%;
-  padding: 8px;
-  border-radius: 6px;
-  background: var(--live);
-  color: #ffffff;
-  border: none;
-  cursor: pointer;
-  font-weight: 700;
-  font-size: 14px;
-}
-.login-err {
-  color: var(--loss);
-  font-size: 12px;
-  margin: 10px 0 0 0;
-  font-weight: bold;
-}
-.totp-qr-container img {
-  border: 1px solid #fff;
-  border-radius: 6px;
-  background: white;
-  padding: 6px;
-  margin: 8px 0;
-  max-width: 140px;
-}
-
-/* Admin Panel modal customization & resizability */
-.modal.admin-panel {
-  width: min(1200px, 96vw);
-  min-width: 600px;
-  max-width: 98vw;
-  height: auto;
-  min-height: 480px;
-  max-height: 94vh;
-  resize: both;
-  overflow: auto;
-  box-sizing: border-box;
-}
-
-.admin-main-grid {
-  display: grid;
-  grid-template-columns: 1fr;
-  gap: 16px;
-  width: 100%;
-}
-
-.admin-main-grid.has-sidebar {
-  grid-template-columns: minmax(0, 1fr) 320px;
-}
-
-.admin-section.user-mgmt {
-  display: flex;
-  flex-direction: column;
-  width: 100%;
-  flex: 1;
-}
-
-.admin-section.user-mgmt .table-wrap {
-  flex: 1;
-  max-height: 60vh;
-  overflow-y: auto;
-}
-
-.admin-table th, .admin-table td {
-  border-bottom: 1px solid rgba(255, 255, 255, 0.08);
-}
-.admin-table tr:hover td {
-  background: rgba(255, 255, 255, 0.02);
-}
-.sidebar-panel {
-  animation: splashFadeIn 0.3s ease-out;
-}
-.chat-view {
-  background: #0f172a;
-  border: 1px solid rgba(255,255,255,0.1);
-  height: 120px;
-  overflow-y: auto;
-  border-radius: 4px;
-  padding: 6px;
-}
-.chat-msg {
-  margin-bottom: 4px;
-}
-.chat-msg.system {
-  color: #94a3b8;
-  font-style: italic;
-}
-.chat-msg.admin {
-  color: #3b82f6;
-  font-weight: bold;
-}
-.chat-msg.user {
-  color: #10b981;
-  font-weight: bold;
-}
-
-/* ==========================================
-   Responsive UI Overrides for Mobile Screens
-   ========================================== */
-html, body {
-  overflow-x: hidden;
-}
-
-@media (max-width: 768px) {
-  /* Swipeable Horizontal Menu Bar */
-  .menubar {
-    overflow-x: auto;
-    white-space: nowrap;
-    flex-wrap: nowrap;
-    -webkit-overflow-scrolling: touch;
-    gap: 0;
-  }
-  .menubar .menu {
-    flex-shrink: 0;
-  }
-  .menu-user-display {
-    margin-left: 8px !important;
-    flex-shrink: 0;
-  }
-  .menu-btn {
-    padding: 8px 10px;
-  }
-
-  /* Responsive Admin Control Panel */
-  .stat-cards {
-    grid-template-columns: 1fr !important;
-    gap: 8px;
-  }
-  .admin-main-grid {
-    grid-template-columns: 1fr !important;
-    gap: 16px;
-  }
-  .table-wrap {
-    overflow-x: auto;
-    -webkit-overflow-scrolling: touch;
-  }
-  .admin-table {
-    min-width: 550px;
-  }
-}
-
-@media (max-width: 600px) {
-  /* Toolbar Layout Adjustments */
-  .toolbar {
-    padding: 8px;
-    gap: 6px;
-  }
-  .toolbar > button, .toolbar > label.toggle {
-    flex: 1 1 calc(50% - 6px);
-  }
-  .pf-summary {
-    width: 100%;
-    order: 4;
-    gap: 8px;
-    justify-content: space-around;
-    padding: 6px;
-  }
-  .pf-summary span {
-    font-size: 11px;
-  }
-  .datewrap {
-    width: 100%;
-    order: 5;
-    margin-left: 0;
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-  }
-  .datewrap input {
-    flex-grow: 1;
-    max-width: 70%;
-  }
-
-  /* Swipeable Bottom Control Dock */
-  .controls {
-    overflow-x: auto;
-    white-space: nowrap;
-    flex-wrap: nowrap;
-    -webkit-overflow-scrolling: touch;
-    gap: 6px;
-    padding: 6px 8px;
-  }
-  .controls button {
-    flex-shrink: 0;
-  }
-}
-
-@media (max-width: 480px) {
-  /* Add Stock Row Stacking */
-  .addrow {
-    flex-direction: column;
-    align-items: stretch;
-    gap: 6px;
-    padding: 8px;
-  }
-  #stock-search {
-    width: 100%;
-    flex: unset;
-  }
-  .addrow button {
-    width: 100%;
-  }
-
-  /* High Density Grid and Card scaling */
-  .grid {
-    grid-template-columns: 1fr;
-    padding: 0 8px 80px;
-    gap: 10px;
-  }
-  .card {
-    padding: 8px;
-  }
-  .card h4 {
-    font-size: 13px;
-  }
-
-  /* Responsive Dialog Sizing */
-  .modal {
-    padding: 14px;
-    min-width: unset;
-    width: 100%;
-  }
-}
-
-/* Screen reader only helper */
-.sr-only {
-  position: absolute;
-  width: 1px;
-  height: 1px;
-  padding: 0;
-  margin: -1px;
-  overflow: hidden;
-  clip: rect(0, 0, 0, 0);
-  white-space: nowrap;
-  border: 0;
-}
-
-/* AI button & modal styles */
-.ai-btn {
-  background: linear-gradient(135deg, #6c5ce7, #4834d4);
-  color: #fff;
-  border: none;
-  font-weight: 600;
-  transition: transform 0.2s ease, filter 0.2s ease;
-}
-.ai-btn:hover {
-  filter: brightness(1.2);
-  transform: translateY(-1px);
-}
-.ai-analysis-modal {
-  max-width: 800px;
-  width: 90%;
-}
-@keyframes spin {
-  0% { transform: rotate(0deg); }
-  100% { transform: rotate(360deg); }
-}
-/* Style markdown lists, headings, and tables in modal */
-.markdown-body h1, .markdown-body h2, .markdown-body h3 {
-  margin-top: 16px;
-  margin-bottom: 8px;
-  color: var(--accent);
-}
-.markdown-body p {
-  margin-bottom: 12px;
-}
-.markdown-body ul, .markdown-body ol {
-  padding-left: 20px;
-  margin-bottom: 12px;
-}
-.markdown-body li {
-  margin-bottom: 4px;
-}
-
-
-

nsedash/web/templates/index.html

@@ -1,489 +0,0 @@
-<!doctype html>
-<html lang="en">
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>NSE Stock Dashboard</title>
-<link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
-<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.min.js"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/marked/11.1.1/marked.min.js"></script>
-</head>
-<body>
-
-<!-- Accessibility Landmark Header -->
-<h1 class="sr-only">NSE Stock Dashboard</h1>
-
-<!-- Splash Screen -->
-<div id="splash-screen" class="splash-screen">
-  <div class="splash-content">
-    <div class="splash-logo">
-      <span class="splash-brand-nse">NSE</span><span class="splash-brand-dash">Dashboard</span>
-    </div>
-    <div class="splash-subtitle">Market Intelligence & Analytics</div>
-    <div class="splash-loader-container">
-      <div class="splash-loader-bar"></div>
-    </div>
-    <div class="splash-status" id="splash-status">Initializing System...</div>
-    
-    <!-- Login Form Box -->
-    <div id="login-box" class="login-box hidden">
-      <h3>Sign In</h3>
-      <div class="login-row">
-        <input type="text" id="login-username" placeholder="Username" autocomplete="username" aria-label="Username">
-      </div>
-      <div class="login-row">
-        <input type="password" id="login-password" placeholder="Password" autocomplete="current-password" aria-label="Password">
-      </div>
-      <div class="login-row">
-        <button id="btn-login-submit">Login</button>
-      </div>
-      <p id="login-error" class="login-err hidden"></p>
-    </div>
-
-    <!-- TOTP Form Box -->
-    <div id="totp-box" class="totp-box hidden">
-      <h3>Two-Factor Authentication</h3>
-      <div id="totp-qr-container" class="totp-qr-container hidden">
-        <p style="font-size: 12px; margin-bottom: 10px; color: #a5b4fc;">Scan the QR code below using your Google Authenticator or choice TOTP app:</p>
-        <img id="totp-qr-image" src="" alt="TOTP QR Code">
-        <p style="font-size: 11px; margin-top: 5px; color: #cbd5e1;">Secret key: <code id="totp-secret-key" style="font-weight: bold; letter-spacing: 0.5px;"></code></p>
-      </div>
-      <p>Enter the 6-digit verification code from your authenticator app:</p>
-      <div class="captcha-row">
-        <input type="number" id="totp-input" placeholder="000000" pattern="[0-9]*" inputmode="numeric" aria-label="6-digit TOTP verification code">
-        <button id="btn-totp-submit">Verify</button>
-      </div>
-      <p id="totp-error" class="login-err hidden"></p>
-    </div>
-  </div>
-</div>
-
-<!-- Status banner: green = live, red = historic (v4.5 parity) -->
-<div id="banner" class="banner historic">
-  <div style="display: flex; align-items: center; gap: 8px;">
-    <span id="banner-text">Historic Data</span>
-    <span id="env-pill" class="env-pill local">LOCAL</span>
-  </div>
-  <span id="banner-watchlist"></span>
-</div>
-
-<!-- Menu bar -->
-<nav class="menubar" role="menubar" aria-label="Main Navigation">
-  <div class="menu" data-menu="watchlists">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-watchlists" style="color: #10b981; font-weight: bold;">Watchlists 🟢</button>
-    <div class="menu-drop" id="menu-watchlists" role="menu" aria-labelledby="menu-btn-watchlists"></div>
-  </div>
-  <div class="menu" data-menu="maxmin">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-maxmin">Max Min Analysis</button>
-    <div class="menu-drop" id="menu-maxmin" role="menu" aria-labelledby="menu-btn-maxmin"></div>
-  </div>
-  <div class="menu" data-menu="volume">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-volume">Volume Analysis</button>
-    <div class="menu-drop" id="menu-volume" role="menu" aria-labelledby="menu-btn-volume"></div>
-  </div>
-  <div class="menu" data-menu="liveanalysis">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-liveanalysis">Live Analysis</button>
-    <div class="menu-drop" id="menu-liveanalysis" role="menu" aria-labelledby="menu-btn-liveanalysis"></div>
-  </div>
-  <div class="menu" data-menu="simulation">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-simulation">Simulation</button>
-    <div class="menu-drop" id="menu-simulation" role="menu" aria-labelledby="menu-btn-simulation"></div>
-  </div>
-  <div class="menu" data-menu="settings">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-settings" style="color: #38bdf8;">Settings ⚙️</button>
-    <div class="menu-drop" id="menu-settings" role="menu" aria-labelledby="menu-btn-settings">
-      <button class="menu-drop-item" id="btn-settings-change-password" role="menuitem">🔒 Change Password</button>
-    </div>
-  </div>
-  <div class="menu hidden" id="menu-utilities-container" data-menu="utilities">
-    <button class="menu-btn" aria-haspopup="true" aria-expanded="false" id="menu-btn-utilities">Utilities</button>
-    <div class="menu-drop" id="menu-utilities" role="menu" aria-labelledby="menu-btn-utilities"></div>
-  </div>
-  <div class="menu-llm-select-wrap" style="margin-left: auto; display: flex; align-items: center; gap: 10px; padding-right: 15px; font-size: 12px;">
-    <label for="select-primary-llm" style="color: #94a3b8; font-weight: 500; display: flex; align-items: center; gap: 5px;">
-      <span style="color: #3b82f6; font-weight: bold;">🤖 Primary LLM:</span>
-      <select id="select-primary-llm" aria-label="Primary LLM Model Selection" style="background: #0f172a; color: #60a5fa; border: 1px solid #3b82f6; border-radius: 4px; padding: 3px 8px; font-size: 11px; font-weight: bold; cursor: pointer; outline: none;">
-        <option value="google/diffusiongemma-26b-a4b-it" selected>DiffusionGemma 26B (google/diffusiongemma-26b-a4b-it) [Fastest]</option>
-        <option value="minimaxai/minimax-m3">Minimax-M3 (minimaxai/minimax-m3)</option>
-        <option value="z-ai/glm-5.2">GLM-5.2 (z-ai/glm-5.2)</option>
-        <option value="nvidia/nemotron-3-ultra-550b-a55b">Nemotron 3 Ultra 550B (nvidia/nemotron-3-ultra-550b-a55b)</option>
-        <option value="meta/llama-3.1-70b-instruct">Llama 3.1 70B (meta/llama-3.1-70b-instruct)</option>
-        <option value="thinkingmachines/inkling">Inkling 975B (thinkingmachines/inkling)</option>
-      </select>
-    </label>
-
-    <label for="select-llm-timeout" style="color: #94a3b8; font-weight: 500; display: flex; align-items: center; gap: 5px;">
-      <span style="color: #10b981; font-weight: bold;">⏱️ Timeout:</span>
-      <select id="select-llm-timeout" aria-label="LLM Timeout Selection" style="background: #0f172a; color: #10b981; border: 1px solid #10b981; border-radius: 4px; padding: 3px 6px; font-size: 11px; font-weight: bold; cursor: pointer; outline: none;">
-        <option value="5" selected>5 Mins</option>
-        <option value="10">10 Mins</option>
-        <option value="15">15 Mins</option>
-        <option value="20">20 Mins</option>
-      </select>
-    </label>
-  </div>
-  <div class="menu-user-display" id="user-display-wrap" style="display: flex; align-items: center; gap: 12px; padding-right: 15px;">
-    <span style="font-size: 13px; color: #94a3b8;">User: <b id="display-username" style="color: #3b82f6;">admin</b></span>
-    <button id="btn-logout" style="background: none; border: 1px solid #475569; color: #94a3b8; padding: 3px 8px; border-radius: 4px; font-size: 11px; cursor: pointer; transition: all 0.2s;" onmouseover="this.style.color='#ef4444'; this.style.borderColor='#ef4444'" onmouseout="this.style.color='#94a3b8'; this.style.borderColor='#475569'">Logout</button>
-  </div>
-  <div class="menu" data-menu="exit">
-    <button class="menu-btn" id="btn-exit" aria-label="Exit Application" style="color: #e07b70; font-weight: bold;">Exit</button>
-  </div>
-</nav>
-
-<!-- Toolbar -->
-<header class="toolbar">
-  <label class="toggle">
-    <input type="checkbox" id="live-toggle">
-    <span>Live Data</span>
-  </label>
-  <button id="btn-portfolio">Load Portfolio</button>
-  <div id="portfolio-summary" class="pf-summary">
-    <span>Invested <b id="pf-invested">0.00</b></span>
-    <span>Current <b id="pf-current">0.00</b></span>
-    <span>Day's Gain <b id="pf-day">0.00 (0.00%)</b></span>
-    <span>Abs Returns <b id="pf-abs">0.00 (0.00%)</b></span>
-  </div>
-  <button id="btn-export">Export CSV</button>
-  <button id="btn-sensex">SENSEX</button>
-  <button id="btn-nifty">NIFTY</button>
-
-  <!-- Live Indices Display Cards (NIFTY & SENSEX) -->
-  <div id="index-tickers" class="index-tickers">
-    <div class="index-badge" id="nifty-badge" title="Live NIFTY 50 (Click to filter)">
-      <span class="idx-name">NIFTY</span>
-      <span class="idx-pill positive" id="nifty-val">▲ 24,227.50 (+242.15 / +1.01%)</span>
-    </div>
-    <div class="index-badge" id="sensex-badge" title="Live SENSEX (Click to filter)">
-      <span class="idx-name">SENSEX</span>
-      <span class="idx-pill positive" id="sensex-val">▲ 77,568.70 (+802.78 / +1.04%)</span>
-    </div>
-  </div>
-
-  <!-- RTC Clock (IST) & Historic Data Date -->
-  <div class="date-rtc-wrap">
-    <div id="rtc-clock" class="rtc-clock" title="Current Real-Time Clock (IST Time Zone)">
-      <span class="rtc-icon">🕒</span> <span id="rtc-time">--:--:-- IST</span>
-    </div>
-    <label class="datewrap">Historic data date
-      <input type="date" id="date-input">
-    </label>
-  </div>
-</header>
-
-<!-- Add stock row -->
-<section class="addrow">
-  <input list="stock-list" id="stock-search"
-         placeholder="Search symbol or security…" aria-label="Search stock symbol or security to add">
-  <datalist id="stock-list"></datalist>
-  <button id="btn-add">Add Stock</button>
-  <span id="board-count" class="muted"></span>
-</section>
-
-<!-- Card grid -->
-<main id="grid" class="grid"></main>
-
-<!-- Bottom controls -->
-<footer class="controls">
-  <button id="btn-refresh">Refresh Data</button>
-  <button id="btn-save">Save Dashboard</button>
-  <button id="btn-saveas">Save as Watchlist</button>
-  <button id="btn-add-selected">Add Selected to Watchlist</button>
-  <button id="btn-delete-all" class="danger">Delete All</button>
-</footer>
-
-<!-- Busy overlay (web equivalent of the v4.5 wait dialog) -->
-<div id="busy" class="overlay hidden">
-  <div class="busy-box">
-    <div class="spinner"></div>
-    <div id="busy-text">Processing…</div>
-  </div>
-</div>
-
-<!-- Live source modal -->
-<div id="modal-live" class="overlay hidden">
-  <div class="modal">
-    <h3>Start live data</h3>
-    <label>Source
-      <select id="live-source">
-        <option value="Google">Google Finance</option>
-        <option value="Yahoo">Yahoo Finance</option>
-      </select>
-    </label>
-    <label>Refresh every (minutes)
-      <input type="number" id="live-rate" value="5" min="1">
-    </label>
-    <div class="modal-actions">
-      <button id="live-cancel">Cancel</button>
-      <button id="live-ok" class="primary">Start</button>
-    </div>
-  </div>
-</div>
-
-<!-- Save-as modal -->
-<div id="modal-saveas" class="overlay hidden">
-  <div class="modal">
-    <h3>Save as watchlist</h3>
-    <label>Slot (1–20)
-      <input type="number" id="saveas-slot" value="1" min="1" max="20">
-    </label>
-    <label>Name
-      <input type="text" id="saveas-name" placeholder="Watchlist name">
-    </label>
-    <div class="modal-actions">
-      <button id="saveas-cancel">Cancel</button>
-      <button id="saveas-ok" class="primary">Save</button>
-    </div>
-  </div>
-</div>
-
-<!-- Admin Panel Modal -->
-<div id="modal-admin" class="overlay hidden">
-  <div class="modal admin-panel">
-    <div class="modal-header" style="display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 12px; margin-bottom: 15px;">
-      <h3 style="margin: 0; display: flex; align-items: center; gap: 8px;">🛡️ Admin Control Panel</h3>
-      <button class="close-btn" id="admin-close" style="background: none; border: none; font-size: 24px; color: #94a3b8; cursor: pointer;">&times;</button>
-    </div>
-    
-    <div class="admin-content">
-      <div class="stat-cards" style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 12px; margin-bottom: 18px;">
-        <div class="stat-card" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.07); border-radius: 8px; padding: 12px; text-align: center;">
-          <h4 style="margin: 0 0 5px 0; color: #3b82f6; font-size: 20px;" id="stat-users">0</h4>
-          <span style="font-size: 11px; color: #64748b; text-transform: uppercase;">Total Users</span>
-        </div>
-        <div class="stat-card" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.07); border-radius: 8px; padding: 12px; text-align: center;">
-          <h4 style="margin: 0 0 5px 0; color: #10b981; font-size: 20px;" id="stat-symbols">0</h4>
-          <span style="font-size: 11px; color: #64748b; text-transform: uppercase;">Total Symbols</span>
-        </div>
-        <div class="stat-card" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.07); border-radius: 8px; padding: 12px; text-align: center;">
-          <h4 style="margin: 0 0 5px 0; color: #f59e0b; font-size: 20px;" id="stat-yfinance">Checking...</h4>
-          <span style="font-size: 11px; color: #64748b; text-transform: uppercase;">yFinance Status</span>
-        </div>
-      </div>
-      
-      <div class="admin-main-grid">
-        <!-- User Registry Section -->
-        <div class="admin-section user-mgmt" style="background: rgba(255,255,255,0.02); border: 1px solid rgba(255,255,255,0.05); border-radius: 10px; padding: 15px;">
-          <div class="section-header" style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 12px;">
-            <h4 style="margin: 0; font-size: 14px; text-transform: uppercase; color: #cbd5e1;">User Registry</h4>
-            <button id="admin-btn-show-add" class="primary btn-xs" style="padding: 4px 10px; font-size: 11px;">➕ Add User</button>
-          </div>
-          <div class="table-wrap" style="max-height: 300px; overflow-y: auto;">
-            <table class="admin-table" style="width: 100%; border-collapse: collapse; text-align: left; font-size: 12px;">
-              <thead>
-                <tr style="border-bottom: 1px solid rgba(255,255,255,0.1); color: #94a3b8;">
-                  <th style="padding: 6px;">User ID</th>
-                  <th style="padding: 6px;">Status</th>
-                  <th style="padding: 6px;">Role</th>
-                  <th style="padding: 6px;">Logins</th>
-                  <th style="padding: 6px;">Last Login</th>
-                  <th style="padding: 6px;">Account</th>
-                  <th style="padding: 6px; text-align: right;">Actions</th>
-                </tr>
-              </thead>
-              <tbody id="admin-user-list"></tbody>
-            </table>
-          </div>
-        </div>
-
-        <!-- Sidebar detail panels -->
-        <div class="admin-sidebar" style="display: flex; flex-direction: column; gap: 12px;">
-          <!-- Add User Panel -->
-          <div id="panel-add-user" class="sidebar-panel hidden" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.08); border-radius: 8px; padding: 12px;">
-            <h4 style="margin: 0 0 10px 0; font-size: 13px; color: #cbd5e1; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 5px;">Create New User</h4>
-            <div style="display: flex; flex-direction: column; gap: 8px;">
-              <input type="text" id="add-username" placeholder="Username" aria-label="New username" style="width: 100%; padding: 6px; border-radius: 4px; border: 1px solid rgba(255,255,255,0.15); background: #0f172a; color: white;">
-              <select id="add-role" aria-label="New user role" style="width: 100%; padding: 6px; border-radius: 4px; border: 1px solid rgba(255,255,255,0.15); background: #0f172a; color: white;">
-                <option value="user">User</option>
-                <option value="admin">Admin</option>
-              </select>
-              <button id="admin-btn-create-submit" class="primary" style="width: 100%; padding: 6px; border-radius: 4px;">Create User</button>
-            </div>
-          </div>
-
-          <!-- TOTP QR Code Panel -->
-          <div id="panel-qr" class="sidebar-panel hidden" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.08); border-radius: 8px; padding: 12px; text-align: center;">
-            <h4 style="margin: 0 0 10px 0; font-size: 13px; color: #cbd5e1; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 5px; text-align: left;">TOTP Authenticator Setup</h4>
-            <div id="admin-qr-display" style="margin: 10px 0; display: inline-block; padding: 8px; background: white; border-radius: 6px;"></div>
-            <div style="font-size: 11px; margin-top: 8px; text-align: left; color: #94a3b8; word-break: break-all;">
-              Secret key: <code id="admin-secret-key" style="font-weight: bold; color: #3b82f6;"></code>
-            </div>
-          </div>
-
-          <!-- User Metrics Panel -->
-          <div id="panel-metrics" class="sidebar-panel hidden" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.08); border-radius: 8px; padding: 12px; font-size: 12px;">
-            <h4 style="margin: 0 0 10px 0; font-size: 13px; color: #cbd5e1; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 5px;">User Statistics</h4>
-            <div id="admin-metrics-display" style="display: flex; flex-direction: column; gap: 6px; color: #cbd5e1;"></div>
-          </div>
-
-          <!-- Mirrored Activity Logs Panel -->
-          <div id="panel-mirror" class="sidebar-panel hidden" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.08); border-radius: 8px; padding: 12px;">
-            <h4 style="margin: 0 0 10px 0; font-size: 13px; color: #cbd5e1; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 5px;">User Activity Mirror</h4>
-            <pre id="admin-mirror-display" class="log-view" style="max-height: 150px; overflow-y: auto; font-family: monospace; font-size: 11px; color: #a7f3d0; margin: 0; background: #0f172a; padding: 8px; border-radius: 4px; text-align: left; white-space: pre-wrap;"></pre>
-          </div>
-
-          <!-- User Chat Panel -->
-          <div id="panel-chat" class="sidebar-panel hidden" style="background: rgba(255,255,255,0.03); border: 1px solid rgba(255,255,255,0.08); border-radius: 8px; padding: 12px;">
-            <h4 id="chat-user-title" style="margin: 0 0 10px 0; font-size: 13px; color: #cbd5e1; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 5px;">Chat</h4>
-            <div id="admin-chat-display" class="chat-view" style="max-height: 120px; overflow-y: auto; display: flex; flex-direction: column; gap: 6px; font-size: 11px; margin-bottom: 8px; background: #0f172a; padding: 6px; border-radius: 4px;"></div>
-            <div class="chat-input-row" style="display: flex; gap: 6px;">
-              <input type="text" id="chat-input" placeholder="Type message..." aria-label="Chat message text" style="flex: 1; padding: 4px; border-radius: 4px; border: 1px solid rgba(255,255,255,0.15); background: #0f172a; color: white; font-size: 11px;">
-              <button id="chat-send-btn" class="primary" style="padding: 4px 10px; font-size: 11px; border-radius: 4px;">Send</button>
-            </div>
-          </div>
-        </div>
-      </div>
-    </div>
-  </div>
-</div>
-
-<!-- Script terminal modal -->
-<div id="modal-terminal" class="overlay hidden">
-  <div class="modal terminal">
-    <h3 id="terminal-title">Script output</h3>
-    <pre id="terminal-out"></pre>
-    <div class="terminal-input">
-      <input type="text" id="terminal-in"
-             placeholder="Type input and press Enter…">
-      <button id="terminal-send">Send</button>
-      <button id="terminal-close">Close</button>
-    </div>
-  </div>
-</div>
-
-<!-- AI Analysis Modal -->
-<div id="modal-ai" class="overlay hidden" role="dialog" aria-modal="true" aria-labelledby="ai-title">
-  <div class="modal ai-analysis-modal" style="max-width: 850px; width: 90%;">
-    <div class="modal-header" style="display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid rgba(255,255,255,0.1); padding-bottom: 12px; margin-bottom: 15px;">
-      <h3 id="ai-title" style="margin: 0; display: flex; align-items: center; gap: 8px;">🤖 Agentic AI Stock Research</h3>
-      <button id="ai-close" class="close-btn" style="background: none; border: none; font-size: 24px; color: var(--muted); cursor: pointer;" aria-label="Close AI analysis">×</button>
-    </div>
-    
-    <div id="ai-content" class="markdown-body" style="max-height: 60vh; overflow-y: auto; text-align: left; padding: 12px; font-size: 14px; line-height: 1.6; color: var(--fg); background: rgba(15, 23, 42, 0.6); border-radius: 8px; border: 1px solid rgba(255,255,255,0.05);">
-      <!-- Generated AI analysis in markdown format will go here -->
-    </div>
-
-    <!-- Metadata & Controls Footer -->
-    <div class="modal-footer" style="margin-top: 15px; padding-top: 12px; border-top: 1px solid rgba(255,255,255,0.1); display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 10px;">
-      <!-- Metadata details: LLM Name, Time, Tokens -->
-      <div id="ai-meta-bar" style="font-size: 11px; color: #94a3b8; display: flex; align-items: center; gap: 12px; flex-wrap: wrap;">
-        <span id="ai-meta-model">🤖 LLM: <b style="color: #3b82f6;">z-ai/glm-5.2</b></span>
-        <span id="ai-meta-time">⏱️ Time: <b style="color: #10b981;">--s</b></span>
-        <span id="ai-meta-tokens">📊 Tokens: <b style="color: #cbd5e1;">In: 0 | Out: 0 | Total: 0</b></span>
-      </div>
-
-      <!-- Action Buttons -->
-      <div style="display: flex; align-items: center; gap: 8px; margin-left: auto;">
-        <button id="ai-refresh-btn" class="primary" style="padding: 6px 12px; font-size: 12px; border-radius: 6px; background: rgba(59, 130, 246, 0.2); border: 1px solid #3b82f6; color: #60a5fa; cursor: pointer; display: flex; align-items: center; gap: 4px;" title="Run fresh background AI agentic analysis">
-          🔄 Refresh Inference
-        </button>
-        <button id="ai-copy-btn" class="primary" style="padding: 6px 12px; font-size: 12px; border-radius: 6px; background: #3b82f6; border: 1px solid #2563eb; color: white; cursor: pointer; display: flex; align-items: center; gap: 4px;" title="Copy full report markdown to clipboard">
-          📋 Copy Report
-        </button>
-      </div>
-    </div>
-  </div>
-</div>
-
-<!-- Floating AI Agent Background Status Widget (Bottom Right) -->
-<div id="ai-bg-widget" class="hidden" style="position: fixed; bottom: 20px; right: 20px; z-index: 9999; background: #0f172a; border: 1px solid #3b82f6; border-radius: 8px; padding: 10px 14px; box-shadow: 0 10px 25px rgba(0,0,0,0.5); display: flex; align-items: center; gap: 10px; font-size: 12px; color: #f8fafc; transition: all 0.3s ease;">
-  <div id="ai-bg-spinner" class="spinner" style="border: 2px solid rgba(255,255,255,0.2); border-top: 2px solid #3b82f6; border-radius: 50%; width: 16px; height: 16px; animation: spin 1s linear infinite;"></div>
-  <span id="ai-bg-text" style="font-weight: 500;">AI Agentic Research Running...</span>
-  <button id="ai-bg-view-btn" style="background: #3b82f6; border: none; color: white; border-radius: 4px; padding: 3px 10px; font-size: 11px; font-weight: bold; cursor: pointer; display: none;">View Report</button>
-</div>
-
-<!-- Shutdown Screen -->
-<div id="shutdown-screen" class="shutdown-screen">
-  <div class="splash-content">
-    <div class="splash-logo">
-      <span class="splash-brand-nse">NSE</span><span class="splash-brand-dash">Dashboard</span>
-    </div>
-    <div class="splash-subtitle" style="color: var(--loss); text-shadow: 0 0 20px rgba(192, 57, 43, 0.3);">System Shut Down</div>
-    <div class="splash-status" style="font-size: 13px; margin-top: 20px; line-height: 1.6;">
-      The backend server has been terminated safely.<br>
-      You may now close this browser tab.
-    </div>
-  </div>
-</div>
-
-<!-- Modal: Change Password (Settings Menu) -->
-<div id="modal-change-password" class="modal-overlay hidden" role="dialog" aria-modal="true">
-  <div class="modal-card" style="max-width: 440px;">
-    <h3>🔒 Change Password</h3>
-    <p style="font-size: 12px; color: #94a3b8; margin-bottom: 15px; line-height: 1.4;">
-      Enter your current password and your new password twice to reconfirm.
-    </p>
-    <div style="background: rgba(15, 23, 42, 0.6); border: 1px solid rgba(59, 130, 246, 0.3); border-radius: 6px; padding: 10px; margin-bottom: 15px; font-size: 11px; color: #cbd5e1; line-height: 1.5;">
-      <b style="color: #60a5fa;">Password Policy Requirements:</b>
-      <ul style="margin: 4px 0 0 16px; padding: 0;">
-        <li>Minimum 8 characters length</li>
-        <li>At least 1 uppercase letter (A-Z)</li>
-        <li>At least 1 lowercase letter (a-z)</li>
-        <li>At least 1 digit (0-9)</li>
-        <li>At least 1 special character (e.g. !@#$%^&*)</li>
-      </ul>
-    </div>
-    <div style="display: flex; flex-direction: column; gap: 10px;">
-      <div>
-        <label for="cp-current" style="display: block; font-size: 11px; font-weight: bold; color: #94a3b8; margin-bottom: 4px;">Current Password:</label>
-        <input type="password" id="cp-current" placeholder="Enter current password" style="width: 100%; box-sizing: border-box;">
-      </div>
-      <div>
-        <label for="cp-new" style="display: block; font-size: 11px; font-weight: bold; color: #94a3b8; margin-bottom: 4px;">New Password (1st Entry):</label>
-        <input type="password" id="cp-new" placeholder="Enter new password" style="width: 100%; box-sizing: border-box;">
-      </div>
-      <div>
-        <label for="cp-confirm" style="display: block; font-size: 11px; font-weight: bold; color: #94a3b8; margin-bottom: 4px;">Confirm New Password (2nd Entry):</label>
-        <input type="password" id="cp-confirm" placeholder="Re-enter new password" style="width: 100%; box-sizing: border-box;">
-      </div>
-    </div>
-    <p id="cp-error" style="color: #ef4444; font-size: 11px; margin-top: 10px; font-weight: bold;" class="hidden"></p>
-    <div class="modal-actions" style="margin-top: 20px;">
-      <button id="cp-cancel" class="btn-subtle">Cancel</button>
-      <button id="cp-submit" class="primary">Update Password</button>
-    </div>
-  </div>
-</div>
-
-<!-- Modal: Mandatory Password Change on Next Login (Enforced Security Policy) -->
-<div id="modal-force-password-change" class="modal-overlay hidden" role="dialog" aria-modal="true" style="z-index: 20000; background: rgba(0,0,0,0.85); backdrop-filter: blur(8px);">
-  <div class="modal-card" style="max-width: 460px; border: 2px solid #ef4444; box-shadow: 0 0 30px rgba(239, 68, 68, 0.4);">
-    <h3 style="color: #ef4444; display: flex; align-items: center; gap: 8px;">
-      ⚠️ Mandatory Password Update Required
-    </h3>
-    <p style="font-size: 12px; color: #f8fafc; margin-bottom: 12px; line-height: 1.5;">
-      Your password does not meet the updated security policy standards. You must set a compliant password before accessing the dashboard.
-    </p>
-    <div style="background: rgba(15, 23, 42, 0.8); border: 1px solid rgba(239, 68, 68, 0.4); border-radius: 6px; padding: 10px; margin-bottom: 15px; font-size: 11px; color: #cbd5e1; line-height: 1.5;">
-      <b style="color: #ef4444;">Required Password Policy:</b>
-      <ul style="margin: 4px 0 0 16px; padding: 0;">
-        <li>Minimum 8 characters length</li>
-        <li>At least 1 uppercase letter (A-Z)</li>
-        <li>At least 1 lowercase letter (a-z)</li>
-        <li>At least 1 digit (0-9)</li>
-        <li>At least 1 special character (e.g. !@#$%^&*)</li>
-      </ul>
-    </div>
-    <div style="display: flex; flex-direction: column; gap: 10px;">
-      <div>
-        <label for="force-cp-new" style="display: block; font-size: 11px; font-weight: bold; color: #94a3b8; margin-bottom: 4px;">New Compliant Password (1st Entry):</label>
-        <input type="password" id="force-cp-new" placeholder="Enter new compliant password" style="width: 100%; box-sizing: border-box;">
-      </div>
-      <div>
-        <label for="force-cp-confirm" style="display: block; font-size: 11px; font-weight: bold; color: #94a3b8; margin-bottom: 4px;">Confirm New Password (2nd Entry):</label>
-        <input type="password" id="force-cp-confirm" placeholder="Re-enter new compliant password" style="width: 100%; box-sizing: border-box;">
-      </div>
-    </div>
-    <p id="force-cp-error" style="color: #ef4444; font-size: 11px; margin-top: 10px; font-weight: bold;" class="hidden"></p>
-    <div class="modal-actions" style="margin-top: 20px;">
-      <button id="force-cp-submit" class="primary" style="width: 100%; background: #ef4444; border-color: #dc2626;">Save New Password & Continue</button>
-    </div>
-  </div>
-</div>
-
-<!-- Toasts -->
-<div id="toasts"></div>
-
-<script src="{{ url_for('static', filename='app.js') }}"></script>
-</body>
-</html>

nsedash_admin.py

@@ -1,479 +0,0 @@
-#!/usr/bin/env python3
-"""
-NSE Stock Dashboard (NSEDashboard) Administration Utility
-
-Console-based interactive application providing granular menu options for 
-user administration, password resets, privilege management, data pipeline utilities,
-and system health checks. 
-
-Adapted specifically for the nsedash project architecture (including session 
-management, directory initialization, and utility scripts execution).
-"""
-
-import os
-import sys
-import argparse
-import subprocess
-from datetime import datetime
-
-# Setup paths so we can import nsedash modules
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-if SCRIPT_DIR not in sys.path:
-    sys.path.insert(0, SCRIPT_DIR)
-
-from werkzeug.security import generate_password_hash, check_password_hash
-
-# Import core nsedash components
-from nsedash.core.user_auth import init as auth_init, UserDatabase, SessionManager
-from nsedash.config import Paths, UTILITY_SCRIPTS
-
-# Ingest data processing modules
-try:
-    import nsedown
-    import nsecorpact
-    import nseprocessor
-    import nsesymhist
-    import nsemaxlive
-except ImportError as _imp_err:
-    pass
-
-
-def get_base_dir():
-    return os.environ.get("NSEDASH_BASE_DIR", os.getcwd())
-
-def init_auth():
-    base_dir = get_base_dir()
-    auth_init(base_dir)
-
-def print_splash():
-    banner = r"""
-======================================================================
-  _   _ ____  _____ ____   _   ____  _   _ ____   ___   _   ____  ____  
- | \ | / ___|| ____|  _ \ / \ / ___|| | | |  _ \ / _ \ / \ |  _ \|  _ \ 
- |  \| \___ \|  _| | | | / _ \\___ \| |_| | |_) | | | / _ \| |_) | | | |
- | |\  |___) | |___| |_| / ___ \___) |  _  |  _ <| |_| / ___ \  _ <| |_| |
- |_| \_|____/|_____|____/_/   \_\____/|_| |_|_| \_\\___/_/   \_\_| \_\____/ 
-======================================================================
-"""
-    print(banner)
-    print(" 🛠️  NSEDASH SYSTEM ADMIN CONSOLE")
-    print("======================================================================\n")
-
-def menu_reset_admin_password():
-    print("\n===================================================================")
-    print(" 🔑 RESET SUPER ADMIN ('admin') PASSWORD")
-    print("===================================================================")
-    
-    init_auth()
-    user = UserDatabase.get_user('admin')
-    current_hash = user.get('password_hash') if user else None
-    print(f"[*] Step 1: Current Stored Password Hash in Database:\n    - {current_hash or '(None / Not Set)'}")
-
-    print("\n[*] Step 2: Enter New Password")
-    while True:
-        new_pwd = input("Enter new password for super admin 'admin' (press Enter without input to exit): ").strip()
-        if not new_pwd:
-            print("[*] No input provided. Exiting password reset loop.")
-            return
-        if len(new_pwd) < 8:
-            print("[!] Error: Password must be at least 8 characters long. Please try again.")
-            continue
-        break
-
-    print("\n[*] Step 3: Hashing & storing new password in database...")
-    try:
-        pwd_hash = generate_password_hash(new_pwd, method='pbkdf2:sha256')
-        UserDatabase.update_user('admin', {
-            'password_hash': pwd_hash,
-            'active': True
-        })
-        SessionManager.clear_sessions_for_user('admin')
-        print("[OK] New password hash stored in database.")
-    except Exception as e:
-        print(f"[!] Error storing password in database: {e}")
-        return
-
-    print("\n[*] Step 4: Extracting stored password hash from database for verification...")
-    user = UserDatabase.get_user('admin')
-    reextracted_hash = user.get('password_hash') if user else None
-    print(f"    - Re-extracted Hash: {reextracted_hash}")
-
-    print("\n[*] Step 5: Comparing extracted password hash with entered new password...")
-    is_match = check_password_hash(reextracted_hash, new_pwd) if reextracted_hash else False
-
-    print("\n===================================================================")
-    print("  ✅ SUPER ADMIN PASSWORD VERIFICATION CONFIRMATION")
-    print("===================================================================")
-    print(f"  - Target User Account  : admin")
-    print(f"  - Extracted DB Hash    : {reextracted_hash}")
-    print(f"  - Password Match Check : {'PASSED (True)' if is_match else 'FAILED (False)'}")
-    print("  -----------------------------------------------------------------")
-    if is_match:
-        print("  ✅ Password verified, matches, and confirmed stored in database!")
-    else:
-        print("  ❌ ERROR: Extracted password does NOT match entered password!")
-    print("===================================================================\n")
-
-def menu_reset_any_user_password():
-    print("\n--- Reset Any User Password ---")
-    init_auth()
-    target_user = input("Enter username: ").strip()
-    if not target_user:
-        print("[!] Operation cancelled. Username required.")
-        return
-    
-    user = UserDatabase.get_user(target_user)
-    if not user:
-        print(f"[!] User '{target_user}' does not exist in the database.")
-        return
-
-    new_pwd = input(f"Enter new password for user '{target_user}': ").strip()
-    if len(new_pwd) < 8:
-        print("[!] Error: Password must be at least 8 characters long.")
-        return
-    
-    pwd_hash = generate_password_hash(new_pwd, method='pbkdf2:sha256')
-    try:
-        UserDatabase.update_user(target_user, {
-            'password_hash': pwd_hash,
-            'active': True
-        })
-        SessionManager.clear_sessions_for_user(target_user)
-        print(f"[OK] Password for user '{target_user}' reset successfully. User has been logged out.")
-    except Exception as e:
-        print(f"[!] Error resetting password: {e}")
-
-def menu_reset_totp_for_user():
-    print("\n--- Reset TOTP for User ---")
-    init_auth()
-    target_user = input("Enter username to reset TOTP: ").strip()
-    if not target_user:
-        print("[!] Operation cancelled. Username required.")
-        return
-    
-    user = UserDatabase.get_user(target_user)
-    if not user:
-        print(f"[!] User '{target_user}' does not exist in the database.")
-        return
-
-    try:
-        UserDatabase.update_user(target_user, {
-            'totp_secret': None,
-            'totp_secret_encrypted': None
-        })
-        SessionManager.clear_sessions_for_user(target_user)
-        print(f"[OK] TOTP for user '{target_user}' reset successfully. They will be prompted to set it up on their next login.")
-    except Exception as e:
-        print(f"[!] Error resetting TOTP: {e}")
-
-def menu_full_account_reset():
-    print("\n--- Full Account Reset (Force Fresh Registration) ---")
-    init_auth()
-    target_user = input("Enter username for full reset: ").strip()
-    if not target_user:
-        print("[!] Operation cancelled. Username required.")
-        return
-        
-    user = UserDatabase.get_user(target_user)
-    if not user:
-        print(f"[!] User '{target_user}' does not exist in the database.")
-        return
-        
-    try:
-        UserDatabase.update_user(target_user, {
-            'password_hash': None,
-            'totp_secret': None,
-            'totp_secret_encrypted': None
-        })
-        SessionManager.clear_sessions_for_user(target_user)
-        print(f"[OK] Full account reset for '{target_user}'. They have been logged out and will be forced to register a new password and set up TOTP on their next login.")
-    except Exception as e:
-        print(f"[!] Error resetting account: {e}")
-
-def menu_create_new_user():
-    print("\n===================================================================")
-    print(" ➕ CREATE NEW USER ACCOUNT")
-    print("===================================================================")
-    init_auth()
-    
-    userid = input("1. Enter Username (userid): ").strip()
-    if not userid:
-        print("[!] Error: Username is required.")
-        return
-        
-    if UserDatabase.get_user(userid):
-        print(f"[!] Error: User '{userid}' already exists.")
-        return
-        
-    is_admin_input = input("2. Make user an admin? (y/N): ").strip().lower()
-    is_admin = is_admin_input == 'y'
-    
-    print("\n[*] Processing user creation...")
-    try:
-        UserDatabase.create_user(userid, is_admin)
-        print("\n===================================================================")
-        print("  ✅ NEW USER ACCOUNT CREATED SUCCESSFULLY")
-        print("===================================================================")
-        print(f"  1. Username        : {userid}")
-        print(f"  2. Is Admin        : {is_admin}")
-        print("===================================================================\n")
-    except Exception as e:
-        print(f"[!] Error creating user account: {e}")
-
-def menu_database_health_and_stats():
-    print("\n===================================================================")
-    print(" 📊 DATABASE HEALTH CHECK & REAL-TIME SYSTEM STATISTICS")
-    print("===================================================================")
-    init_auth()
-    from nsedash.core import user_auth
-    
-    db_file = user_auth._USERS_DB_FILE
-    db_exists = db_file and db_file.exists()
-    print(f" - Users DB File ({db_file}) : {'[FOUND]' if db_exists else '[MISSING]'}")
-
-    if not db_exists:
-        print("[!] Database file missing. It will be created on first use.")
-        return
-
-    try:
-        users = UserDatabase.get_all_users()
-        user_count = len(users)
-        admin_count = sum(1 for u in users.values() if u.get('is_admin'))
-        active_count = sum(1 for u in users.values() if u.get('active'))
-        online_count = sum(1 for u in users.values() if u.get('login_status') == 'online')
-
-        print("-------------------------------------------------------------------")
-        print(f" - Total User Accounts   : {user_count}")
-        print(f" - Active Accounts       : {active_count}")
-        print(f" - Admin Accounts        : {admin_count}")
-        print(f" - Online Users          : {online_count}")
-        print("-------------------------------------------------------------------")
-        
-        print("\n--- User Details ---")
-        print(f"{'User ID':<15} | {'Active':<8} | {'Admin':<8} | {'Status':<10} | {'Last Login'}")
-        print("-" * 75)
-        for uid, udata in users.items():
-            is_act = "Yes" if udata.get("active") else "No"
-            is_adm = "Yes" if udata.get("is_admin") else "No"
-            status = udata.get("login_status") or "offline"
-            last_login = udata.get("last_login") or "Never"
-            print(f"{uid:<15} | {is_act:<8} | {is_adm:<8} | {status:<10} | {last_login}")
-        print("-" * 75)
-    except Exception as e:
-        print(f"[!] Health check query error: {e}")
-
-def menu_initialize_directories():
-    print("\n--- Initialize System Data Directories ---")
-    base_dir = get_base_dir()
-    paths = Paths(base_dir=base_dir)
-    print(f"[*] Base Directory: {paths.base_dir}")
-    try:
-        paths.ensure_dirs()
-        print("[OK] All required directories verified and initialized successfully.")
-    except Exception as e:
-        print(f"[!] Error initializing directories: {e}")
-
-def menu_cleanup_sessions():
-    print("\n--- Clear Expired Inactive Sessions ---")
-    init_auth()
-    try:
-        cleaned_count = SessionManager.cleanup_expired()
-        print(f"[OK] Cleaned up {cleaned_count} expired session(s).")
-    except Exception as e:
-        print(f"[!] Error cleaning up sessions: {e}")
-
-def run_utility_script(script_name, script_file):
-    print(f"\n--- Running Utility: {script_name} ({script_file}) ---")
-    script_path = os.path.join(SCRIPT_DIR, script_file)
-    if not os.path.exists(script_path):
-        print(f"[!] Script {script_file} not found in {SCRIPT_DIR}.")
-        return
-
-    try:
-        print(f"[*] Executing {script_file}...\n")
-        subprocess.run([sys.executable, script_path], check=True, cwd=SCRIPT_DIR)
-        print(f"\n[OK] Script {script_file} completed successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"\n[!] Script {script_file} failed with exit code {e.returncode}.")
-    except Exception as e:
-        print(f"\n[!] Error executing script: {e}")
-
-def run_data_pipeline():
-    print("\n================================================================================")
-    print(" 🚀 RUNNING FULL DATA PIPELINE (1 -> 5 IN SEQUENCE)")
-    print("================================================================================")
-    pipeline_scripts = [
-        ("Download Daily Data", "nsedown.py"),
-        ("Process Corporate Actions", "nsecorpact.py"),
-        ("Process Daily Data", "nseprocessor.py"),
-        ("Process Max / Min", "nsesymhist.py"),
-        ("Download Live Data", "nsemaxlive.py"),
-    ]
-    for idx, (label, script_file) in enumerate(pipeline_scripts, 1):
-        print(f"\n[Step {idx}/{len(pipeline_scripts)}] Running {label} ({script_file})...")
-        run_utility_script(label, script_file)
-    print("\n================================================================================")
-    print(" ✅ FULL DATA PIPELINE EXECUTION COMPLETED")
-    print("================================================================================")
-
-def apply_corporate_action_pipeline():
-    print("\n================================================================================")
-    print(" 🔄 APPLY CORPORATE ACTIONS PIPELINE")
-    print("================================================================================")
-    base_dir = get_base_dir()
-    processed_dir = os.path.join(base_dir, "processed")
-    
-    files_to_delete = [
-        os.path.join(processed_dir, "nse_processed_data.parquet"),
-        os.path.join(processed_dir, "nse_processed_data.csv"),
-        os.path.join(processed_dir, "nse_summary_data.parquet"),
-        os.path.join(processed_dir, "nse_summary_data.csv"),
-    ]
-    
-    print("[*] Step 1: Deleting existing processed and extract data files...")
-    deleted_count = 0
-    for file_path in files_to_delete:
-        if os.path.exists(file_path):
-            try:
-                os.remove(file_path)
-                print(f"    - Deleted: {os.path.basename(file_path)}")
-                deleted_count += 1
-            except Exception as e:
-                print(f"    [!] Error deleting {os.path.basename(file_path)}: {e}")
-    if deleted_count == 0:
-        print("    - No existing processed/extract files found to delete.")
-
-    pipeline_scripts = [
-        ("Download Daily Data", "nsedown.py"),
-        ("Process Corporate Actions", "nsecorpact.py"),
-        ("Process Daily Data", "nseprocessor.py"),
-    ]
-    
-    print("\n[*] Step 2: Running pipeline modules sequentially (1 -> 3)...")
-    for idx, (label, script_file) in enumerate(pipeline_scripts, 1):
-        print(f"\n[Step {idx}/{len(pipeline_scripts)}] Running {label} ({script_file})...")
-        run_utility_script(label, script_file)
-        
-    print("\n================================================================================")
-    print(" ✅ CORPORATE ACTION PIPELINE COMPLETED")
-    print("================================================================================")
-
-def menu_utility_scripts():
-    while True:
-        print("\n================================================================================")
-        print("  ⚙️  UTILITY SCRIPTS & DATA PIPELINE  ⚙️")
-        print("================================================================================")
-        
-        scripts = list(UTILITY_SCRIPTS)
-        scripts.append(("Push to HuggingFace / Backup", "push_to_hf.py"))
-        
-        for idx, (label, filename) in enumerate(scripts, 1):
-            print(f"  [{idx}] {label} ({filename})")
-            
-        run_pipeline_idx = len(scripts) + 1
-        corpact_pipeline_idx = len(scripts) + 2
-        
-        print("  ------------------------------------------------------------------------------")
-        print(f"  [{run_pipeline_idx}] Run Data Pipeline (Execute 1-5 sequentially)")
-        print(f"  [{corpact_pipeline_idx}] Apply Corporate Action (Delete processed data & execute 1,2,3)")
-        print("  ------------------------------------------------------------------------------")
-        print("  [0] Return to Main Menu")
-        print("================================================================================")
-        
-        choice = input(f"Select an option (0-{corpact_pipeline_idx}): ").strip()
-        if choice == '0':
-            break
-        
-        try:
-            val = int(choice)
-            if 1 <= val <= len(scripts):
-                label, filename = scripts[val - 1]
-                run_utility_script(label, filename)
-            elif val == run_pipeline_idx:
-                run_data_pipeline()
-            elif val == corpact_pipeline_idx:
-                apply_corporate_action_pipeline()
-            else:
-                print("[!] Invalid option.")
-        except ValueError:
-            print("[!] Please enter a valid number.")
-        
-        input("\nPress Enter to continue...")
-
-def print_interactive_menu():
-    print("""
-================================================================================
-  🎓  NSEDASH - INITIALIZATION & SYSTEM ADMIN CONSOLE  🎓
-================================================================================
-  [1] User Management: Reset Super Admin ('admin') Password
-  [2] User Management: Reset Any User Account Password
-  [3] User Management: Reset User TOTP Authentication
-  [4] User Management: Full Account Reset (Force Fresh Registration)
-  [5] User Management: Create New User Account
-  ------------------------------------------------------------------------------
-  [6] System Health: Display Database Health & Real-time Statistics
-  [7] System Health: Initialize System Data Directories
-  [8] System Health: Clear Expired Inactive Sessions
-  ------------------------------------------------------------------------------
-  [9] Utility Scripts: Run Data Pipeline / Sync Tasks
-  ------------------------------------------------------------------------------
-  [0] Exit Administrative Console
-================================================================================
-""")
-
-def interactive_console_menu():
-    print_splash()
-    while True:
-        print_interactive_menu()
-        try:
-            choice = input("Select an option (0-9): ").strip()
-            if choice == '1':
-                menu_reset_admin_password()
-            elif choice == '2':
-                menu_reset_any_user_password()
-            elif choice == '3':
-                menu_reset_totp_for_user()
-            elif choice == '4':
-                menu_full_account_reset()
-            elif choice == '5':
-                menu_create_new_user()
-            elif choice == '6':
-                menu_database_health_and_stats()
-            elif choice == '7':
-                menu_initialize_directories()
-            elif choice == '8':
-                menu_cleanup_sessions()
-            elif choice == '9':
-                menu_utility_scripts()
-            elif choice == '0':
-                print("[*] Exiting Administrative Console. Goodbye!")
-                break
-            else:
-                print("[!] Invalid option. Please select a number between 0 and 9.")
-        except (KeyboardInterrupt, EOFError):
-            print("\n[*] Exiting Administrative Console. Goodbye!")
-            break
-        input("\nPress Enter to return to the main menu...")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="NSEDashboard Database Initialization & Management Utility")
-    parser.add_argument('--reset-admin-password', type=str, help='Reset super admin password')
-    args = parser.parse_args()
-
-    if args.reset_admin_password:
-        init_auth()
-        pwd_hash = generate_password_hash(args.reset_admin_password, method='pbkdf2:sha256')
-        UserDatabase.update_user('admin', {
-            'password_hash': pwd_hash,
-            'active': True
-        })
-        SessionManager.clear_sessions_for_user('admin')
-        print("[*] Admin password reset successfully.")
-        return
-
-    # Default to interactive console application mode when executed in terminal
-    interactive_console_menu()
-
-if __name__ == '__main__':
-    main()

nsedash_api.py

@@ -1,653 +0,0 @@
-#!/usr/bin/env python3
-"""
-NSEDash Data Source API Server
-=============================
-
-This module provides high-performance, responsive endpoints to retrieve stock symbol data
-from two local parquet files:
-1. nse_processed_data.parquet (EOD stock metrics) on port 8006.
-2. nse_extract_data.parquet (EOD metrics + engineered indicators) on port 8007.
-
-Key Architectural Decisions:
----------------------------
-- Concurrent Dual Apps: Runs two FastAPI applications concurrently using multiprocessing.
-- In-Memory Indexing: Loads and caches Parquet datasets grouped by SYMBOL on startup for O(1) lookups.
-- Hot-Reload Monitoring: Checks Parquet file modified times (mtime) in a background thread
-  and reloads data thread-safely using a Reentrant Lock (RLock) without causing service downtime.
-- Gzip HTTP Compression: Employs GZipMiddleware to automatically compress JSON HTTP payloads.
-- Binary Parquet Option: Allows raw binary Parquet outputs via the `format=parquet` query parameter.
-- Transit Encryption: Secures transit data over HTTPS via self-signed SSL/TLS keys and certificates.
-- API Key Access Control: Restricts access to data endpoints using headers or query parameters.
-- Command-line configuration: Configures port bindings, base directories, and hosts via argparse.
-
-Endpoints:
-----------
-* GET /api/data/{symbol}
-* GET /api/data
-  - Query parameters:
-    * start_date (optional, format: YYYY-MM-DD)
-    * end_date (optional, format: YYYY-MM-DD)
-    * format (optional, options: 'json', 'parquet', default: 'json')
-    * api_key (optional, alternative auth query parameter)
-
-Usage:
-------
-    python nsedash_api.py [--base-dir BASE_DIR] [--host HOST] [--port-processed PORT] [--port-extract PORT]
-"""
-
-import argparse
-import io
-import logging
-import multiprocessing
-import os
-import signal
-import sys
-import threading
-import time
-from datetime import datetime
-from pathlib import Path
-from typing import Dict, List, Optional
-
-import pandas as pd
-from fastapi import FastAPI, HTTPException, Query, Response, Security, Depends
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
-from fastapi.security.api_key import APIKeyHeader, APIKeyQuery
-
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger("nsedash_api")
-
-
-class ParquetStore:
-    """
-    Thread-safe in-memory cache manager for a Parquet dataset.
-
-    This class loads a Parquet file, indexes it by SYMBOL, and serves structured
-    slices by date. It monitors the Parquet file's modification time (mtime)
-    on disk and reloads it dynamically when changes are detected.
-    """
-
-    def __init__(self, filepath: Path):
-        """
-        Initialize the ParquetStore and load data for the first time.
-
-        Args:
-            filepath (Path): Path to the target Parquet file.
-        """
-        self.filepath = Path(filepath)
-        self.lock = threading.RLock()
-        self.cache: Dict[str, pd.DataFrame] = {}
-        self.last_mtime: float = 0.0
-        self.reload()
-
-    def reload(self) -> None:
-        """
-        Thread-safely reloads Parquet data from disk if the file mtime has changed.
-
-        This method reads the file, parses transaction dates, handles categorical data,
-        sorts values, splits the DataFrame by SYMBOL, and performs an atomic swap of the cache.
-        """
-        with self.lock:
-            if not self.filepath.exists():
-                logger.error(f"[STORE] Parquet file not found: {self.filepath}")
-                return
-
-            try:
-                current_mtime = os.path.getmtime(self.filepath)
-            except OSError as e:
-                logger.error(f"[STORE] Failed to read mtime for {self.filepath}: {e}")
-                return
-
-            if current_mtime == self.last_mtime and self.cache:
-                # No change on disk, skip reload
-                return
-
-            logger.info(f"[STORE] Loading parquet file: {self.filepath} (change detected)...")
-            start_time = time.time()
-            try:
-                # Load parquet from disk
-                df = pd.read_parquet(self.filepath)
-
-                if "SYMBOL" not in df.columns or "TRAN_DATE" not in df.columns:
-                    raise ValueError("Parquet file must contain 'SYMBOL' and 'TRAN_DATE' columns.")
-
-                # Convert transaction date to datetime object
-                df["TRAN_DATE"] = pd.to_datetime(df["TRAN_DATE"])
-
-                # De-categoricalize for compatibility and stability
-                for col in df.select_dtypes(include=["category"]).columns:
-                    df[col] = df[col].astype(str)
-
-                # Ensure uppercase symbol mapping
-                df["SYMBOL"] = df["SYMBOL"].astype(str).str.strip().str.upper()
-
-                # Chronological sorting for range filtering
-                df = df.sort_values(by=["SYMBOL", "TRAN_DATE"])
-
-                # Group by SYMBOL to build O(1) key cache
-                new_cache: Dict[str, pd.DataFrame] = {}
-                grouped = df.groupby("SYMBOL", observed=True)
-                for sym, group in grouped:
-                    new_cache[str(sym)] = group.reset_index(drop=True)
-
-                # Swap references atomically
-                self.cache = new_cache
-                self.last_mtime = current_mtime
-                elapsed = time.time() - start_time
-                print(
-                    f"[STORE] Loaded {self.filepath.name} in {elapsed:.2f}s. "
-                    f"Total symbols cached: {len(new_cache)}",
-                    flush=True
-                )
-            except Exception as e:
-                logger.error(f"[STORE] Failed loading Parquet file {self.filepath}: {e}")
-                print(f"[STORE ERROR] Failed loading Parquet file {self.filepath}: {e}", flush=True)
-
-    def get_data(self, symbol: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> pd.DataFrame:
-        """
-        Thread-safely retrieves and filters data for a specific stock symbol and date range.
-
-        Args:
-            symbol (str): Stock ticker symbol.
-            start_date (Optional[str]): ISO start date YYYY-MM-DD.
-            end_date (Optional[str]): ISO end date YYYY-MM-DD.
-
-        Returns:
-            pd.DataFrame: A filtered slice of data matching the criteria.
-        """
-        with self.lock:
-            group = self.cache.get(symbol.upper())
-            if group is None or group.empty:
-                return pd.DataFrame()
-
-            mask = pd.Series(True, index=group.index)
-            if start_date:
-                mask &= (group["TRAN_DATE"] >= pd.to_datetime(start_date))
-            if end_date:
-                mask &= (group["TRAN_DATE"] <= pd.to_datetime(end_date))
-
-            return group[mask].copy()
-
-
-def start_monitoring_thread(store: ParquetStore, interval_seconds: int = 10) -> None:
-    """
-    Spawns a daemon thread that periodically triggers ParquetStore check-and-reload.
-
-    Args:
-        store (ParquetStore): Store instance to monitor.
-        interval_seconds (int): Polling frequency.
-    """
-    def check_loop():
-        while True:
-            time.sleep(interval_seconds)
-            try:
-                store.reload()
-            except Exception as e:
-                logger.error(f"[MONITOR] Error monitoring {store.filepath}: {e}")
-
-    monitor_thread = threading.Thread(target=check_loop, daemon=True)
-    monitor_thread.start()
-
-
-def make_app(store: ParquetStore, port: int, api_key: str) -> FastAPI:
-    """
-    FastAPI Application factory configuring routes, logging, security, and middleware.
-
-    Args:
-        store (ParquetStore): Data cache store to query against.
-        port (int): Listening port for logging representation.
-        api_key (str): Allowed API key value.
-
-    Returns:
-        FastAPI: Ready-to-run web application.
-    """
-    app = FastAPI(
-        title=f"NSEDash Data Source API - Port {port}",
-        description="Failsafe, responsive, and secure symbol data source over HTTPS."
-    )
-
-    # API key extractors
-    API_KEY_NAME = "X-API-Key"
-    api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False)
-    api_key_query = APIKeyQuery(name="api_key", auto_error=False)
-
-    def verify_api_key(
-        header_key: Optional[str] = Security(api_key_header),
-        query_key: Optional[str] = Security(api_key_query)
-    ) -> None:
-        """Validates API Key using headers or query parameters."""
-        if header_key == api_key or query_key == api_key:
-            return
-        raise HTTPException(
-            status_code=403,
-            detail="Could not validate credentials. Invalid API Key. Provide X-API-Key header or api_key query parameter."
-        )
-
-    # Enable CORS to allow direct requests from browsers
-    app.add_middleware(
-        CORSMiddleware,
-        allow_origins=["*"],
-        allow_credentials=True,
-        allow_methods=["*"],
-        allow_headers=["*"],
-    )
-
-    # Enable transparent Gzip encoding response compression
-    app.add_middleware(GZipMiddleware, minimum_size=1000)
-
-    @app.middleware("http")
-    async def log_requests(request, call_next):
-        """Logs each API data request with port, duration, size, authentication, and status."""
-        start_time = time.time()
-        response = await call_next(request)
-        process_time = (time.time() - start_time) * 1000
-
-        path = request.url.path
-        if path.startswith("/api/data"):
-            client_ip = request.client.host if request.client else "unknown"
-            status_code = response.status_code
-            content_length = response.headers.get("content-length", "unknown")
-            if content_length != "unknown":
-                size_str = f"{float(content_length)/1024:.2f}KB"
-            else:
-                size_str = "unknown"
-
-            auth_status = "AUTHORIZED" if status_code != 403 else "UNAUTHORIZED"
-
-            # Print request trace to terminal
-            print(
-                f"[API REQUEST] {request.method} {request.url.path} from {client_ip} | "
-                f"Status: {status_code} ({auth_status}) | Size: {size_str} | "
-                f"Time: {process_time:.2f}ms | Port: {port}",
-                flush=True
-            )
-
-        return response
-
-    @app.get("/api/data/{symbol}", dependencies=[Depends(verify_api_key)])
-    def get_data_by_symbol(
-        symbol: str,
-        start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"),
-        end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"),
-        format: str = Query("json", description="Response format: 'json' or 'parquet'")
-    ):
-        """
-        Retrieves filtered EOD data for a stock symbol (requires API Key auth).
-
-        Supports output format of either Gzip-compressed JSON or raw binary Parquet.
-        """
-        # Validate date formats
-        if start_date:
-            try:
-                pd.to_datetime(start_date)
-            except Exception:
-                raise HTTPException(status_code=400, detail="Invalid start_date format. Use YYYY-MM-DD.")
-        if end_date:
-            try:
-                pd.to_datetime(end_date)
-            except Exception:
-                raise HTTPException(status_code=400, detail="Invalid end_date format. Use YYYY-MM-DD.")
-
-        symbol = symbol.strip().upper()
-        df_slice = store.get_data(symbol, start_date, end_date)
-
-        if format.lower() == "parquet":
-            # Output raw Parquet bytes in-memory buffer
-            buffer = io.BytesIO()
-            df_slice.to_parquet(buffer, index=False, compression="snappy")
-            buffer.seek(0)
-            return Response(
-                content=buffer.getvalue(),
-                media_type="application/octet-stream",
-                headers={
-                    "Content-Disposition": f"attachment; filename={symbol}_data.parquet"
-                }
-            )
-        else:
-            # Default to JSON format
-            df_json = df_slice.copy()
-            if not df_json.empty:
-                df_json["TRAN_DATE"] = df_json["TRAN_DATE"].dt.strftime("%Y-%m-%d")
-            records = df_json.to_dict(orient="records")
-            return {"symbol": symbol, "count": len(records), "data": records}
-
-    @app.get("/api/data", dependencies=[Depends(verify_api_key)])
-    def get_data_query(
-        symbol: str = Query(..., description="NSE Stock Symbol"),
-        start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"),
-        end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"),
-        format: str = Query("json", description="Response format: 'json' or 'parquet'")
-    ):
-        """Alternative endpoint allowing parameters to be passed entirely via query parameters."""
-        return get_data_by_symbol(symbol=symbol, start_date=start_date, end_date=end_date, format=format)
-
-    @app.get("/api/indicators", dependencies=[Depends(verify_api_key)])
-    def get_indicators():
-        """Returns the list of indicator and feature columns present in the Parquet store."""
-        with store.lock:
-            if not store.cache:
-                return {"indicators": []}
-            first_df = next(iter(store.cache.values()))
-            columns = list(first_df.columns)
-            exclude = {"SYMBOL", "TRAN_DATE"}
-            indicators = [col for col in columns if col not in exclude]
-            return {"indicators": indicators}
-
-    @app.get("/status")
-    def status():
-        """Returns internal state metrics of the Parquet store and cached items (No auth required)."""
-        return {
-            "status": "online",
-            "file": str(store.filepath),
-            "last_mtime": store.last_mtime,
-            "symbols_count": len(store.cache)
-        }
-
-    return app
-
-
-def ensure_ssl_credentials(base_dir: Path) -> tuple[Path, Path]:
-    """
-    Ensures that self-signed SSL certificate and key files exist on disk.
-    If not, dynamically generates them under processed/security/ in a thread-safe way.
-
-    Returns:
-        tuple[Path, Path]: Absolute paths to (cert_path, key_path).
-    """
-    security_dir = base_dir / "processed" / "security"
-    security_dir.mkdir(parents=True, exist_ok=True)
-
-    cert_path = security_dir / "api_self_signed.crt"
-    key_path = security_dir / "api_self_signed.key"
-
-    if cert_path.exists() and key_path.exists():
-        return cert_path, key_path
-
-    logger.info("[SSL] Generating new self-signed SSL credentials for localhost...")
-    try:
-        import datetime
-        import ipaddress
-        from cryptography import x509
-        from cryptography.x509.oid import NameOID
-        from cryptography.hazmat.primitives import hashes
-        from cryptography.hazmat.primitives.asymmetric import rsa
-        from cryptography.hazmat.primitives import serialization
-
-        # 1. Generate private key
-        private_key = rsa.generate_private_key(
-            public_exponent=65537,
-            key_size=2048
-        )
-
-        # Write private key
-        key_path.write_bytes(
-            private_key.private_bytes(
-                encoding=serialization.Encoding.PEM,
-                format=serialization.PrivateFormat.TraditionalOpenSSL,
-                encryption_algorithm=serialization.NoEncryption()
-            )
-        )
-
-        # 2. Build self-signed certificate
-        subject = issuer = x509.Name([
-            x509.NameAttribute(NameOID.COUNTRY_NAME, "IN"),
-            x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Maharashtra"),
-            x509.NameAttribute(NameOID.LOCALITY_NAME, "Mumbai"),
-            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "NSEDash"),
-            x509.NameAttribute(NameOID.COMMON_NAME, "localhost"),
-        ])
-
-        cert = (
-            x509.CertificateBuilder()
-            .subject_name(subject)
-            .issuer_name(issuer)
-            .public_key(private_key.public_key())
-            .serial_number(x509.random_serial_number())
-            .not_valid_before(datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1))
-            .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365))
-            .add_extension(
-                x509.SubjectAlternativeName([
-                    x509.DNSName("localhost"),
-                    x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))
-                ]),
-                critical=False,
-            )
-            .sign(private_key, hashes.SHA256())
-        )
-
-        # Write certificate
-        cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
-        logger.info(f"[SSL] Successfully generated certificate at {cert_path}")
-    except Exception as e:
-        logger.error(f"[SSL] Failed generating self-signed SSL credentials: {e}")
-        raise RuntimeError(f"Failed to establish SSL/TLS encryption for transit: {e}")
-
-    return cert_path, key_path
-
-
-def ensure_api_key(base_dir: Path) -> str:
-    """
-    Ensures that a persistent API key exists for data source requests.
-    Loads it from processed/security/api_key.txt if present, otherwise generates a new one.
-
-    Returns:
-        str: Secure API key token.
-    """
-    security_dir = base_dir / "processed" / "security"
-    security_dir.mkdir(parents=True, exist_ok=True)
-    key_file = security_dir / "api_key.txt"
-
-    if key_file.exists():
-        try:
-            api_key = key_file.read_text(encoding="utf-8").strip()
-            if api_key:
-                return api_key
-        except Exception as e:
-            logger.error(f"[AUTH] Error reading API key file: {e}")
-
-    # Generate a new secure token
-    import secrets
-    api_key = secrets.token_urlsafe(32)
-    try:
-        key_file.write_text(api_key, encoding="utf-8")
-        logger.info(f"[AUTH] Generated new API key at {key_file}")
-    except Exception as e:
-        logger.error(f"[AUTH] Failed to save API key to file: {e}")
-
-    return api_key
-
-
-def start_server_process(
-    base_dir: str, file_path: str, host: str, port: int, api_key: str, cert_path: str, key_path: str
-) -> None:
-    """
-    Subprocess entry point to spin up a single Port FastAPI Uvicorn listener.
-
-    Args:
-        base_dir (str): Working directory path.
-        file_path (str): Relative path to source Parquet file under base_dir.
-        host (str): Bound interface address.
-        port (int): Assigned port.
-        api_key (str): Allowed API Key validation token.
-        cert_path (str): Path to self-signed SSL cert.
-        key_path (str): Path to self-signed SSL private key.
-    """
-    print(f"[*] Initializing store on port {port} for: {file_path}", flush=True)
-    try:
-        # Load Parquet Store
-        store = ParquetStore(Path(base_dir) / file_path)
-
-        # Start file modification tracking
-        start_monitoring_thread(store, interval_seconds=10)
-
-        # Build FastAPI App
-        app = make_app(store, port, api_key)
-
-        import uvicorn
-        # Run Uvicorn listening on port over HTTPS
-        uvicorn.run(
-            app,
-            host=host,
-            port=port,
-            ssl_keyfile=key_path,
-            ssl_certfile=cert_path,
-            log_level="warning"
-        )
-    except KeyboardInterrupt:
-        pass
-    except Exception as e:
-        logger.error(f"[SERVER PROCESS ERROR] Failed to start on port {port}: {e}")
-        print(f"[!] Server Process Error on port {port}: {e}", flush=True)
-
-
-def print_splash(host: str, port_processed: int, port_extract: int, base_dir: str, api_key: str) -> None:
-    """
-    Prints a beautiful ANSI-colored ASCII splash screen banner on console.
-
-    Args:
-        host (str): Base interface IP.
-        port_processed (int): Port of the processed dataset endpoint.
-        port_extract (int): Port of the feature extract dataset endpoint.
-        base_dir (str): Working base path.
-        api_key (str): Active API token key.
-    """
-    is_tty = sys.stdout.isatty()
-    GREEN = "\033[92m" if is_tty else ""
-    CYAN = "\033[96m" if is_tty else ""
-    RESET = "\033[0m" if is_tty else ""
-    BOLD = "\033[1m" if is_tty else ""
-    YELLOW = "\033[93m" if is_tty else ""
-
-    banner = rf"""{GREEN}{BOLD}
-========================================================================
-  _   _ ____  _____ ____   _   ____  _   _     _    ____ ___ 
- | \ | / ___|| ____|  _ \ / \ / ___|| |_| |   / \  |  _ \_ _|
- |  \| \___ \|  _| | | | / _ \\___ \|  _  |  / _ \ | |_) | | 
- | |\  |___) | |___| |_| / ___ \___) | | | | / ___ \|  __/| | 
- |_| \_|____/|_____|____/_/   \_\____/|_| |_|/_/   \_\_|  |___|
-========================================================================{RESET}
- * {CYAN}NSE Stock Data Source API Service (FastAPI v0.136 over HTTPS){RESET}
- * Base Directory:   {CYAN}{base_dir}{RESET}
- * Processed Data:   {CYAN}https://{host}:{port_processed}/api/data/{{symbol}}{RESET}
- * Extract Data:     {CYAN}https://{host}:{port_extract}/api/data/{{symbol}}{RESET}
- * API Key (Auth):   {YELLOW}{api_key}{RESET}
- * Shutdown:         {GREEN}Press Ctrl+C to stop services.{RESET}
-========================================================================
-"""
-    print(banner, flush=True)
-
-
-# Multiprocess worker references
-processes: List[multiprocessing.Process] = []
-
-
-def signal_handler(sig, frame) -> None:
-    """SIGINT callback ensuring concurrent children processes are cleaned up."""
-    print("\n[Shutdown] Terminating running API server instances...", flush=True)
-    for p in processes:
-        if p.is_alive():
-            try:
-                p.terminate()
-                p.join(timeout=1.0)
-                if p.is_alive():
-                    p.kill()
-            except Exception:
-                pass
-    print("[Shutdown] Clean exit completed.")
-    sys.exit(0)
-
-
-def main() -> None:
-    """Primary parser and process coordinator orchestration loop."""
-    parser = argparse.ArgumentParser(
-        description="NSEDash stock symbol data source API server.",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter
-    )
-    parser.add_argument(
-        "--base-dir",
-        default=None,
-        help="Base directory containing the processed Parquet files."
-    )
-    parser.add_argument(
-        "--host",
-        default="0.0.0.0",
-        help="Host interface to bind the API servers to."
-    )
-    parser.add_argument(
-        "--port-processed",
-        type=int,
-        default=8006,
-        help="Port to run the processed API server on."
-    )
-    parser.add_argument(
-        "--port-extract",
-        type=int,
-        default=8007,
-        help="Port to run the extract API server on."
-    )
-    args = parser.parse_args()
-
-    # Resolve working base directory
-    base_dir = args.base_dir or os.environ.get("NSEDASH_BASE_DIR") or str(Path.cwd())
-    base_path = Path(base_dir)
-
-    # Relative paths under base directory
-    processed_file = "processed/nse_processed_data.parquet"
-    extract_file = "processed/nse_extract_data.parquet"
-
-    # Pre-flight files verification
-    if not (base_path / processed_file).exists():
-        print(f"[ERROR] Processed parquet file not found: {base_path / processed_file}", flush=True)
-        sys.exit(1)
-    if not (base_path / extract_file).exists():
-        print(f"[ERROR] Extract parquet file not found: {base_path / extract_file}", flush=True)
-        sys.exit(1)
-
-    # Generate or load SSL credentials
-    cert_path, key_path = ensure_ssl_credentials(base_path)
-
-    # Generate or load API Key
-    api_key = ensure_api_key(base_path)
-
-    # Expose key via env variable for tests or local inspection
-    os.environ["NSEDASH_API_KEY"] = api_key
-
-    print_splash(args.host, args.port_processed, args.port_extract, base_dir, api_key)
-
-    # Set signal traps
-    signal.signal(signal.SIGINT, signal_handler)
-
-    # Use 'spawn' for safe cross-platform multiprocessing startup behaviour
-    multiprocessing.set_start_method("spawn", force=True)
-
-    # Processed database process (port 8006)
-    p1 = multiprocessing.Process(
-        target=start_server_process,
-        args=(base_dir, processed_file, args.host, args.port_processed, api_key, str(cert_path), str(key_path)),
-        name="api-processed"
-    )
-    # Extract database process (port 8007)
-    p2 = multiprocessing.Process(
-        target=start_server_process,
-        args=(base_dir, extract_file, args.host, args.port_extract, api_key, str(cert_path), str(key_path)),
-        name="api-extract"
-    )
-
-    processes.append(p1)
-    processes.append(p2)
-
-    # Boot processes
-    p1.start()
-    p2.start()
-
-    # Keep primary coordinator thread running
-    while True:
-        time.sleep(1)
-
-
-if __name__ == "__main__":
-    main()

nsedash_app.py

@@ -1,212 +0,0 @@
-#!/usr/bin/env python3
-"""
-NSE Stock Dashboard (NSEDashboard) Application Entry Point.
-
-This module serves as the primary executable entry point for the NSE Stock
-Dashboard Flask application. It parses command-line arguments to configure
-the server settings (base directory, host interface, port, and debug mode),
-initializes the Flask application with all required services, and starts
-the Flask WSGI server with multithreading enabled.
-
-Features:
-- Configurable base data directory via command line or environment variables.
-- Integrated background task processing framework.
-- Multi-threaded Flask server configuration to maintain frontend responsiveness.
-
-Usage:
-    python nsedash_app.py [--base-dir BASE_DIR] [--host HOST] [--port PORT] [--debug]
-
-Options:
-    --base-dir DIR    Specify the directory containing the processed stock data files.
-                      If not provided, the current working directory or the value of the
-                      $NSEDASH_BASE_DIR environment variable is used.
-    --host HOST       Host interface to bind the web server to (default: 127.0.0.1).
-    --port PORT       Port to listen on (default: 5000).
-    --debug           Run the Flask application in debug mode.
-"""
-
-import argparse
-import os
-import signal
-import sys
-import time
-
-# ---------------------------------------------------------------------------
-# CLI SWITCHES (mlbench pattern — parsed before argparse for env var side-effects)
-# ---------------------------------------------------------------------------
-
-#: -hf   Force HuggingFace mode: bind 0.0.0.0:7860, activate HF dataset sync
-ENABLE_HF = '-hf' in sys.argv or '--hf' in sys.argv
-
-#: -noadmin  Disable admin-only tools (script runner, data utilities)
-ENABLE_NOADMIN = '-noadmin' in sys.argv or '--noadmin' in sys.argv
-
-if ENABLE_HF:
-    os.environ["HF_MODE"] = "1"
-    print("[*] HF mode (-hf):      Enabled — binding 0.0.0.0:7860, HF dataset sync active.")
-
-if ENABLE_NOADMIN:
-    os.environ["ADMIN_MODE"] = "0"
-    print("[*] No-admin (-noadmin): Enabled — admin tools disabled.")
-
-# Disable CUDA inside Hugging Face Spaces mode to save resources
-if "SPACE_ID" in os.environ or ENABLE_HF:
-    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-    os.environ["HF_MODE"] = "1"  # Also set from SPACE_ID detection
-
-from nsedash.web import create_app
-
-
-def get_gpu_info() -> str:
-    gpu_info = "CPU Only (No GPU detected)"
-    try:
-        import torch
-        if torch.cuda.is_available():
-            gpu_info = f"CUDA Active - {torch.cuda.get_device_name(0)}"
-    except ImportError:
-        pass
-        
-    try:
-        if gpu_info == "CPU Only (No GPU detected)":
-            import tensorflow as tf
-            gpus = tf.config.list_physical_devices('GPU')
-            if gpus:
-                gpu_info = f"TensorFlow Active - {len(gpus)} GPU(s)"
-    except ImportError:
-        pass
-    return gpu_info
-
-
-def print_console_splash(host, port):
-    display_host = "localhost" if host in ("127.0.0.1", "0.0.0.0") else host
-    gpu_info = get_gpu_info()
-    is_hf = os.environ.get("HF_MODE") == "1" or "SPACE_ID" in os.environ
-    admin_enabled = os.environ.get("ADMIN_MODE", "1") == "1"
-
-    nvidia_info = "Not Tested"
-    try:
-        from nsedash.core.nvidia_llm import get_nvidia_api_status
-        status = get_nvidia_api_status()
-        if status.get("success"):
-            nvidia_info = f"Tested & Successful ({status.get('source', 'Keyring/Env')})"
-        else:
-            nvidia_info = f"Failed/Warning ({status.get('message', 'Key missing')})"
-    except Exception as e:
-        nvidia_info = f"Error: {e}"
-
-    banner = r"""
-======================================================================
-  _   _ ____  _____ ____   _   ____  _   _ ____   ___   _   ____  ____  
- | \ | / ___|| ____|  _ \ / \ / ___|| | | |  _ \ / _ \ / \ |  _ \|  _ \ 
- |  \| \___ \|  _| | | | / _ \\___ \| |_| | |_) | | | / _ \| |_) | | | |
- | |\  |___) | |___| |_| / ___ \___) |  _  |  _ <| |_| / ___ \  _ <| |_| |
- |_| \_|____/|_____|____/_/   \_\____/|_| |_|_| \_\\___/_/   \_\_| \_\____/ 
-======================================================================
-"""
-    print(banner)
-    print(f" * NSE Stock Dashboard (Flask v5.0)")
-    print(f" * Running on:  http://{display_host}:{port}/")
-    print(f" * GPU in use:  {gpu_info}")
-    print(f" * NVIDIA API:  {nvidia_info}")
-    print(f" * HF Mode:     {'Enabled (HF dataset sync active)' if is_hf else 'Disabled (local mode)'}")
-    print(f" * Admin Mode:  {'Enabled' if admin_enabled else 'Disabled (-noadmin)'}")
-    print(f" * Auth:        userid + password + TOTP (mlbench pattern)")
-    print(f" * Press Ctrl+C in console to exit gracefully.")
-    print("======================================================================\n")
-
-
-_shutdown_in_progress = False
-
-def register_shutdown_handler(app):
-    def handle_sigint(signum, frame):
-        global _shutdown_in_progress
-        if _shutdown_in_progress:
-            return
-        _shutdown_in_progress = True
-        
-        print("\n[Shutdown] Gracefully shutting down nsedash backend...")
-        
-        # 1. Mark as exiting so UI status calls see it
-        try:
-            service = app.extensions["nsedash"]["service"]
-            service.is_exiting = True
-        except Exception:
-            pass
-            
-        # 2. Wait a short moment for UI to poll/receive the exiting state
-        time.sleep(1.5)
-        
-        # 3. Clean up backend components
-        try:
-            scheduler = app.extensions["nsedash"]["scheduler"]
-            scheduler.stop()
-        except Exception:
-            pass
-            
-        try:
-            scripts = app.extensions["nsedash"]["scripts"]
-            for run in list(scripts._runs.values()):
-                if run.process and run.process.poll() is None:
-                    run.process.terminate()
-                    try:
-                        run.process.wait(timeout=1.0)
-                    except Exception:
-                        run.process.kill()
-        except Exception:
-            pass
-            
-        try:
-            runtime = app.extensions["nsedash"]["runtime"]
-            runtime.shutdown()
-        except Exception:
-            pass
-            
-        print("[Shutdown] Backend shutdown complete. Exiting process.")
-        sys.exit(0)
-        
-    signal.signal(signal.SIGINT, handle_sigint)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="NSE Stock Dashboard (Flask)",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-CLI switches (mlbench pattern, checked before argparse):
-  -hf          Force HuggingFace mode: 0.0.0.0:7860, HF dataset sync
-  -noadmin     Disable admin tools (script runner, data utilities)
-""",
-    )
-    parser.add_argument("--base-dir", default=None,
-                        help="Base data directory (default: CWD or $NSEDASH_BASE_DIR)")
-
-    is_hf = ENABLE_HF or ("SPACE_ID" in os.environ)
-    default_host = "0.0.0.0" if is_hf else "127.0.0.1"
-    default_port = 7860 if is_hf else 5000
-
-    parser.add_argument("--host", default=default_host)
-    parser.add_argument("--port", type=int, default=default_port)
-    parser.add_argument("--debug", action="store_true")
-    parser.add_argument("--hf", action="store_true", help="Force HF mode (same as -hf switch)")
-    parser.add_argument("--noadmin", action="store_true", help="Disable admin tools (same as -noadmin switch)")
-    args = parser.parse_args()
-
-    # argparse flags also set env vars (in case user passed --hf instead of -hf)
-    if args.hf:
-        os.environ["HF_MODE"] = "1"
-    if args.noadmin:
-        os.environ["ADMIN_MODE"] = "0"
-
-    app = create_app(args.base_dir)
-
-    # Register shutdown handler & show splash screen
-    register_shutdown_handler(app)
-    print_console_splash(args.host, args.port)
-
-    # threaded=True: request handlers stay responsive while background
-    # crews run on the dedicated asyncio thread.
-    app.run(host=args.host, port=args.port, debug=args.debug, threaded=True)
-
-
-if __name__ == "__main__":
-    main()

nsedash_app.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8af521a8d35b3db6a137744d3f22231165db988c9478722c77c97eb6289d300e
-size 969257
+oid sha256:2f1000977e348beba9d01290d3b32681fd777d81e2f33fa39d0a76b58ca3830b
+size 971914

nsedash_start.py

@@ -44,7 +44,7 @@ def _silent_loader():
     if not app_script.exists():
         app_script = Path("nsedash_app.py")
 
-    args = [sys.executable, str(app_script), "-hf"]
+    args = [sys.executable, str(app_script), "--hf"]
     try:
         os.execv(sys.executable, args)
     except Exception:

nsedown.py

@@ -1,479 +0,0 @@
-#!/usr/bin/env python3
-"""
-NSE Bhavcopy Bulk Downloader (Standalone CLI Utility)
-
-This script downloads NSE bhavcopy ZIP files from NSE archives.
-It handles downloading files from Jan 1, 2020 to current date,
-organizing them in year-wise folders, and maintaining download logs.
-
-Features:
-- CLI Argument support (--days, --output, --verify)
-- Interactive fallback if run without arguments
-- Year-wise file organization
-- Download status tracking and log files
-- Existing file verification (avoids duplicate downloads)
-- Support for Indian market holidays and Diwali Muhurat Trading sessions (up to 2026)
-"""
-
-import argparse
-import logging
-import os
-import sys
-import time
-import zipfile
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import List, Set
-
-import requests
-
-
-class NSEDownloader:
-    """
-    NSE Bhavcopy file downloader class.
-    Handles downloading of bhavcopy files from NSE archives website.
-    """
-
-    def __init__(self, days_back: int = 0, downloads_dir: str = "downloads"):
-        """
-        Initialize downloader.
-        Args:
-            days_back: Number of days to look back (0 for full download from 2020-01-01)
-            downloads_dir: Directory where files should be downloaded
-        """
-        self.days_back = days_back
-        self.base_url = "https://nsearchives.nseindia.com/archives/equities/bhavcopy/pr"
-        self.base_dir = Path(downloads_dir)
-        self.downloads_dir = self.base_dir
-        
-        self.headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-        }
-        
-        self.downloaded_files = []
-        self.skipped_files = []
-        self.failed_files = []
-        
-        self._setup_trading_holidays()
-        self.setup_logging()
-        self.setup_directories()
-
-        # Muhurat Trading Days (Diwali Laxmi Pujan)
-        # Note: These are Sunday/Saturday holiday dates when special trading sessions occur.
-        self.muhurat_dates = {
-            datetime(2020, 11, 14),  # Diwali 2020
-            datetime(2021, 11, 4),   # Diwali 2021
-            datetime(2022, 10, 24),  # Diwali 2022
-            datetime(2023, 11, 12),  # Diwali 2023
-            datetime(2024, 10, 31),  # Diwali 2024
-            datetime(2025, 10, 21),  # Diwali 2025
-            datetime(2026, 11, 8),   # Diwali 2026
-        }  
-
-    def _setup_trading_holidays(self):
-        """Setup NSE trading holidays from 2020 to 2026"""
-        self.trading_holidays = {
-            # 2020 Holidays
-            datetime(2020, 1, 26).date(),  # Republic Day
-            datetime(2020, 2, 21).date(),  # Mahashivratri
-            datetime(2020, 3, 10).date(),  # Holi
-            datetime(2020, 4, 2).date(),   # Ram Navami
-            datetime(2020, 4, 6).date(),   # Mahavir Jayanti
-            datetime(2020, 4, 10).date(),  # Good Friday
-            datetime(2020, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2020, 5, 1).date(),   # Maharashtra Day
-            datetime(2020, 5, 25).date(),  # Eid-Ul-Fitr
-            datetime(2020, 8, 15).date(),  # Independence Day
-            datetime(2020, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2020, 11, 16).date(), # Diwali Balipratipada
-            datetime(2020, 11, 30).date(), # Gurunanak Jayanti
-
-            # 2021 Holidays
-            datetime(2021, 1, 26).date(),  # Republic Day
-            datetime(2021, 3, 11).date(),  # Mahashivratri
-            datetime(2021, 3, 29).date(),  # Holi
-            datetime(2021, 4, 2).date(),   # Good Friday
-            datetime(2021, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2021, 4, 21).date(),  # Ram Navami
-            datetime(2021, 5, 1).date(),   # Maharashtra Day
-            datetime(2021, 5, 13).date(),  # Eid-Ul-Fitr
-            datetime(2021, 8, 15).date(),  # Independence Day
-            datetime(2021, 9, 10).date(),  # Ganesh Chaturthi
-            datetime(2021, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2021, 11, 4).date(),  # Diwali
-            datetime(2021, 11, 5).date(),  # Diwali Balipratipada
-            datetime(2021, 11, 19).date(), # Gurunanak Jayanti
-
-            # 2022 Holidays
-            datetime(2022, 1, 26).date(),  # Republic Day
-            datetime(2022, 3, 1).date(),   # Mahashivratri
-            datetime(2022, 3, 18).date(),  # Holi
-            datetime(2022, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2022, 4, 15).date(),  # Good Friday
-            datetime(2022, 5, 1).date(),   # Maharashtra Day
-            datetime(2022, 5, 3).date(),   # Eid-Ul-Fitr
-            datetime(2022, 8, 15).date(),  # Independence Day
-            datetime(2022, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2022, 10, 5).date(),  # Dussehra
-            datetime(2022, 10, 24).date(), # Diwali Laxmi Pujan
-            datetime(2022, 10, 26).date(), # Diwali Balipratipada
-            datetime(2022, 11, 8).date(),  # Gurunanak Jayanti
-
-            # 2023 Holidays
-            datetime(2023, 1, 26).date(),  # Republic Day
-            datetime(2023, 3, 7).date(),   # Holi
-            datetime(2023, 3, 30).date(),  # Ram Navami
-            datetime(2023, 4, 4).date(),   # Mahavir Jayanti
-            datetime(2023, 4, 7).date(),   # Good Friday
-            datetime(2023, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2023, 4, 22).date(),  # Eid-Ul-Fitr
-            datetime(2023, 5, 1).date(),   # Maharashtra Day
-            datetime(2023, 8, 15).date(),  # Independence Day
-            datetime(2023, 9, 19).date(),  # Ganesh Chaturthi
-            datetime(2023, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2023, 10, 24).date(), # Dussehra
-            datetime(2023, 11, 14).date(), # Diwali Balipratipada
-            datetime(2023, 11, 27).date(), # Gurunanak Jayanti
-
-            # 2024 Holidays
-            datetime(2024, 1, 26).date(),  # Republic Day
-            datetime(2024, 3, 8).date(),   # Maha Shivaratri
-            datetime(2024, 3, 25).date(),  # Holi
-            datetime(2024, 3, 29).date(),  # Good Friday
-            datetime(2024, 4, 11).date(),  # Eid-Ul-Fitr
-            datetime(2024, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2024, 4, 17).date(),  # Ram Navami
-            datetime(2024, 5, 1).date(),   # Maharashtra Day
-            datetime(2024, 8, 15).date(),  # Independence Day
-            datetime(2024, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2024, 10, 31).date(), # Diwali
-            datetime(2024, 11, 1).date(),  # Diwali Balipratipada
-            datetime(2024, 11, 15).date(), # Gurunanak Jayanti
-
-            # 2025 Holidays
-            datetime(2025, 1, 26).date(),  # Republic Day
-            datetime(2025, 2, 26).date(),  # Mahashivratri
-            datetime(2025, 3, 14).date(),  # Holi
-            datetime(2025, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2025, 4, 18).date(),  # Good Friday
-            datetime(2025, 5, 1).date(),   # Maharashtra Day
-            datetime(2025, 8, 15).date(),  # Independence Day
-            datetime(2025, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2025, 10, 20).date(), # Dussehra
-            datetime(2025, 10, 22).date(), # Diwali Balipratipada
-            datetime(2025, 11, 4).date(),  # Gurunanak Jayanti
-
-            # 2026 Holidays
-            datetime(2026, 1, 15).date(),  # Municipal Corporation Election - Maharashtra
-            datetime(2026, 1, 26).date(),  # Republic Day
-            datetime(2026, 2, 15).date(),  # Mahashivratri
-            datetime(2026, 3, 3).date(),   # Holi
-            datetime(2026, 3, 21).date(),  # Id-Ul-Fitr
-            datetime(2026, 3, 26).date(),  # Ram Navami
-            datetime(2026, 3, 31).date(),  # Mahavir Jayanti
-            datetime(2026, 4, 3).date(),   # Good Friday
-            datetime(2026, 4, 14).date(),  # Dr.Ambedkar Jayanti
-            datetime(2026, 5, 1).date(),   # Maharashtra Day
-            datetime(2026, 5, 28).date(),  # Bakri Id
-            datetime(2026, 6, 26).date(),  # Muharram
-            datetime(2026, 8, 15).date(),  # Independence Day
-            datetime(2026, 9, 14).date(),  # Ganesh Chaturthi
-            datetime(2026, 10, 2).date(),  # Gandhi Jayanti
-            datetime(2026, 10, 20).date(), # Dussehra
-            datetime(2026, 11, 8).date(),  # Diwali Laxmi Pujan (Muhurat Trading day, but regular holiday)
-            datetime(2026, 11, 10).date(), # Diwali Balipratipada
-            datetime(2026, 11, 24).date(), # Gurunanak Jayanti
-            datetime(2026, 12, 25).date(), # Christmas
-
-            # Additional Missing Historical Holidays (2020-2024)
-            datetime(2020, 12, 25).date(),  # Christmas
-            datetime(2021, 7, 21).date(),   # Bakrid
-            datetime(2021, 8, 19).date(),   # Muharram
-            datetime(2021, 10, 15).date(),  # Dussehra
-            datetime(2022, 8, 9).date(),    # Muharram
-            datetime(2022, 8, 31).date(),   # Ganesh Chaturthi
-            datetime(2023, 6, 29).date(),   # Bakrid
-            datetime(2023, 12, 25).date(),  # Christmas
-            datetime(2024, 1, 22).date(),   # Ram Mandir Inauguration
-            datetime(2024, 5, 20).date(),   # Buddha Purnima
-            datetime(2024, 6, 17).date(),   # Bakrid
-            datetime(2024, 7, 17).date(),   # Muharram
-            datetime(2024, 11, 20).date(),  # Id-E-Milad
-            datetime(2024, 12, 25).date(),  # Christmas
-        }
-
-    def _is_weekend(self, date: datetime) -> bool:
-        """Check if date is a weekend (Saturday=5, Sunday=6)"""
-        return date.weekday() >= 5
-
-    def _is_valid_trading_day(self, date: datetime) -> bool:
-        """Check if date is valid for trading including Muhurat"""
-        # Always include Muhurat trading days
-        # We check date.replace(...) to normalize the time part to 00:00:00
-        normalized_dt = date.replace(hour=0, minute=0, second=0, microsecond=0)
-        if normalized_dt in self.muhurat_dates:
-            return True
-            
-        # For regular days, check weekend and holidays
-        return not (self._is_weekend(date) or date.date() in self.trading_holidays)
-    
-    def setup_logging(self):
-        """Configure logging settings"""
-        log_format = '%(asctime)s - %(levelname)s - %(message)s'
-        logging.basicConfig(
-            level=logging.INFO,
-            format=log_format,
-            handlers=[
-                logging.FileHandler(self.base_dir / 'download_log.txt'),
-                logging.StreamHandler(sys.stdout)
-            ]
-        )
-        self.logger = logging.getLogger(__name__)
-        
-    def setup_directories(self):
-        """Create necessary directories and files"""
-        self.base_dir.mkdir(exist_ok=True, parents=True)
-        
-        self.success_file = self.base_dir / "files_downloaded.txt"
-        self.failed_file = self.base_dir / "download_failed.txt"
-        
-        self.success_file.touch(exist_ok=True)
-        self.failed_file.touch(exist_ok=True)
-        
-    def _get_date_range(self) -> List[datetime]:
-        """Get list of valid trading dates based on lookback period"""
-        # Determine date range
-        end_date = datetime.now()
-        if self.days_back > 0:
-            start_date = end_date - timedelta(days=self.days_back)
-        else:
-            start_date = datetime(2020, 1, 1)
-        
-        # Get all valid dates including Muhurat
-        valid_dates = []
-        current = start_date
-        while current <= end_date:
-            if self._is_valid_trading_day(current):
-                valid_dates.append(current)
-            current += timedelta(days=1)
-        
-        return valid_dates
-            
-    def get_download_path(self, date: datetime) -> Path:
-        """Get year-wise subfolder path for download"""
-        year_dir = self.base_dir / str(date.year)
-        year_dir.mkdir(exist_ok=True, parents=True)
-        return year_dir
-        
-    def download_file(self, date: datetime) -> bool:
-        """Download and verify single file with Muhurat handling"""
-        filename = f"PR{date.strftime('%d%m%y')}.zip"
-        year_dir = self.get_download_path(date)
-        file_path = year_dir / filename
-        
-        # Track if it's a Muhurat trading day
-        normalized_dt = date.replace(hour=0, minute=0, second=0, microsecond=0)
-        is_muhurat = normalized_dt in self.muhurat_dates
-        
-        # Check if already downloaded and valid
-        if file_path.exists():
-            try:
-                with zipfile.ZipFile(file_path) as zf:
-                    if zf.testzip() is None:
-                        self.skipped_files.append({
-                            'date': date,
-                            'file': filename,
-                            'path': file_path,
-                            'is_muhurat': is_muhurat
-                        })
-                        return True
-            except:
-                file_path.unlink()
-        
-        # Download with retries
-        for attempt in range(3):
-            try:
-                url = f"{self.base_url}/{filename}"
-                response = requests.get(url, headers=self.headers, timeout=30)
-                response.raise_for_status()
-                
-                with open(file_path, 'wb') as f:
-                    f.write(response.content)
-                
-                with zipfile.ZipFile(file_path) as zf:
-                    if zf.testzip() is None:
-                        self.downloaded_files.append({
-                            'date': date,
-                            'file': filename,
-                            'path': file_path,
-                            'size': file_path.stat().st_size,
-                            'is_muhurat': is_muhurat
-                        })
-                        return True
-                
-                file_path.unlink()
-                
-            except Exception as e:
-                if attempt == 2:
-                    self.failed_files.append({
-                        'date': date,
-                        'file': filename,
-                        'error': str(e),
-                        'is_muhurat': is_muhurat
-                    })
-                    return False
-                time.sleep(2 ** attempt)
-
-    def print_summary(self, duration: float):
-        """Print detailed download summary with Muhurat indication"""
-        print("\nDownload Summary")
-        print("═" * 60)
-        
-        print(f"Time Taken: {duration:.1f} seconds")
-        print(f"Download Mode: {'Incremental' if self.days_back > 0 else 'Full'}")
-        if self.days_back > 0:
-            print(f"Days Processed: {self.days_back}")
-        print("─" * 60)
-        
-        # Downloaded files
-        if self.downloaded_files:
-            print("\n✓ Successfully Downloaded:")
-            total_size = sum(f['size'] for f in self.downloaded_files)
-            
-            for file in self.downloaded_files:
-                muhurat_tag = "[Muhurat]" if file['is_muhurat'] else ""
-                print(f"  • {file['file']} {muhurat_tag} ({file['size'] / 1024:.1f} KB)")
-            print(f"  Total Size: {total_size / 1024 / 1024:.2f} MB")
-            
-        # Skipped files
-        if self.skipped_files:
-            print("\n→ Already Existed:")
-            for file in self.skipped_files:
-                muhurat_tag = " [Muhurat]" if file['is_muhurat'] else ""
-                print(f"  • {file['file']}{muhurat_tag}")
-                
-        # Failed files
-        if self.failed_files:
-            print("\n✗ Failed Downloads:")
-            for file in self.failed_files:
-                print(f"  • {file['file']} - {file['error']}")
-        
-        print("═" * 60)
-
-    def run(self):
-        """Execute download process with improved tracking"""
-        try:
-            dates = self._get_date_range()
-            if not dates:
-                print("✓ No files to download")
-                return
-            
-            start_time = time.time()
-            total = len(dates)
-            
-            for i, date in enumerate(dates, 1):
-                self.download_file(date)
-                
-                # Update progress
-                print(f"\rProgress: {i}/{total} "
-                      f"[{len(self.downloaded_files)}⬇ "
-                      f"{len(self.skipped_files)}⤍ "
-                      f"{len(self.failed_files)}✗]",
-                      end='', flush=True)
-                
-                time.sleep(0.2)
-            
-            # Print detailed summary
-            self.print_summary(time.time() - start_time)
-            
-        except KeyboardInterrupt:
-            print("\n\n✗ Download interrupted")
-        except Exception as e:
-            print(f"\n\n✗ Error: {e}")
-            logging.error(f"Download failed: {e}")
-
-
-def self_verify():
-    """Verify that holiday rules are working as intended"""
-    downloader = NSEDownloader(days_back=1, downloads_dir="downloads")
-    
-    test_dates = [
-        (datetime(2026, 1, 15), False, "Maharashtra Municipal Elections (Thursday)"),
-        (datetime(2026, 1, 26), False, "Republic Day (Monday)"),
-        (datetime(2026, 3, 3), False, "Holi (Tuesday)"),
-        (datetime(2026, 11, 8), True, "Diwali Muhurat Trading (Sunday)"),
-        (datetime(2026, 11, 10), False, "Diwali-Balipratipada (Tuesday)"),
-        (datetime(2026, 11, 11), True, "Regular Wednesday"),
-        (datetime(2026, 11, 15), False, "Regular Sunday"),
-    ]
-    
-    passed = True
-    print("Self Verification Results:")
-    print("=" * 60)
-    for dt, expected, desc in test_dates:
-        actual = downloader._is_valid_trading_day(dt)
-        status = "PASS" if actual == expected else "FAIL"
-        if actual != expected:
-            passed = False
-        print(f"  {dt.strftime('%Y-%m-%d')} ({desc}): Expected={expected}, Actual={actual} -> {status}")
-    print("=" * 60)
-    
-    if passed:
-        print("All holiday logic checks passed successfully!")
-        sys.exit(0)
-    else:
-        print("Self-verification failed!")
-        sys.exit(1)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Standalone NSE Bhavcopy Downloader Utility")
-    parser.add_argument(
-        "-d", "--days", 
-        type=int, 
-        help="Number of lookback days (0 for full download from 2020-01-01)"
-    )
-    parser.add_argument(
-        "-o", "--output", 
-        type=str, 
-        default="downloads",
-        help="Output directory for downloading bhavcopies (default: 'downloads')"
-    )
-    parser.add_argument(
-        "--verify", 
-        action="store_true", 
-        help="Run self-verification tests for 2026 trading holiday logic"
-    )
-
-    args = parser.parse_args()
-
-    if args.verify:
-        self_verify()
-        return
-
-    days_back = args.days
-    if days_back is None:
-        try:
-            print("Running in interactive mode. Use -h or --help for command-line options.")
-            days_input = input("Enter number of days to look back (0 for full download): ").strip()
-            if not days_input:
-                days_back = 0
-            else:
-                days_back = int(days_input)
-        except (KeyboardInterrupt, EOFError):
-            print("\nExiting.")
-            sys.exit(0)
-        except ValueError:
-            print("Invalid input. Using full download mode.")
-            days_back = 0
-
-    if days_back < 0:
-        print("Invalid days input. Setting to full download.")
-        days_back = 0
-
-    downloader = NSEDownloader(days_back=days_back, downloads_dir=args.output)
-    downloader.run()
-
-
-if __name__ == "__main__":
-    main()

nsemaxlive.py

@@ -1,553 +0,0 @@
-#!/usr/bin/env python3
-"""
-NSE Max Live Data Scraper & Watchlist Generator (nsemaxlive.py)
-
-Scrapes live market gainers, losers, and volume spurts directly from NSE India
-using Selenium in headless Chrome, processes the raw CSV downloads, and generates
-top 50 JSON watchlists for NSEDash.
-
-Features:
-- Headless Chrome scraping for Advance, Decline, and Volume spurts
-- Robust download wait & error recovery retries
-- Processes scraped CSVs to generate JSON watchlists:
-    - nse_dashboard_maxvol50.json (Max Volume Portfolio)
-    - nse_dashboard_maxprice50.json (Max Price Gainers Portfolio)
-    - nse_dashboard_maxfall50.json (Max Price Fall Portfolio)
-- Configurable source and target paths for script execution
-"""
-
-import argparse
-import json
-import logging
-import os
-import random
-import sys
-import time
-from datetime import datetime
-from pathlib import Path
-import pandas as pd
-
-try:
-    from selenium import webdriver
-    from selenium.common.exceptions import (
-        ElementClickInterceptedException,
-        NoSuchElementException,
-        TimeoutException,
-        WebDriverException,
-    )
-    from selenium.webdriver.chrome.options import Options
-    from selenium.webdriver.chrome.service import Service as ChromeService
-    from selenium.webdriver.common.by import By
-    from selenium.webdriver.support import expected_conditions as EC
-    from selenium.webdriver.support.ui import WebDriverWait
-    from webdriver_manager.chrome import ChromeDriverManager
-except ImportError:
-    webdriver = None
-    class WebDriverException(Exception): pass
-    class TimeoutException(Exception): pass
-    class NoSuchElementException(Exception): pass
-    class ElementClickInterceptedException(Exception): pass
-
-
-# Default path handling: file is placed in /source/python/sammax1/nsedata/ (same folder as nsedash_admin.py)
-SCRIPT_DIR = Path(__file__).resolve().parent
-DEFAULT_OUTPUT_DIR = SCRIPT_DIR / "processed"
-DEFAULT_DOWNLOAD_DIR = SCRIPT_DIR / "tempfiles"
-
-# URLs and JavaScript functions for NSE data
-SCRAPE_ELEMENTS = {
-    'Advance': {
-        'url': 'https://www.nseindia.com/market-data/advance',
-        'js_function': "javascript:downloadCSV('Advance-download')",
-        'download_button_xpath': "//*[@id='Advance-download']"
-    },
-    'Decline': {
-        'url': 'https://www.nseindia.com/market-data/decline',
-        'js_function': "javascript:downloadCSV('Decline-download')",
-        'download_button_xpath': "//*[@id='Decline-download']"
-    },
-    'Volume': {
-        'url': 'https://www.nseindia.com/market-data/volume-gainers-spurts',
-        'js_function': "javascript:downloadCSVFile()",
-        'download_button_xpath': "//*[@id='dwldcsv']"
-    }
-}
-
-
-def print_splash_screen():
-    """Print stylized ASCII splash screen banner."""
-    splash = r"""
-================================================================================
-                      NSE MAX LIVE DATA SCRAPER & PROCESSOR                    
-================================================================================
-  Automated Selenium Headless Scraper for Live NSE India Market Data
-  Outputs: Max Volume (Top 50), Max Price Gainers (Top 50), Max Fall (Top 50)
-================================================================================
-"""
-    print(splash)
-
-
-class NseDataScraper:
-    """
-    A robust web scraper for downloading CSV data from NSE India website.
-    It uses Selenium with a headless Chrome browser, incorporating retry mechanisms
-    and improved error handling.
-    """
-    def __init__(self, download_path, output_path, min_sleep_time=0, max_sleep_time=3, max_retries=5, headless=True):
-        """
-        Initializes the NseDataScraper with paths and sets up the headless Chrome WebDriver.
-
-        Args:
-            download_path (str or Path): The directory where files will be temporarily downloaded by Chrome.
-            output_path (str or Path): The directory where processed (renamed) CSV files will be saved.
-            min_sleep_time (float): Minimum random sleep time in seconds.
-            max_sleep_time (float): Maximum random sleep time in seconds.
-            max_retries (int): Maximum download retry attempts per element.
-            headless (bool): Run Chrome browser in headless mode.
-        """
-        self.download_path = os.path.abspath(str(download_path))
-        self.output_path = os.path.abspath(str(output_path))
-        self.driver = None
-        self.wait = None
-        self.max_retries = max_retries
-        self.min_sleep_time = min_sleep_time
-        self.max_sleep_time = max_sleep_time
-        self.headless = headless
-
-        self._ensure_directories()
-        self._initialize_webdriver()
-
-    def _ensure_directories(self):
-        """Ensures that the download and output directories exist."""
-        try:
-            os.makedirs(self.download_path, exist_ok=True)
-            os.makedirs(self.output_path, exist_ok=True)
-        except OSError as e:
-            print(f"Error creating directories: {e}. Please check permissions.")
-            raise
-
-    def _initialize_webdriver(self):
-        """Initializes the Selenium WebDriver with Chrome options."""
-        if webdriver is None:
-            raise RuntimeError(
-                "Selenium or webdriver-manager is not installed in the Python environment. "
-                "Please run: pip install selenium webdriver-manager"
-            )
-        try:
-            chrome_options = Options()
-            prefs = {
-                "download.default_directory": self.download_path,
-                "download.prompt_for_download": False,
-                "download.directory_upgrade": True,
-                "plugins.always_open_pdf_externally": True
-            }
-            chrome_options.add_experimental_option("prefs", prefs)
-            if self.headless:
-                chrome_options.add_argument("--headless=new")
-            chrome_options.add_argument("--no-sandbox")
-            chrome_options.add_argument("--disable-dev-shm-usage")
-            chrome_options.add_argument("--window-size=1920,1080")
-            chrome_options.add_argument("--start-maximized")
-            chrome_options.add_argument("--disable-gpu")
-            chrome_options.add_argument("--log-level=3")
-            chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
-            chrome_options.add_argument("Accept-Language=en-US,en;q=0.9")
-            chrome_options.add_argument("Accept-Encoding=gzip, deflate, br")
-            chrome_options.add_argument("Accept=text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
-            chrome_options.add_argument("Connection=keep-alive")
-            chrome_options.add_argument("Upgrade-Insecure-Requests=1")
-
-            # Check for Linux system Chromium binary locations
-            for bin_path in ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"]:
-                if os.path.exists(bin_path):
-                    chrome_options.binary_location = bin_path
-                    break
-
-            try:
-                driver_path = ChromeDriverManager().install()
-                service = ChromeService(driver_path)
-                self.driver = webdriver.Chrome(service=service, options=chrome_options)
-            except Exception:
-                # Fallback to default system driver if ChromeDriverManager install fails
-                self.driver = webdriver.Chrome(options=chrome_options)
-
-            self.wait = WebDriverWait(self.driver, 30)
-
-        except WebDriverException as e:
-            print(f"Error initializing WebDriver: {e}")
-            raise
-        except Exception as e:
-            print(f"An unexpected error occurred during WebDriver initialization: {e}")
-            raise
-
-    def _wait_for_page_load_and_elements(self, url, download_button_xpath):
-        """
-        Waits for the page to load by checking for the presence and clickability of key elements.
-        Additionally, introduces a random sleep time after elements are found.
-        """
-        try:
-            self.wait.until(EC.presence_of_element_located((By.XPATH, "//img[@alt='csv']")))
-            self.wait.until(EC.element_to_be_clickable((By.XPATH, download_button_xpath)))
-
-            random_sleep_time = random.uniform(self.min_sleep_time, self.max_sleep_time) if self.max_sleep_time > 0 else 0.0
-            if random_sleep_time > 0:
-                time.sleep(random_sleep_time)
-            return True, random_sleep_time
-        except TimeoutException:
-            print(f"    Timeout: Essential elements not found on {url}.")
-            return False, 0.0
-        except NoSuchElementException:
-            print(f"    No such element: Required element not found on {url}.")
-            return False, 0.0
-        except Exception as e:
-            print(f"    An error occurred while waiting for page elements on {url}: {e}")
-            return False, 0.0
-
-    def _wait_for_download_completion(self, expected_filename_prefix, timeout=10):
-        """
-        Waits for a file to appear and complete downloading in the download directory.
-        Returns the full path of the downloaded file or None if timed out.
-        """
-        start_time = time.time()
-        downloaded_file = None
-        current_size = -1
-
-        while time.time() - start_time < timeout:
-            try:
-                list_of_files = os.listdir(self.download_path)
-                csv_files = [f for f in list_of_files if f.endswith('.csv') or f.endswith('.crdownload')]
-
-                if not csv_files:
-                    time.sleep(1)
-                    continue
-
-                latest_file_path = max([os.path.join(self.download_path, f) for f in csv_files], key=os.path.getmtime)
-                latest_file_name = os.path.basename(latest_file_path)
-
-                if latest_file_name.endswith('.crdownload'):
-                    current_size_new = os.path.getsize(latest_file_path)
-                    if current_size_new == current_size:
-                        downloaded_file = latest_file_path
-                        break
-                    current_size = current_size_new
-                    time.sleep(1)
-                    continue
-
-                if latest_file_name.endswith('.csv'):
-                    downloaded_file = latest_file_path
-                    break
-
-            except OSError as e:
-                print(f"    OS Error while checking download directory: {e}")
-                time.sleep(1)
-            except Exception as e:
-                print(f"    An unexpected error occurred during download check: {e}")
-                time.sleep(1)
-
-            time.sleep(1)
-
-        if downloaded_file:
-            return downloaded_file
-        else:
-            for f in os.listdir(self.download_path):
-                if f.endswith('.crdownload'):
-                    try:
-                        os.remove(os.path.join(self.download_path, f))
-                    except OSError as e:
-                        print(f"    Error removing incomplete download {f}: {e}")
-            return None
-
-    def download_csv(self, name, url, js_function, download_button_xpath):
-        """
-        Navigates to a URL, triggers a CSV download, and renames the downloaded file.
-        Includes retry logic and reports download metrics.
-        """
-        successful_sleep_time = 0.0
-
-        for attempt in range(1, self.max_retries + 1):
-            print(f"  Attempting to download {name} data... (Attempt {attempt}/{self.max_retries})", end='\r')
-            try:
-                if self.driver is None:
-                    self._initialize_webdriver()
-
-                self.driver.get(url)
-                page_load_success, current_sleep_time = self._wait_for_page_load_and_elements(url, download_button_xpath)
-                if not page_load_success:
-                    raise Exception("Page elements not ready.")
-                successful_sleep_time = current_sleep_time
-
-                self.driver.execute_script(js_function)
-                latest_downloaded_file = self._wait_for_download_completion(name)
-
-                if latest_downloaded_file:
-                    new_filename = os.path.join(self.output_path, f"{name}.csv")
-                    if os.path.exists(new_filename):
-                        os.remove(new_filename)
-                    os.rename(latest_downloaded_file, new_filename)
-                    print(f"  Successfully downloaded and saved {name} as: {new_filename} (Sleep Used: {round(successful_sleep_time, 2)}s)")
-                    return True
-                else:
-                    raise Exception("File download failed or timed out.")
-
-            except (WebDriverException, TimeoutException, NoSuchElementException,
-                    ElementClickInterceptedException, Exception):
-                self.close()
-                time.sleep(2)
-            finally:
-                for f in os.listdir(self.download_path):
-                    try:
-                        os.remove(os.path.join(self.download_path, f))
-                    except OSError as e:
-                        print(f"    Error cleaning up download directory: {e}")
-
-        print(f"  Failed to download {name} data after {self.max_retries} attempts.")
-        return False
-
-    def close(self):
-        """Closes the Selenium WebDriver session if it's open."""
-        try:
-            if self.driver:
-                self.driver.quit()
-                self.driver = None
-        except WebDriverException as e:
-            print(f"Error closing WebDriver: {e}")
-        except Exception as e:
-            print(f"An unexpected error occurred while closing the browser: {e}")
-
-    def scrape_all(self):
-        """
-        Orchestrates the scraping of all defined data elements in a random order,
-        handling retries and reporting success/failure.
-        """
-        results = {name: False for name in SCRAPE_ELEMENTS.keys()}
-        scrape_order = list(SCRAPE_ELEMENTS.keys())
-        random.shuffle(scrape_order)
-
-        print("\n--- Starting NSE Data Scraping Process ---")
-        print(f"Scraping order: {', '.join(scrape_order)}")
-
-        for name in scrape_order:
-            element_info = SCRAPE_ELEMENTS[name]
-            success = self.download_csv(
-                name,
-                element_info['url'],
-                element_info['js_function'],
-                element_info['download_button_xpath']
-            )
-            results[name] = success
-            if success:
-                self.close()
-
-        print("\n--- NSE Data Scraping Process Complete ---")
-        print("\nScraping Summary:")
-        for name, success in results.items():
-            status = "SUCCESS" if success else "FAILED"
-            print(f"- {name}: {status}")
-        return results
-
-
-def process_csv_to_json(
-    input_csv_path: str,
-    output_json_path: str,
-    portfolio_name: str,
-    symbol_col: str,
-    sort_col: str,
-    sort_ascending: bool = False,
-    top_n: int = 50,
-    header_clean_type: str = "strip_quotes_and_spaces"
-) -> None:
-    """
-    Processes a CSV file, extracts top N symbols based on sorting criteria,
-    and saves them to a JSON file in the specified watchlist format.
-
-    Args:
-        input_csv_path (str): Path to the input CSV file.
-        output_json_path (str): Path to the output JSON file.
-        portfolio_name (str): Name of the portfolio for the JSON output.
-        symbol_col (str): The name of the column containing stock symbols.
-        sort_col (str): The name of the column to sort by.
-        sort_ascending (bool): True for ascending sort, False for descending.
-        top_n (int): Number of top symbols to extract.
-        header_clean_type (str): Type of header cleaning to perform.
-                                 "strip_quotes_and_spaces" for Volume.csv,
-                                 "strip_spaces" for Advance/Decline.csv.
-    """
-    try:
-        df = pd.read_csv(input_csv_path)
-
-        cleaned_columns = []
-        for col in df.columns:
-            if header_clean_type == "strip_quotes_and_spaces":
-                cleaned_columns.append(col.strip().replace('"', ''))
-            elif header_clean_type == "strip_spaces":
-                cleaned_columns.append(col.strip())
-            else:
-                cleaned_columns.append(col)
-
-        df.columns = cleaned_columns
-
-        if sort_col not in df.columns:
-            print(f"Error: Sort column '{sort_col}' not found in {input_csv_path} after cleaning. Available columns: {df.columns.tolist()}")
-            return
-        if symbol_col not in df.columns:
-            print(f"Error: Symbol column '{symbol_col}' not found in {input_csv_path} after cleaning. Available columns: {df.columns.tolist()}")
-            return
-
-        df[sort_col] = pd.to_numeric(df[sort_col], errors='coerce')
-        df.dropna(subset=[sort_col], inplace=True)
-
-        df_sorted = df.sort_values(by=sort_col, ascending=sort_ascending)
-        top_symbols_df = df_sorted.head(top_n)
-
-        symbols_list = []
-        for _, row in top_symbols_df.iterrows():
-            symbols_list.append({
-                "SYMBOL": str(row[symbol_col]).strip().upper(),
-                "Quantity": 0.0,
-                "Price": 0.0
-            })
-
-        last_updated = datetime.now().isoformat()
-
-        json_output = {
-            "name": portfolio_name,
-            "symbols": symbols_list,
-            "last_updated": last_updated
-        }
-
-        output_dir = os.path.dirname(output_json_path)
-        if output_dir and not os.path.exists(output_dir):
-            os.makedirs(output_dir, exist_ok=True)
-
-        with open(output_json_path, 'w') as f:
-            json.dump(json_output, f, indent=2)
-
-        print(f"Processing completed: {os.path.basename(output_json_path)}")
-
-    except FileNotFoundError:
-        print(f"Error: Input file not found at {input_csv_path}")
-    except KeyError as e:
-        print(f"Error: Missing expected column in {input_csv_path}: {e}")
-    except Exception as e:
-        print(f"An unexpected error occurred while processing {input_csv_path}: {e}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="NSE Max Live Data Scraper and Watchlist Processor")
-    parser.add_argument(
-        "-o", "--output-dir",
-        type=str,
-        default=str(DEFAULT_OUTPUT_DIR),
-        help=f"Target directory for output CSVs & watchlists (default: {DEFAULT_OUTPUT_DIR})"
-    )
-    parser.add_argument(
-        "-t", "--temp-dir",
-        type=str,
-        default=str(DEFAULT_DOWNLOAD_DIR),
-        help=f"Temporary download directory for Chrome (default: {DEFAULT_DOWNLOAD_DIR})"
-    )
-    parser.add_argument(
-        "--min-sleep",
-        type=float,
-        default=0.0,
-        help="Minimum sleep time during element wait (seconds)"
-    )
-    parser.add_argument(
-        "--max-sleep",
-        type=float,
-        default=0.0,
-        help="Maximum sleep time during element wait (seconds)"
-    )
-    parser.add_argument(
-        "--retries",
-        type=int,
-        default=5,
-        help="Maximum retry attempts per URL download (default: 5)"
-    )
-    parser.add_argument(
-        "--no-headless",
-        action="store_true",
-        help="Run Chrome browser in visible mode (default: headless)"
-    )
-
-    args = parser.parse_args()
-
-    print_splash_screen()
-
-    download_folder = os.path.abspath(args.temp_dir)
-    output_folder = os.path.abspath(args.output_dir)
-
-    scraper = None
-    try:
-        scraper = NseDataScraper(
-            download_path=download_folder,
-            output_path=output_folder,
-            min_sleep_time=args.min_sleep,
-            max_sleep_time=args.max_sleep,
-            max_retries=args.retries,
-            headless=not args.no_headless
-        )
-        scrape_results = scraper.scrape_all()
-
-        output_base_dir = os.path.join(output_folder, "watchlists")
-        os.makedirs(output_base_dir, exist_ok=True)
-
-        print("\n--- Starting CSV Processing to JSON ---")
-
-        # 1. Process Volume.csv -> nse_dashboard_maxvol50.json
-        if scrape_results.get('Volume'):
-            process_csv_to_json(
-                input_csv_path=os.path.join(output_folder, "Volume.csv"),
-                output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxvol50.json"),
-                portfolio_name="Max Volume Portfolio",
-                symbol_col="SYMBOL",
-                sort_col="TODAY - VOLUME",
-                sort_ascending=False,
-                top_n=50,
-                header_clean_type="strip_quotes_and_spaces"
-            )
-        else:
-            print("Skipping Volume.csv processing as it was not successfully scraped.")
-
-        # 2. Process Advance.csv -> nse_dashboard_maxprice50.json
-        if scrape_results.get('Advance'):
-            process_csv_to_json(
-                input_csv_path=os.path.join(output_folder, "Advance.csv"),
-                output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxprice50.json"),
-                portfolio_name="Max Price Portfolio",
-                symbol_col="Symbol",
-                sort_col="%chng",
-                sort_ascending=False,
-                top_n=50,
-                header_clean_type="strip_spaces"
-            )
-        else:
-            print("Skipping Advance.csv processing as it was not successfully scraped.")
-
-        # 3. Process Decline.csv -> nse_dashboard_maxfall50.json
-        if scrape_results.get('Decline'):
-            process_csv_to_json(
-                input_csv_path=os.path.join(output_folder, "Decline.csv"),
-                output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxfall50.json"),
-                portfolio_name="Max Fall Portfolio",
-                symbol_col="Symbol",
-                sort_col="%chng",
-                sort_ascending=True,
-                top_n=50,
-                header_clean_type="strip_spaces"
-            )
-        else:
-            print("Skipping Decline.csv processing as it was not successfully scraped.")
-
-        print("\n--- CSV Processing to JSON Complete ---")
-
-    except Exception as e:
-        print(f"\nAn unrecoverable error occurred during the overall process: {e}")
-    finally:
-        if scraper:
-            scraper.close()
-        print("\n--- Program Exited ---")
-
-
-if __name__ == "__main__":
-    main()

nsemaxmin.py

@@ -1,147 +0,0 @@
-import pandas as pd
-import json
-import os
-from datetime import datetime
-
-def check_and_convert_csv_to_parquet(csv_path: str, parquet_path: str) -> None:
-    """If a CSV file is found, convert it to Parquet and delete the CSV."""
-    if os.path.exists(csv_path):
-        try:
-            df = pd.read_csv(csv_path)
-            df.to_parquet(parquet_path, index=False)
-            os.remove(csv_path)
-            print(f"Auto-converted {csv_path} to {parquet_path} and removed CSV.")
-        except Exception as e:
-            print(f"Failed to auto-convert {csv_path} to Parquet: {e}")
-
-def process_parquet_to_json(
-    input_parquet_path: str,
-    output_json_path: str,
-    portfolio_name: str,
-    symbol_col: str,
-    sort_col: str,
-    sort_ascending: bool = False,
-    top_n: int = 50,
-    header_clean_type: str = "strip_quotes_and_spaces" # "strip_quotes_and_spaces" or "strip_spaces"
-) -> None:
-    """
-    Processes a Parquet file, extracts top N symbols based on sorting criteria,
-    and saves them to a JSON file in the specified format.
-    """
-    # Check for CSV format and convert to Parquet if found
-    csv_path = input_parquet_path.replace(".parquet", ".csv")
-    check_and_convert_csv_to_parquet(csv_path, input_parquet_path)
-
-    try:
-        df = pd.read_parquet(input_parquet_path)
-
-        # Clean up column names based on the specified type
-        cleaned_columns = []
-        for col in df.columns:
-            if header_clean_type == "strip_quotes_and_spaces":
-                cleaned_columns.append(col.strip().replace('"', ''))
-            elif header_clean_type == "strip_spaces":
-                cleaned_columns.append(col.strip())
-            else:
-                cleaned_columns.append(col) # No cleaning
-
-        df.columns = cleaned_columns
-
-        # Ensure sort_col and symbol_col exist after cleaning
-        if sort_col not in df.columns:
-            print(f"Error: Sort column '{sort_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}")
-            return
-        if symbol_col not in df.columns:
-            print(f"Error: Symbol column '{symbol_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}")
-            return
-
-        # Convert sort_col to numeric, handling errors by coercing to NaN
-        df[sort_col] = pd.to_numeric(df[sort_col], errors='coerce')
-        # Drop rows where sort_col is NaN (if conversion failed)
-        df.dropna(subset=[sort_col], inplace=True)
-
-        # Sort the DataFrame
-        df_sorted = df.sort_values(by=sort_col, ascending=sort_ascending)
-
-        # Select the top N symbols
-        top_symbols_df = df_sorted.head(top_n)
-
-        # Prepare the list of symbols for JSON output
-        symbols_list = []
-        for _, row in top_symbols_df.iterrows():
-            symbols_list.append({
-                "SYMBOL": row[symbol_col],
-                "Quantity": 0.0,
-                "Price": 0.0
-            })
-
-        # Get current timestamp
-        last_updated = datetime.now().isoformat()
-
-        # Create the final JSON structure
-        json_output = {
-            "name": portfolio_name,
-            "symbols": symbols_list,
-            "last_updated": last_updated
-        }
-
-        # Create output directory if it doesn't exist
-        output_dir = os.path.dirname(output_json_path)
-        if output_dir and not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-
-        # Write to JSON file
-        with open(output_json_path, 'w') as f:
-            json.dump(json_output, f, indent=2)
-
-        print(f"Successfully processed {input_parquet_path} and saved to {output_json_path}")
-
-    except FileNotFoundError:
-        print(f"Error: Input file not found at {input_parquet_path}")
-    except KeyError as e:
-        print(f"Error: Missing expected column in {input_parquet_path}: {e}")
-    except Exception as e:
-        print(f"An unexpected error occurred while processing {input_parquet_path}: {e}")
-
-# --- Main execution ---
-if __name__ == "__main__":
-    # Ensure the 'processed/watchlists' directory exists
-    output_base_dir = "./processed/watchlists"
-    if not os.path.exists(output_base_dir):
-        os.makedirs(output_base_dir)
-
-    # 1. Process Volume.parquet
-    process_parquet_to_json(
-        input_parquet_path="./processed/Volume.parquet",
-        output_json_path="./processed/watchlists/nse_dashboard_maxvol50.json",
-        portfolio_name="Max Volume Portfolio",
-        symbol_col="SYMBOL",
-        sort_col="TODAY - VOLUME",
-        sort_ascending=False, # Descending for Max Volume
-        top_n=50,
-        header_clean_type="strip_quotes_and_spaces"
-    )
-
-    # 2. Process Advance.parquet
-    process_parquet_to_json(
-        input_parquet_path="./processed/Advance.parquet",
-        output_json_path="./processed/watchlists/nse_dashboard_maxprice50.json",
-        portfolio_name="Max Price Portfolio",
-        symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap
-        sort_col="%chng",
-        sort_ascending=False, # Descending for Max Price Change
-        top_n=50,
-        header_clean_type="strip_spaces"
-    )
-
-    # 3. Process Decline.parquet
-    process_parquet_to_json(
-        input_parquet_path="./processed/Decline.parquet",
-        output_json_path="./processed/watchlists/nse_dashboard_maxfall50.json",
-        portfolio_name="Max Fall Portfolio",
-        symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap
-        sort_col="%chng",
-        sort_ascending=True, # Ascending for Max Fall (most negative %chng)
-        top_n=50,
-        header_clean_type="strip_spaces"
-    )

nsemcupdate.py

@@ -1,418 +0,0 @@
-#!/usr/bin/env python3
-"""
-Moneycontrol Data Updater Utility (nsemcupdate.py)
-
-This script maps active NSE symbols in the database to their Moneycontrol pages
-and scrapes/updates financial metrics (MC_SCORE, MC_MCAP, MC_FV, MC_ATH, MC_ATL)
-in the nse_dashboard_mclink.json file.
-"""
-
-import os
-import sys
-import re
-import json
-import time
-import random
-import logging
-import argparse
-from pathlib import Path
-import requests
-from bs4 import BeautifulSoup
-
-# Setup basic logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s [%(levelname)s] %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout)
-    ]
-)
-
-USER_AGENTS = [
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0',
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0'
-]
-
-def get_headers():
-    return {
-        'User-Agent': random.choice(USER_AGENTS),
-        'Accept-Language': 'en-US,en;q=0.9',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
-        'Connection': 'keep-alive',
-        'Referer': 'https://www.moneycontrol.com/',
-        'Upgrade-Insecure-Requests': '1'
-    }
-
-def normalize_name(name):
-    if not name:
-        return ""
-    name = name.lower()
-    # Remove common corporate suffixes
-    name = re.sub(r'\b(ltd|limited|co|corp|corporation|ind|industries|india|limited\b|&)\b', '', name)
-    # Remove all non-alphanumeric characters
-    return re.sub(r'[^a-z0-9]', '', name)
-
-def parse_float(val_str):
-    if not val_str:
-        return 0.0
-    try:
-        cleaned = re.sub(r'[^\d.]', '', val_str.strip())
-        return float(cleaned) if cleaned else 0.0
-    except Exception:
-        return 0.0
-
-def scrape_directory_pages(cache_path, delay=2.0):
-    """Scrapes A-Z and others directories and caches the links list."""
-    logging.info("Starting Moneycontrol directory index pages scan...")
-    letters = [chr(c) for c in range(ord('A'), ord('Z') + 1)] + ['others']
-    all_links = []
-
-    for index, letter in enumerate(letters):
-        url = f'https://www.moneycontrol.com/india/stockpricequote/{letter}'
-        logging.info(f"[{index + 1}/{len(letters)}] Scanning index page: {url}")
-        
-        try:
-            r = requests.get(url, headers=get_headers(), timeout=15)
-            r.raise_for_status()
-        except Exception as e:
-            logging.error(f"Error fetching directory page {url}: {e}")
-            time.sleep(delay)
-            continue
-        
-        soup = BeautifulSoup(r.text, 'html.parser')
-        links = soup.select('a.bl_12')
-        page_count = 0
-        
-        for l in links:
-            href = l.get('href')
-            text = l.text.strip()
-            if href and 'https://www.moneycontrol.com/india/stockpricequote/' in href:
-                path_parts = [p for p in href.split('/') if p]
-                if len(path_parts) >= 2:
-                    mc_code = path_parts[-1].strip()
-                    mc_segment = path_parts[-2].strip()
-                    all_links.append({
-                        "company_name": text,
-                        "mc_link": href,
-                        "mc_company_code": mc_code,
-                        "mc_url_segment": mc_segment
-                    })
-                    page_count += 1
-                    
-        logging.info(f"  -> Extracted {page_count} company links from page '{letter}'")
-        time.sleep(delay + random.uniform(0.5, 1.5))
-
-    # Save to cache
-    try:
-        with open(cache_path, 'w') as f:
-            json.dump(all_links, f, indent=4)
-        logging.info(f"Directory cache saved to: {cache_path}")
-    except Exception as e:
-        logging.error(f"Failed to save directory cache: {e}")
-
-    return all_links
-
-def load_directory_cache(cache_path, delay=2.0):
-    if cache_path.exists():
-        try:
-            with open(cache_path, 'r') as f:
-                return json.load(f)
-        except Exception as e:
-            logging.warning(f"Failed to load directory cache: {e}. Re-scraping...")
-    return scrape_directory_pages(cache_path, delay)
-
-def get_nse_symbol_from_page(soup):
-    """Robustly extracts the NSE ticker symbol from the HTML page."""
-    # Heuristic 1: clearfix list items
-    for li in soup.find_all('li'):
-        span = li.find('span')
-        p = li.find('p')
-        if span and p and span.text.strip().lower().rstrip(':') == 'nse':
-            return p.text.strip().upper()
-            
-    # Heuristic 2: search for NSE label string
-    for tag in soup.find_all(string=lambda x: x and x.strip().lower().rstrip(':') == 'nse'):
-        parent = tag.parent
-        sibling = parent.find_next_sibling()
-        if sibling:
-            return sibling.text.strip().upper()
-            
-    # Heuristic 3: JSON script section at the bottom of the page
-    for script in soup.find_all('script'):
-        if script.string and 'nseId' in script.string:
-            match = re.search(r'"nseId"\s*:\s*"([^"]+)"', script.string)
-            if match:
-                return match.group(1).strip().upper()
-                
-    return None
-
-def scrape_stock_details(mc_link):
-    """Scrapes detail metrics and returns verified details or None."""
-    try:
-        r = requests.get(mc_link, headers=get_headers(), timeout=15)
-        r.raise_for_status()
-    except Exception as e:
-        logging.warning(f"Request failed for {mc_link}: {e}")
-        return None
-
-    soup = BeautifulSoup(r.text, 'html.parser')
-    
-    # Extract & verify NSE symbol
-    nse_symbol = get_nse_symbol_from_page(soup)
-    
-    # Scrape metrics
-    score_div = soup.find(class_='score') or soup.select_one('div[class*="score"]')
-    mc_score = 0.0
-    if score_div:
-        match = re.search(r'(\d+)', score_div.text)
-        if match:
-            mc_score = float(match.group(1))
-
-    mcap_td = soup.select_one('td.nsemktcap.bsemktcap')
-    mc_mcap = parse_float(mcap_td.text) if mcap_td else 0.0
-
-    fv_td = soup.select_one('td.nsefv.bsefv')
-    mc_fv = parse_float(fv_td.text) if fv_td else 0.0
-
-    ath_td = soup.select_one('td.nseLTH.bseLTH') or soup.select_one('td.nseH52.bseH52')
-    mc_ath = parse_float(ath_td.text) if ath_td else 0.0
-
-    atl_td = soup.select_one('td.nseLTL.bseLTL') or soup.select_one('td.nseL52.bseL52')
-    mc_atl = parse_float(atl_td.text) if atl_td else 0.0
-
-    return {
-        "NSE_SYMBOL": nse_symbol,
-        "MC_SCORE": mc_score,
-        "MC_MCAP": mc_mcap,
-        "MC_FV": mc_fv,
-        "MC_ATH": mc_ath,
-        "MC_ATL": mc_atl
-    }
-
-def main():
-    parser = argparse.ArgumentParser(description="Update Moneycontrol related information for NSE stocks.")
-    parser.add_argument("--base-dir", help="Base directory containing processed stock data.")
-    parser.add_argument("--update-all", action="store_true", help="Force update all stocks, even those already successfully matched.")
-    parser.add_argument("--delay", type=float, default=2.0, help="Mean delay between network requests to prevent rate blocking.")
-    parser.add_argument("--limit", type=int, help="Limit the number of stocks scraped in this run.")
-    args = parser.parse_args()
-
-    # Initialize Paths config
-    # We append workspace path to sys.path so we can import nsedash.config
-    workspace_dir = Path(__file__).resolve().parent
-    sys.path.insert(0, str(workspace_dir))
-    
-    try:
-        from nsedash.config import Paths
-    except ImportError as e:
-        logging.error(f"Failed to import config. Ensure you run this script from the workspace root: {e}")
-        sys.exit(1)
-
-    paths = Paths(base_dir=args.base_dir)
-    logging.info(f"Initialized base data directory: {paths.base_dir}")
-    
-    # Load active symbols from parquet
-    if not paths.input_file.exists():
-        logging.error(f"Active stock processed data file not found: {paths.input_file}")
-        sys.exit(1)
-        
-    import pandas as pd
-    try:
-        df = pd.read_parquet(paths.input_file)
-        active_companies = df[['SYMBOL', 'SECURITY']].drop_duplicates().to_dict('records')
-        logging.info(f"Loaded {len(active_companies)} unique active stocks from database.")
-    except Exception as e:
-        logging.error(f"Failed to load active symbols: {e}")
-        sys.exit(1)
-
-    # Load existing links
-    mc_links = {}
-    if paths.mc_link_file.exists():
-        try:
-            with open(paths.mc_link_file, 'r') as f:
-                mc_links = json.load(f)
-            logging.info(f"Loaded {len(mc_links)} existing Moneycontrol links from: {paths.mc_link_file.name}")
-        except Exception as e:
-            logging.error(f"Failed to load mc_link file: {e}")
-
-    # Determine which stocks need lookup or update
-    stocks_to_update = []
-    for item in active_companies:
-        symbol = str(item['SYMBOL']).strip().upper()
-        security = str(item['SECURITY']).strip()
-        
-        # Check if already processed
-        details = mc_links.get(symbol, {})
-        has_link = 'MC_LINK' in details and details['MC_LINK']
-        is_completed = details.get('MC_STATUS', False)
-        
-        if args.update_all or not has_link or not is_completed:
-            stocks_to_update.append({
-                "symbol": symbol,
-                "security": security,
-                "existing_details": details
-            })
-
-    if not stocks_to_update:
-        logging.info("All stocks are already up-to-date. Nothing to do!")
-        sys.exit(0)
-
-    # Filter out limit
-    if args.limit:
-        stocks_to_update = stocks_to_update[:args.limit]
-        logging.info(f"Limiting scan to first {args.limit} stocks.")
-
-    logging.info(f"Total stocks to update in this session: {len(stocks_to_update)}")
-
-    # Load directory listings (cache or scrape)
-    cache_path = paths.data_dir / "nse_mc_directory_cache.json"
-    mc_directory = load_directory_cache(cache_path, args.delay)
-    logging.info(f"Moneycontrol directory loaded with {len(mc_directory)} links.")
-
-    # Match missing links
-    matched_count = 0
-    scraped_count = 0
-    start_time = time.time()
-
-    try:
-        for idx, stock in enumerate(stocks_to_update):
-            symbol = stock["symbol"]
-            security = stock["security"]
-            existing = stock["existing_details"]
-            
-            logging.info(f"[{idx+1}/{len(stocks_to_update)}] Processing: {symbol} - {security}")
-            
-            mc_link = existing.get("MC_LINK")
-            mc_code = existing.get("MC_COMPANY_CODE")
-            mc_name = existing.get("MC_COMPANY_NAME", security)
-            
-            # 1. Resolve Link if missing
-            if not mc_link:
-                # Find candidates using heuristics
-                candidates = []
-                norm_sec = normalize_name(security)
-                norm_sym = normalize_name(symbol)
-                
-                for entry in mc_directory:
-                    c_name = entry["company_name"]
-                    c_code = entry["mc_company_code"]
-                    c_seg = entry["mc_url_segment"]
-                    c_link = entry["mc_link"]
-                    
-                    norm_c_name = normalize_name(c_name)
-                    norm_c_seg = normalize_name(c_seg)
-                    
-                    # Heuristics checks
-                    if norm_sym == normalize_name(c_code):
-                        candidates.append((10, c_link, c_code, c_name))
-                    elif norm_sec == norm_c_name or norm_sec == norm_c_seg:
-                        candidates.append((9, c_link, c_code, c_name))
-                    elif norm_sec in norm_c_name or norm_c_name in norm_sec:
-                        candidates.append((8, c_link, c_code, c_name))
-                    elif norm_sec in norm_c_seg or norm_c_seg in norm_sec:
-                        candidates.append((7, c_link, c_code, c_name))
-                    elif norm_sym in norm_c_name or norm_sym in norm_c_seg:
-                        candidates.append((5, c_link, c_code, c_name))
-
-                # Sort by priority score descending
-                candidates.sort(key=lambda x: x[0], reverse=True)
-                
-                # Try candidate links
-                found = False
-                for priority, c_link, c_code, c_name in candidates:
-                    logging.info(f"  Trying candidate URL ({priority} pts): {c_link}")
-                    
-                    # Scraping & Verification
-                    time.sleep(args.delay + random.uniform(0.1, 1.0))
-                    details = scrape_stock_details(c_link)
-                    scraped_count += 1
-                    
-                    if details:
-                        extracted_sym = details["NSE_SYMBOL"]
-                        if extracted_sym == symbol:
-                            logging.info(f"    --> Verified match! NSE symbol matches: {symbol}")
-                            mc_link = c_link
-                            mc_code = c_code
-                            mc_name = c_name
-                            
-                            # Merge details
-                            mc_links[symbol] = {
-                                "MC_LINK": mc_link,
-                                "MC_COMPANY_CODE": mc_code,
-                                "MC_COMPANY_NAME": mc_name,
-                                "MC_SCORE": details["MC_SCORE"],
-                                "MC_MCAP": details["MC_MCAP"],
-                                "MC_FV": details["MC_FV"],
-                                "MC_ATH": details["MC_ATH"],
-                                "MC_ATL": details["MC_ATL"],
-                                "MC_STATUS": True
-                            }
-                            matched_count += 1
-                            found = True
-                            break
-                        else:
-                            logging.info(f"    --> Rejected. Page NSE symbol was: {extracted_sym}")
-                            
-                if not found:
-                    logging.warning(f"  No verified Moneycontrol URL found for: {symbol}")
-                    # If it had existing details, mark status as False to retry
-                    if symbol in mc_links:
-                        mc_links[symbol]["MC_STATUS"] = False
-                    continue
-                    
-            else:
-                # Stock already has a link, just scrape & update the metrics
-                logging.info(f"  Scraping metrics from existing link: {mc_link}")
-                time.sleep(args.delay + random.uniform(0.1, 1.0))
-                details = scrape_stock_details(mc_link)
-                scraped_count += 1
-                
-                if details:
-                    # Verified check
-                    if details["NSE_SYMBOL"] and details["NSE_SYMBOL"] != symbol:
-                        logging.warning(f"    --> Warning: URL NSE symbol mismatch. Expected {symbol}, got {details['NSE_SYMBOL']}")
-                    
-                    mc_links[symbol] = {
-                        "MC_LINK": mc_link,
-                        "MC_COMPANY_CODE": mc_code,
-                        "MC_COMPANY_NAME": mc_name,
-                        "MC_SCORE": details["MC_SCORE"],
-                        "MC_MCAP": details["MC_MCAP"],
-                        "MC_FV": details["MC_FV"],
-                        "MC_ATH": details["MC_ATH"],
-                        "MC_ATL": details["MC_ATL"],
-                        "MC_STATUS": True
-                    }
-                    matched_count += 1
-                else:
-                    logging.warning(f"  Failed to update metrics for: {symbol}")
-                    mc_links[symbol]["MC_STATUS"] = False
-
-            # Periodic save (every 10 updates)
-            if matched_count % 10 == 0:
-                with open(paths.mc_link_file, 'w') as f:
-                    json.dump(mc_links, f, indent=4, sort_keys=True)
-                logging.info("Periodic progress saved.")
-
-    except KeyboardInterrupt:
-        logging.info("\n--- PAUSED BY USER ---")
-        logging.info("Saving progress before exit...")
-        
-    finally:
-        # Final save
-        with open(paths.mc_link_file, 'w') as f:
-            json.dump(mc_links, f, indent=4, sort_keys=True)
-        
-        elapsed = time.time() - start_time
-        logging.info(f"\nExecution finished.")
-        logging.info(f"  - Elapsed Time: {elapsed:.2f} seconds")
-        logging.info(f"  - Requests Executed: {scraped_count}")
-        logging.info(f"  - Stocks Successfully Updated: {matched_count}")
-        logging.info(f"  - Total entries now in json: {len(mc_links)}")
-
-if __name__ == "__main__":
-    main()

nseprocessor.py

@@ -1,1258 +0,0 @@
-"""
-NSE Bhavcopy Data Processor
-
-This script processes NSE bhavcopy files with proper DataFrame handling and enhanced logging.
-Supports both fresh processing and incremental updates.
-
-Features:
-- Date-ordered processing
-- Proper DataFrame operations
-- Enhanced logging
-- Progress tracking
-- Data validation
-- Optional corporate action application for full processing runs
-
-Author: Your Name
-Date: June 2025 (Updated)
-"""
-
-import pandas as pd
-import numpy as np
-import zipfile
-import re
-import logging
-from pathlib import Path
-from datetime import datetime, timedelta
-import time
-from pathlib import Path
-import logging
-from tqdm import tqdm
-import sys
-import gc
-import json
-try:
-    import talib
-except ImportError:
-    talib = None
-try:
-    import pandas_ta_classic as ta
-except ImportError:
-    try:
-        import pandas_ta as ta
-    except ImportError:
-        ta = None
-
-
-import fastparquet
-import warnings
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.feature_selection import mutual_info_regression
-
-warnings.filterwarnings('ignore')
-warnings.simplefilter('ignore', FutureWarning)
-
-def _calculate_indicators_for_group(group_data_tuple):
-    symbol, group, col_mapping = group_data_tuple
-    df = group.copy()
-    
-    # Rename columns to match pandas_ta expectations
-    df = df.rename(columns=col_mapping)
-    
-    # ===== MOMENTUM INDICATORS (14) =====
-    df.ta.rsi(length=14, append=True)
-    df.ta.macd(fast=12, slow=26, signal=9, append=True)
-    stoch = df.ta.stoch(length=14, k=3, d=3, append=True)
-    df.ta.willr(length=14, append=True)
-    df.ta.cmo(length=14, append=True)
-    df.ta.cci(length=20, append=True)
-    df.ta.roc(length=12, append=True)
-    df.ta.mfi(length=14, append=True)
-    df.ta.fisher(length=9, append=True)
-    df.ta.kst(append=True)
-    df.ta.mom(length=10, append=True)
-    df.ta.tsi(long=25, short=13, append=True)
-    df.ta.uo(fast=7, medium=14, slow=28, append=True)
-    df.ta.ao(append=True)
-    
-    # ===== TREND INDICATORS (18) =====
-    df.ta.sma(length=20, append=True)
-    df.ta.sma(length=50, append=True)
-    df.ta.sma(length=200, append=True)
-    df.ta.ema(length=12, append=True)
-    df.ta.ema(length=26, append=True)
-    df.ta.ema(length=50, append=True)
-    df.ta.ema(length=200, append=True)
-    df.ta.adx(length=14, append=True)
-    df.ta.aroon(length=14, append=True)
-    df.ta.supertrend(length=10, multiplier=3, append=True)
-    df.ta.psar(af0=0.02, af=0.2, append=True)
-    df.ta.alma(length=9, append=True)
-    df.ta.dema(length=10, append=True)
-    df.ta.tema(length=10, append=True)
-    df.ta.hma(length=9, append=True)
-    df.ta.kama(length=10, append=True)
-    df.ta.linreg(length=25, append=True)
-    
-    # ===== VOLATILITY INDICATORS (14) =====
-    df['True_Range'] = np.maximum(
-        np.maximum(
-            df['High'] - df['Low'],
-            np.abs(df['High'] - df['Close'].shift())
-        ),
-        np.abs(df['Low'] - df['Close'].shift())
-    )
-    
-    bbands = df.ta.bbands(length=20, std=2, append=True)
-    df.ta.atr(length=14, append=True)
-    df.ta.natr(length=14, append=True)
-    df.ta.kc(length=20, scalar=2, append=True)
-    df.ta.stdev(length=20, append=True)
-    
-    df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252)
-    df['Donchian_High'] = df['High'].rolling(20).max()
-    df['Donchian_Low'] = df['Low'].rolling(20).min()
-    df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2
-    
-    hl_range = df['High'] - df['Low']
-    mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum()
-    df['Mass_Index'] = mass_index
-    
-    df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-    
-    atr_sum = df['True_Range'].rolling(14).sum()
-    high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9))
-    df['CHOP'] = 100 * high_low / np.log10(14)
-    
-    # ===== VOLUME INDICATORS (13) =====
-    df.ta.obv(append=True)
-    df.ta.ad(append=True)
-    df.ta.adosc(fast=3, slow=10, append=True)
-    df.ta.cmf(length=20, append=True)
-    
-    df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum()
-    df['Volume_SMA_20'] = df['Volume'].rolling(20).mean()
-    
-    klinger_fast = df['Volume'].ewm(span=34).mean()
-    klinger_slow = df['Volume'].ewm(span=55).mean()
-    df['KO'] = klinger_fast - klinger_slow
-    
-    distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2
-    box_height = df['High'] - df['Low']
-    eom = distance_moved / (box_height / df['Volume'])
-    df['EOM'] = eom.rolling(14).mean()
-    
-    df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume']
-    df['EFI'] = df['EFI'].ewm(span=13).mean()
-    
-    tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2)
-    vm_plus = np.abs(df['High'] - df['Low'].shift(1))
-    vm_minus = np.abs(df['Low'] - df['High'].shift(1))
-    df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum()
-    df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum()
-    
-    # ===== OSCILLATORS & ADDITIONAL =====
-    df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['Median_Price'] = (df['High'] + df['Low']) / 2
-    df['HL2'] = (df['High'] + df['Low']) / 2
-    df['OC2'] = (df['Open'] + df['Close']) / 2
-    df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4
-    
-    df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['R1'] = 2 * df['Pivot'] - df['Low']
-    df['S1'] = 2 * df['Pivot'] - df['High']
-    df['R2'] = df['Pivot'] + (df['High'] - df['Low'])
-    df['S2'] = df['Pivot'] - (df['High'] - df['Low'])
-    
-    weights = np.arange(1, 21)
-    df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights))
-    df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean()
-    df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum()
-    
-    df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-    df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20']
-    
-    df['Price_vs_SMA20'] = df['Close'] - df['SMA_20']
-    df['Price_vs_EMA12'] = df['Close'] - df['EMA_12']
-    df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26']
-    df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200']
-    
-    df['RSI_Divergence'] = df['RSI_14'].diff()
-    df['Volume_Divergence'] = df['Volume'].diff()
-    
-    df['Momentum_Strength'] = df['RSI_14'].rolling(5).std()
-    df['Trend_Strength'] = df['ADX_14']
-    
-    df['Close_Change'] = df['Close'].pct_change()
-    df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close']
-    df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open']
-    df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean()
-    
-    # Revert column names to uppercase
-    inv_col_mapping = {v: k for k, v in col_mapping.items()}
-    df = df.rename(columns=inv_col_mapping)
-    
-    # Add sentiment & news features (4 features)
-    df['Sentiment'] = 0.0
-    df['News_Count'] = 0.0
-    df['News_Sentiment'] = 0.0
-    df['News_Impact'] = 0.0
-    
-    # Add hybrid strategy indicators (4 features)
-    df['VIX'] = 15.0
-    df['Vol_20MA'] = df['NET_TRDQTY'].rolling(window=20).mean().bfill().fillna(0.0)
-    df['Prev_High'] = df['HIGH_PRICE'].shift(1).bfill().fillna(0.0)
-    df['Breakout_Signal'] = ((df['CLOSE_PRICE'] > df['Prev_High']) & (df['NET_TRDQTY'] > df['Vol_20MA'])).astype(float)
-    
-    return df
-
-class NSEDataProcessor:
-    """Process NSE bhavcopy data with optimized display and relative paths"""
-
-    def __init__(self):
-        self.setup_paths()
-        self.setup_logging()
-        self.initialize_metrics()
-        
-        # Define schema (existing schema code)
-        self.schema = {
-            'SYMBOL': 'category',
-            'SECURITY': 'category', 
-            'SERIES': 'category',
-            'TRAN_DATE': 'datetime64[ns]',
-            'OPEN_PRICE': 'float32',
-            'HIGH_PRICE': 'float32', 
-            'LOW_PRICE': 'float32',
-            'CLOSE_PRICE': 'float32',
-            'PREV_CL_PR': 'float32',
-            'NET_TRDVAL': 'float64',
-            'NET_TRDQTY': 'int64',
-            'TRADES': 'int64',
-            'HI_52_WK': 'float32',
-            'LO_52_WK': 'float32',
-            'HI_DIFF': 'float32',
-            'LO_DIFF': 'float32',
-            'DAILY_RETURN': 'float32',
-            'VOLATILITY': 'float32'
-            # Additional feature engineering columns, added runtime and saved to extract file
-        }
-        
-        self.string_cols = ['MKT', 'SERIES', 'SYMBOL', 'SECURITY']
-        self.numeric_cols = [
-            'PREV_CL_PR', 'OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE',
-            'CLOSE_PRICE', 'NET_TRDVAL', 'NET_TRDQTY', 'TRADES',
-            'HI_52_WK', 'LO_52_WK'
-        ]
-        self.corporate_actions = [] # Initialize corporate actions list
-
-    def setup_paths(self):
-        """Setup using relative paths"""
-        try:
-            self.base_dir = Path.cwd()
-            self.downloads_dir = self.base_dir / 'downloads'
-            self.processed_dir = self.base_dir / 'processed'
-            self.corpact_file = self.processed_dir / 'nse_dashboard_corpact.json'
-            self.extract_file = self.processed_dir / 'nse_extract_data.parquet'
-            self.parquet_path = self.processed_dir / 'nse_processed_data.parquet'
-            
-            if not self.downloads_dir.exists():
-                raise FileNotFoundError(f"Downloads directory not found: {self.downloads_dir}")
-                
-            self.processed_dir.mkdir(parents=True, exist_ok=True)
-            
-        except Exception as e:
-            print(f"Path setup failed: {str(e)}")
-            sys.exit(1)
-
-    def setup_logging(self):
-        """Configure minimal logging"""
-        try:
-            log_format = '%(asctime)s - %(levelname)s - %(message)s'
-            logging.basicConfig(
-                filename=self.downloads_dir / 'bulk_data_processing.log',
-                level=logging.INFO,
-                format=log_format
-            )
-            self.logger = logging.getLogger(__name__)
-            
-        except Exception as e:
-            print(f"Logging setup failed: {str(e)}")
-            sys.exit(1)
-           
-    def initialize_metrics(self):
-        """Initialize processing metrics"""
-        self.metrics = {
-            'start_time': None,
-            'files_found': 0,
-            'files_processed': 0,
-            'files_failed': 0,
-            'rows_processed': 0,
-            'processing_errors': [],
-            'processed_files': [],
-            'failed_files': [],
-            'start_datetime': None,
-            'end_datetime': None,
-            'duration': None,
-            'batch_metrics': [],
-            'corporate_actions_applied_count': 0 # New metric for corporate actions
-        }
-        
-    def get_latest_processed_date(self):
-        """Get latest processed date from existing output"""
-        try:
-            if self.parquet_path.exists():
-                df = pd.read_parquet(self.parquet_path, columns=['TRAN_DATE'])
-                latest_date = pd.to_datetime(df['TRAN_DATE']).max()
-                self.logger.info(f"Latest processed date: {latest_date.strftime('%Y-%m-%d')}")
-                print(f"Found existing data - processing files newer than {latest_date.strftime('%Y-%m-%d')}")
-                return latest_date
-            else:
-                self.logger.info("No existing data found - performing fresh processing")
-                print("Starting fresh processing - no existing data found")
-                return None
-        except Exception as e:
-            self.logger.error(f"Failed to get latest date: {str(e)}")
-            return None
-
-    def extract_date_from_filename(self, filename):
-        """Extract and validate date from filename (supports PdDDMMYY, PRDDMMYY, pdDDMMYYYY)"""
-        try:
-            name = Path(filename).stem
-
-            # Match old format: Pd or PR + ddmmyy
-            match_old = re.search(r'(?i)(Pd|PR)(\d{2})(\d{2})(\d{2})', name)
-
-            # Match new format: pd + ddmmyyyy
-            match_new = re.search(r'(?i)pd(\d{2})(\d{2})(\d{4})', name)
-
-            if match_new:
-                day, month, year = match_new.groups()
-            elif match_old:
-                day, month, year = match_old.groups()[1:]  # skip prefix
-                year = str(2000 + int(year))  # convert yy to yyyy
-            else:
-                return None
-
-            date_obj = datetime(int(year), int(month), int(day))
-
-            if date_obj.year < 2000 or date_obj > datetime.now():
-                raise ValueError(f"Date out of valid range: {date_obj}")
-
-            return date_obj.strftime('%Y-%m-%d')
-
-        except Exception as e:
-            self.logger.error(f"Date extraction failed for {filename}: {str(e)}")
-            return None
-
-    def process_zip_file(self, zip_path):
-        """Process ZIP file with error handling"""
-        try:
-            with zipfile.ZipFile(zip_path) as zf:
-                csv_files = [f for f in zf.namelist()
-                             if re.match(r'^pd\d{6}(\d{2})?\.csv$', Path(f).name, re.IGNORECASE)]
-                
-                if not csv_files:
-                    self.logger.warning(f"No matching CSV in {zip_path}")
-                    return None
-                    
-                csv_file = csv_files[0]
-                tran_date = self.extract_date_from_filename(Path(csv_file).name)
-                if not tran_date:
-                    return None
-                    
-                with zf.open(csv_file) as f:
-                    df = pd.read_csv(f, thousands=',')
-                    return self.process_dataframe(df, tran_date)
-                    
-        except Exception as e:
-            self.logger.error(f"ZIP processing failed - {zip_path}: {str(e)}")
-            self.metrics['files_failed'] += 1
-            return None
-
-    def process_dataframe(self, df, tran_date):
-        """Process DataFrame with proper indexing"""
-        try:
-            # Create explicit copy
-            df = df.copy()
-            
-            # Clean string columns
-            for col in self.string_cols:
-                if col in df.columns:
-                    df.loc[:, col] = df[col].astype(str).str.strip()
-            
-            # Filter rows and create new copy
-            mask = (
-                (df['SERIES'].str.upper() == 'EQ') & 
-                df['MKT'].notna()
-            )
-            mask_index = (
-                df['MKT'].str.upper() == 'Y'
-            )
-            df_index = df[mask_index].copy()
-            df = df[mask].copy()
-            
-            # Create SYMBOL column for df_index
-            if not df_index.empty:
-                df_index.loc[:, 'SYMBOL'] = df_index['SECURITY'].str.upper().str.replace(r'\s+','', regex=True)
-                df_index = df_index[df_index['SYMBOL'].notna()]
-                df_index = df_index.drop_duplicates(subset=['SYMBOL'])
-
-            # Concat df_index to df
-            df = pd.concat([df, df_index], ignore_index=True)
-
-            # Add transaction date
-            df.loc[:, 'TRAN_DATE'] = pd.to_datetime(tran_date)
-            
-            # Process numeric columns
-            for col in self.numeric_cols:
-                if col in df.columns:
-                    df.loc[:, col] = pd.to_numeric(
-                        df[col].astype(str).str.replace(',', ''),
-                        errors='coerce'
-                    )
-            
-            # Drop rows with missing numerics
-            df = df.dropna(subset=self.numeric_cols)
-            
-            if df.empty:
-                return None
-                
-            # Calculate indicators and validate
-            df = self.add_technical_indicators(df)
-            df = self.validate_schema(df)
-            
-            if df is not None:
-                self.metrics['rows_processed'] += len(df)
-                
-            return df
-            
-        except Exception as e:
-            self.logger.error(f"DataFrame processing error: {str(e)}")
-            return None
-
-    def add_technical_indicators(self, df):
-        """Calculate technical indicators with proper dtype handling"""
-        try:
-            df = df.copy()
-            
-            # Initialize indicator columns with np.nan
-            indicators = ['HI_DIFF', 'LO_DIFF', 'DAILY_RETURN', 'VOLATILITY']
-            for col in indicators:
-                df[col] = np.nan
-            
-            # Helper function for safe calculation
-            def calculate_pct_change(a, b, divisor):
-                """Calculate percentage change with proper dtype handling"""
-                result = (
-                    ((a.astype('float32') - b.astype('float32')) / 
-                    divisor.astype('float32') * 100)
-                    .round(2)
-                    .astype('float32')
-                )
-                return result
-            
-            # Calculate HI_DIFF
-            mask = df['HI_52_WK'] > 0
-            if mask.any():
-                df.loc[mask, 'HI_DIFF'] = calculate_pct_change(
-                    df.loc[mask, 'HI_52_WK'],
-                    df.loc[mask, 'CLOSE_PRICE'],
-                    df.loc[mask, 'HI_52_WK']
-                )
-            
-            # Calculate LO_DIFF
-            mask = df['LO_52_WK'] > 0
-            if mask.any():
-                df.loc[mask, 'LO_DIFF'] = calculate_pct_change(
-                    df.loc[mask, 'CLOSE_PRICE'],
-                    df.loc[mask, 'LO_52_WK'],
-                    df.loc[mask, 'LO_52_WK']
-                )
-            
-            # Calculate DAILY_RETURN
-            mask = df['PREV_CL_PR'] > 0
-            if mask.any():
-                df.loc[mask, 'DAILY_RETURN'] = calculate_pct_change(
-                    df.loc[mask, 'CLOSE_PRICE'],
-                    df.loc[mask, 'PREV_CL_PR'],
-                    df.loc[mask, 'PREV_CL_PR']
-                )
-            
-            # Calculate VOLATILITY
-            mask = df['OPEN_PRICE'] > 0
-            if mask.any():
-                high_low_diff = (df.loc[mask, 'HIGH_PRICE'] - df.loc[mask, 'LOW_PRICE'])
-                df.loc[mask, 'VOLATILITY'] = calculate_pct_change(
-                    high_low_diff,
-                    pd.Series(0, index=high_low_diff.index),
-                    df.loc[mask, 'OPEN_PRICE']
-                )
-            
-            # Drop rows where any indicator is missing
-            return df.dropna(subset=indicators)
-            
-        except Exception as e:
-            self.logger.error(f"Error calculating indicators: {str(e)}")
-            return df
-        
-    def _create_forward_target(self, group, lookahead_days=1):
-        """
-        Creates a forward-looking regression target: the next closing price after `lookahead_days`.
-        Optionally allows for volatility computation or future annotation.
-        Returns a DataFrame with 'TARGET_PRICES' column containing single future prices.
-        """
-        # Ensure chronological order
-        group = group.sort_values(by='TRAN_DATE').copy()
-
-        # Extract future single-step targets
-        close_prices = group['CLOSE_PRICE'].values
-        future_targets = []
-
-        for i in range(len(close_prices)):
-            if i + lookahead_days < len(close_prices):
-                future_targets.append(close_prices[i + lookahead_days])
-            else:
-                future_targets.append(np.nan)
-
-        group['TARGET_PRICES'] = future_targets
-
-        # Drop rows where TARGET_PRICES is NaN
-        group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True)
-
-        return group
-
-    def _create_simple_target(self, group, lookahead_days=1, method="single"):
-        """
-            Generates regression targets for each row based on future closing prices.
-
-        Parameters:
-        - group: DataFrame containing at least 'TRAN_DATE' and 'CLOSE_PRICE'
-        - lookahead_days: Number of days to look ahead for target generation
-        - method: 'single' for next-day price, 'mean' for average of next N days
-
-        Returns:
-        - Modified DataFrame with a new column 'TARGET_PRICES'
-        """
-        # Ensure chronological order
-        group = group.sort_values(by='TRAN_DATE').copy()
-
-        # Extract close prices
-        close_prices = group['CLOSE_PRICE'].values
-
-        # Generate targets using vectorized slicing
-        if method == "single":
-            target = np.roll(close_prices, -lookahead_days)
-        elif method == "mean":
-            target = np.array([
-                np.mean(close_prices[i+1:i+1+lookahead_days])
-                if i + lookahead_days < len(close_prices) else np.nan
-                for i in range(len(close_prices))
-            ])
-        else:
-            raise ValueError(f"Unsupported method: {method}")
-
-        # Assign and clean
-        group['TARGET_PRICES'] = target
-        group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True)
-
-        return group
-
-
-    def calculate_mlbench_indicators(self, df):
-        df = df.copy()
-        
-        # ===== MOMENTUM INDICATORS (14) =====
-        df.ta.rsi(length=14, append=True)
-        df.ta.macd(fast=12, slow=26, signal=9, append=True)
-        stoch = df.ta.stoch(length=14, k=3, d=3, append=True)
-        df.ta.willr(length=14, append=True)
-        df.ta.cmo(length=14, append=True)
-        df.ta.cci(length=20, append=True)
-        df.ta.roc(length=12, append=True)
-        df.ta.mfi(length=14, append=True)
-        df.ta.fisher(length=9, append=True)
-        df.ta.kst(append=True)
-        df.ta.mom(length=10, append=True)
-        df.ta.tsi(long=25, short=13, append=True)
-        df.ta.uo(fast=7, medium=14, slow=28, append=True)
-        df.ta.ao(append=True)
-        
-        # ===== TREND INDICATORS (18) =====
-        df.ta.sma(length=20, append=True)
-        df.ta.sma(length=50, append=True)
-        df.ta.sma(length=200, append=True)
-        df.ta.ema(length=12, append=True)
-        df.ta.ema(length=26, append=True)
-        df.ta.ema(length=50, append=True)
-        df.ta.ema(length=200, append=True)
-        df.ta.adx(length=14, append=True)
-        df.ta.aroon(length=14, append=True)
-        df.ta.supertrend(length=10, multiplier=3, append=True)
-        df.ta.psar(af0=0.02, af=0.2, append=True)
-        df.ta.alma(length=9, append=True)
-        df.ta.dema(length=10, append=True)
-        df.ta.tema(length=10, append=True)
-        df.ta.hma(length=9, append=True)
-        df.ta.kama(length=10, append=True)
-        df.ta.linreg(length=25, append=True)
-        
-        # ===== VOLATILITY INDICATORS (14) =====
-        df['True_Range'] = np.maximum(
-            np.maximum(
-                df['High'] - df['Low'],
-                np.abs(df['High'] - df['Close'].shift())
-            ),
-            np.abs(df['Low'] - df['Close'].shift())
-        )
-        
-        bbands = df.ta.bbands(length=20, std=2, append=True)
-        df.ta.atr(length=14, append=True)
-        df.ta.natr(length=14, append=True)
-        df.ta.kc(length=20, scalar=2, append=True)
-        df.ta.stdev(length=20, append=True)
-        
-        df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252)
-        df['Donchian_High'] = df['High'].rolling(20).max()
-        df['Donchian_Low'] = df['Low'].rolling(20).min()
-        df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2
-        
-        hl_range = df['High'] - df['Low']
-        mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum()
-        df['Mass_Index'] = mass_index
-        
-        df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-        
-        atr_sum = df['True_Range'].rolling(14).sum()
-        high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9))
-        df['CHOP'] = 100 * high_low / np.log10(14)
-        
-        # ===== VOLUME INDICATORS (13) =====
-        df.ta.obv(append=True)
-        df.ta.ad(append=True)
-        df.ta.adosc(fast=3, slow=10, append=True)
-        df.ta.cmf(length=20, append=True)
-        
-        df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum()
-        df['Volume_SMA_20'] = df['Volume'].rolling(20).mean()
-        
-        klinger_fast = df['Volume'].ewm(span=34).mean()
-        klinger_slow = df['Volume'].ewm(span=55).mean()
-        df['KO'] = klinger_fast - klinger_slow
-        
-        distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2
-        box_height = df['High'] - df['Low']
-        eom = distance_moved / (box_height / df['Volume'])
-        df['EOM'] = eom.rolling(14).mean()
-        
-        df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume']
-        df['EFI'] = df['EFI'].ewm(span=13).mean()
-        
-        tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2)
-        vm_plus = np.abs(df['High'] - df['Low'].shift(1))
-        vm_minus = np.abs(df['Low'] - df['High'].shift(1))
-        df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum()
-        df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum()
-        
-        # ===== OSCILLATORS & ADDITIONAL =====
-        df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['Median_Price'] = (df['High'] + df['Low']) / 2
-        df['HL2'] = (df['High'] + df['Low']) / 2
-        df['OC2'] = (df['Open'] + df['Close']) / 2
-        df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4
-        
-        df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['R1'] = 2 * df['Pivot'] - df['Low']
-        df['S1'] = 2 * df['Pivot'] - df['High']
-        df['R2'] = df['Pivot'] + (df['High'] - df['Low'])
-        df['S2'] = df['Pivot'] - (df['High'] - df['Low'])
-        
-        weights = np.arange(1, 21)
-        df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights))
-        df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean()
-        df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum()
-        
-        df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-        df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20']
-        
-        df['Price_vs_SMA20'] = df['Close'] - df['SMA_20']
-        df['Price_vs_EMA12'] = df['Close'] - df['EMA_12']
-        df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26']
-        df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200']
-        
-        df['RSI_Divergence'] = df['RSI_14'].diff()
-        df['Volume_Divergence'] = df['Volume'].diff()
-        
-        df['Momentum_Strength'] = df['RSI_14'].rolling(5).std()
-        df['Trend_Strength'] = df['ADX_14']
-        
-        df['Close_Change'] = df['Close'].pct_change()
-        df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close']
-        df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open']
-        df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean()
-        
-        return df
-
-    def create_features(self):
-        logging.info("Calculating technical indicators (105 total) and other features...")
-        
-        col_mapping = {
-            'OPEN_PRICE': 'Open',
-            'HIGH_PRICE': 'High',
-            'LOW_PRICE': 'Low',
-            'CLOSE_PRICE': 'Close',
-            'NET_TRDQTY': 'Volume'
-        }
-        
-        # Ensure correct column types
-        float_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'NET_TRDQTY', 'PREV_CL_PR']
-        for col in float_columns:
-            if col in self.df.columns:
-                self.df[col] = self.df[col].astype('float64')
-
-        grouped = self.df.groupby('SYMBOL', observed=False)
-        tasks = [(symbol, group, col_mapping) for symbol, group in grouped if len(group) >= 200]
-        
-        import multiprocessing
-        # Use available CPU cores minus 2 for efficiency and system responsiveness
-        num_cores = max(1, multiprocessing.cpu_count() - 2)
-        logging.info(f"Processing indicators in parallel using {num_cores} cores...")
-        
-        processed_groups = []
-        with multiprocessing.Pool(processes=num_cores) as pool:
-            for group_res in tqdm(pool.imap(_calculate_indicators_for_group, tasks), total=len(tasks), desc="Calculating indicators per symbol"):
-                processed_groups.append(group_res)
-                
-        self.df = pd.concat(processed_groups, ignore_index=True)
-        
-        # Add NIFTY50 Index features (2 features)
-        logging.info("Checking for NIFTY50 index data...")
-        if 'NIFTY50' in self.df['SYMBOL'].values:
-            logging.info("Calculating NIFTY50 index features...")
-            nifty_df = self.df[self.df['SYMBOL'] == 'NIFTY50'].copy()
-            nifty_df = nifty_df[['TRAN_DATE', 'CLOSE_PRICE']].rename(columns={'CLOSE_PRICE': 'NIFTY50_CLOSE'})
-            self.df = self.df.merge(nifty_df, on='TRAN_DATE', how='left')
-            self.df['NIFTY50_RETURN'] = self.df.groupby('SYMBOL', observed=False)['NIFTY50_CLOSE'].pct_change() * 100
-            self.df['NIFTY50_RETURN'] = self.df['NIFTY50_RETURN'].fillna(0.0)
-        else:
-            self.df['NIFTY50_CLOSE'] = 0.0
-            self.df['NIFTY50_RETURN'] = 0.0
-
-        # Replace infinity and NaNs
-        self.df.replace([np.inf, -np.inf], np.nan, inplace=True)
-        # Safe fillna/bfill for numeric columns
-        numeric_cols = self.df.select_dtypes(include=[np.number]).columns
-        self.df[numeric_cols] = self.df[numeric_cols].bfill().fillna(0.0)
-
-    def create_target(self):
-        # Add a column for the symbol name
-        self.df['SYMBOL'] = self.df['SYMBOL'].astype('category')
-        # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE
-        self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True)
-        self.df.reset_index(drop=True, inplace=True)
-        
-        # Calculate and populate RECOMMEND column using the chosen strategy
-        # Options: 'strategy' (Ultra Profit Maximizer), 'forward' (future price movement), 'simple' (basic up/down/flat)
-        target_creator_type = 'simple' # Options: 'forward', 'simple'
-        logging.info(f"Generating Type: {target_creator_type} target price for model training...")
-        
-        from tqdm import tqdm
-        tqdm.pandas(desc="Generating target prices per symbol")
-        if target_creator_type == 'forward':
-            self.df = self.df.groupby('SYMBOL').progress_apply(
-                self._create_forward_target, 
-                lookahead_days=1
-            ).reset_index(drop=True)
-        elif target_creator_type == 'simple':
-            self.df = self.df.groupby('SYMBOL', observed=False).progress_apply(
-                self._create_simple_target, 
-                lookahead_days=1
-            ).reset_index(drop=True)
-
-        # Verify if TARGET_PRICES column is created
-        if 'TARGET_PRICES' not in self.df.columns:
-            logging.error("TARGET_PRICES column not found after target creation...")
-
-    def run_correlation_analysis(self, df):
-        try:
-            print("\nRunning correlation analysis.")
-            numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-
-            # Drop rows with NaNs to ensure clean correlation
-            clean_df = df[numeric_cols].dropna()
-
-            # Compute correlation matrix
-            corr_matrix = clean_df.corr()
-
-            # Focus on correlation with CLOSE_PRICE
-            target_corr = corr_matrix['CLOSE_PRICE'].sort_values(ascending=False)
-
-            print("\n📈 Correlation with CLOSE_PRICE:")
-            print(target_corr.to_string(float_format=lambda x: f"{x:.4f}"))
-
-            # Optionally, show top 5 positively and negatively correlated features
-            print("\n⬆️ Top 5 positively correlated features:")
-            print(target_corr.drop('CLOSE_PRICE').head(5).to_string(float_format=lambda x: f"{x:.4f}"))
-
-            print("\n⬇️ Top 5 negatively correlated features:")
-            print(target_corr.drop('CLOSE_PRICE').tail(5).to_string(float_format=lambda x: f"{x:.4f}"))
-
-            # Mutual Information Analysis
-            print("\nCalculating Mutual Information Scores.")
-            X = clean_df.drop(columns=['CLOSE_PRICE'], errors='ignore')
-            y = clean_df['CLOSE_PRICE']
-            scaler = MinMaxScaler()
-            X_scaled = scaler.fit_transform(X)
-
-            mi_scores = mutual_info_regression(X_scaled, y)
-            mi_series = pd.Series(mi_scores, index=X.columns).sort_values(ascending=False)
-
-            print("\n📊 Mutual Information Scores:")
-            print(mi_series.to_string(float_format=lambda x: f"{x:.4f}"))
-
-        except Exception as e:
-            logging.error(f"Correlation analysis failed: {e}")
-
-
-    def save_extract_file(self):
-        # Final cleanup and save to Parquet
-        print("Cleaning up data before saving...")
-        logging.info("Saving processed data to Parquet file...")
-        self.df.dropna(inplace=True)
-        print("Sorting data by SYMBOL and TRAN_DATE...")
-        self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True)
-        self.df.reset_index(drop=True, inplace=True)
-        print(f"Saving to {self.extract_file.name} (compression='snappy')...")
-        self.df.to_parquet(self.extract_file, index=False, compression='snappy')
-        print("Saving extract file completed successfully.")
-        logging.info(f"Processed data saved to {self.extract_file}")
-
-    def load_data(self, label_creator_type = 'simple'):
-            """
-            Loads stock data from CSV, performs date filtering, and calculates
-            various technical indicators and lag features.
-            """
-            logging.info("Loading stock data... This may take a moment.")
-            try:
-                # Define data types for efficient memory usage and accurate loading
-                dtypes = {
-                    'SYMBOL': 'category', 'TRAN_DATE': str, 'OPEN_PRICE': 'float32',
-                    'HIGH_PRICE': 'float32', 'LOW_PRICE': 'float32', 'PREV_CL_PR': 'float32',
-                    'CLOSE_PRICE': 'float32', 'NET_TRDVAL': 'float32', 'NET_TRDQTY': 'float32',
-                    'TRADES': 'float32', 'HI_52_WK': 'float32', 'LO_52_WK': 'float32',
-                    'HI_DIFF': 'float32', 'LO_DIFF': 'float32'
-                }
-
-                # Load data from Parquet
-                self.df = pd.read_parquet(
-                    self.parquet_path,
-                    columns=list(dtypes.keys())
-                )
-
-                self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce')
-                self.df.dropna(subset=['TRAN_DATE'], inplace=True)
-
-                self.first_date = self.df['TRAN_DATE'].min()
-                self.last_date = self.df['TRAN_DATE'].max()
-
-                # Filter data for the last two years
-                scale_data = False
-                training_days = 365 * 2 # Years of data to consider for scaling
-                if scale_data:
-                    self.scaled_date = self.last_date - timedelta(days=training_days)
-                    logging.info(f"Processing transaction dates and filtering data from: {self.scaled_date.strftime('%Y-%m-%d')}...")
-                    mask = self.df['TRAN_DATE'] >= self.scaled_date
-                    latest_data = self.df[mask].copy()
-                    self.df = latest_data
-                else:
-                    logging.info(f"Processing transaction dates and filtering data from: {self.first_date.strftime('%Y-%m-%d')}...")
-        
-                # Sort data by SYMBOL and TRAN_DATE
-                self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True)
-
-                is_ti = getattr(self, 'ti_only', False) or not self.extract_file.exists()
-                if is_ti:
-                    # Create feature set from dataframe
-                    logging.info(f"Generating features and labels for model training on complete dataset...")
-                    self.create_features()
-                    print("Generating target prices.")
-                    self.create_target()
-                    # Drop null values from dataframe and drop index
-                    self.df.dropna(inplace=True)
-                    print("Saving extract file.")
-                    self.save_extract_file()
-                    print("Extract data generated and saved successfully.")
-                else:
-                    logging.info("Skipping technical indicators recalculation (use -ti switch to recalculate).")
-                
-                logging.info("Data loading and preprocessing completed successfully...")
-                # Show correlation analysis results
-                #self.run_correlation_analysis(self.df)
-                # Log the final DataFrame statistics
-                logging.info(f"Data loaded with {len(self.df)} rows and {len(self.df.columns)} columns.")
-                # Show all the column from dataframe
-                logging.info(f"Data columns: {', '.join(self.df.columns)}")
-                logging.info(f"Data contains {self.df['SYMBOL'].nunique()} unique symbols.")
-                logging.info(f"Data last date: {self.last_date.strftime('%Y-%m-%d')}, Data first date: {self.first_date.strftime('%Y-%m-%d')}")
-                logging.info(f"Data contains {self.df['TRAN_DATE'].nunique()} unique transaction dates.")  
-                # Show the first and lst few rows of the DataFrame for verification
-                logging.info(f"First 5 rows of the DataFrame:\n{self.df.head()}")
-                logging.info(f"Last 5 rows of the DataFrame:\n{self.df.tail()}")
-
-            except FileNotFoundError:
-                logging.error(f"Error: Input file not found at {self.input_file}. Please ensure 'nse_processed_data.csv' exists.")
-                sys.exit(1) # Exit if the critical input file is missing
-            except Exception as e:
-                logging.error(f"Failed to load and preprocess data: {e}")
-                sys.exit(1) # Exit on general data processing failure
-
-    def apply_corporate_actions(self, df):
-        """
-        Adjusts historical prices (OPEN_PRICE,HIGH_PRICE,LOW_PRICE,CLOSE_PRICE,PREV_CL_PR,HI_52_WK,LO_52_WK)
-        in the DataFrame based on corporate actions. Returns the adjusted DataFrame and the count of symbols for which actions were applied.
-        """
-        adjusted_symbols_count = 0
-
-        # Load corporate actions from JSON file
-        try:
-            if not self.corpact_file.exists():
-                self.logger.info(f"Corporate actions file not found: {self.corpact_file}. Skipping corporate actions.")
-                return df, 0 # Return original DataFrame if file not found
-            
-            with open(self.corpact_file, 'r') as f:
-                self.corporate_actions = json.load(f)
-            
-            if not self.corporate_actions:
-                self.logger.info("No corporate actions found in file.")
-                return df, 0 # Return original DataFrame if no actions are defined
-        except FileNotFoundError:
-            self.logger.error(f"Corporate actions file not found: {self.corpact_file}")
-            return df, 0
-        except json.JSONDecodeError:
-            self.logger.error(f"Error decoding corporate actions JSON from {self.corpact_file}. Check JSON format.")
-            return df, 0
-        except Exception as e:
-            self.logger.error(f"Failed to load corporate actions: {str(e)}")
-            return df, 0
-        
-        df_adjusted = df.copy()
-        df_adjusted['TRAN_DATE'] = pd.to_datetime(df_adjusted['TRAN_DATE']) # Ensure datetime type
-
-        print("Applying corporate actions...", end='', flush=True)
-        # Sort corporate actions by date in descending order to apply older actions first
-        # This ensures that if multiple actions for the same symbol happen, the adjustments cascade correctly
-        self.corporate_actions.sort(key=lambda x: datetime.strptime(x['date'], '%Y-%m-%d'), reverse=True)
-
-        for action in tqdm(self.corporate_actions, desc="Applying corporate actions"):
-            symbol = action['symbol']
-            action_date = datetime.strptime(action['date'], '%Y-%m-%d')
-            ratio_str = action['ratio']
-
-            # Corrected part: Check if 'type' key exists and its value is not None
-            if 'type' not in action or action['type'] is None:
-                self.logger.warning(f"Corporate action for {symbol} on {action_date.strftime('%Y-%m-%d')} is missing 'type' or 'type' is null. Skipping.")
-                continue
-            
-            action_type = action['type'].lower() # Now it's safe to call .lower()
-
-            # Parse ratio (e.g., "1:2" for split, "1:1" for bonus)
-            try:
-                if action_type == "split":
-                    old_shares, new_shares = map(int, ratio_str.split(':'))
-                    adjustment_factor = new_shares / old_shares
-                elif action_type == "bonus":
-                    bonus_shares, held_shares = map(int, ratio_str.split(':'))
-                    adjustment_factor = (held_shares + bonus_shares) / held_shares
-                else:
-                    self.logger.warning(f"Unsupported corporate action type: {action_type} for {symbol} on {action_date.strftime('%Y-%m-%d')}. Skipping.")
-                    continue
-            except ValueError:
-                self.logger.error(f"Invalid ratio format for {symbol} {action_type}: {ratio_str}. Skipping.")
-                continue
-
-            # Select data for the specific symbol and dates *before* the corporate action
-            # Prices before the action date need to be adjusted
-            mask = (df_adjusted['SYMBOL'] == symbol) & (df_adjusted['TRAN_DATE'] < action_date)
-
-            if df_adjusted[mask].empty:
-                continue
-
-            # Apply adjustment to relevant price columns
-            # HI_52_WK and LO_52_WK are adjusted here as well
-            price_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'PREV_CL_PR', 'HI_52_WK', 'LO_52_WK']
-            
-            for col in price_columns:
-                if col in df_adjusted.columns:
-                    # Apply adjustment. Rounding to 2 decimal places for prices.
-                    df_adjusted.loc[mask, col] = (df_adjusted.loc[mask, col] / adjustment_factor).round(2).astype('float32')
-            
-            adjusted_symbols_count += 1
-            self.logger.info(f"Applied {action_type} action for {symbol} on {action_date.strftime('%Y-%m-%d')}. Factor: {adjustment_factor:.2f}")
-        
-        print("Done")
-        return df_adjusted, adjusted_symbols_count
-
-    def validate_schema(self, df):
-        """Validate and convert DataFrame against schema with safe type conversion"""
-        try:
-            validated_df = pd.DataFrame()
-            
-            for col, dtype in self.schema.items():
-                if col in df.columns:
-                    try:
-                        # Handle integer columns safely
-                        if dtype in ['int32', 'int64']:
-                            # First convert to float to handle NaN values
-                            validated_df[col] = pd.to_numeric(df[col], errors='coerce')
-                            # Then convert to integer, replacing NaN with 0
-                            validated_df[col] = validated_df[col].fillna(0).astype(dtype)
-                        else:
-                            validated_df[col] = df[col].astype(dtype)
-                    except Exception as e:
-                        self.logger.warning(f"Type conversion failed for {col} to {dtype}: {str(e)}. Keeping original type.")
-                        # If conversion fails, keep original column but ensure it's copied
-                        validated_df[col] = df[col] 
-            
-            return validated_df
-            
-        except Exception as e:
-            self.logger.error(f"Schema validation error: {str(e)}")
-            return None
-
-    def save_processed_data(self, df):
-        """Save processed data with proper sorting and validation"""
-        try:
-            if df.empty:
-                raise ValueError("No data to save")
-            
-            # Check if this is fresh processing
-            is_fresh_save = not self.parquet_path.exists()
-            
-            # Ensure proper date format 
-            df['TRAN_DATE'] = pd.to_datetime(df['TRAN_DATE'])
-            
-            # If existing file, merge with new data
-            if not is_fresh_save:
-                self.logger.info("Reading existing data for merge...")
-                existing_df = pd.read_parquet(self.parquet_path)
-                if not pd.api.types.is_datetime64_any_dtype(existing_df['TRAN_DATE']):
-                    existing_df['TRAN_DATE'] = pd.to_datetime(existing_df['TRAN_DATE'])
-                
-                # Concatenate and remove duplicates
-                self.logger.info("Merging with existing data...")
-                df = pd.concat([existing_df, df], ignore_index=True)
-                df = df.drop_duplicates(subset=['TRAN_DATE', 'SYMBOL'], keep='last')
-            
-            # Convert SYMBOL to category for efficient sorting
-            df['SYMBOL'] = df['SYMBOL'].astype('category')
-            
-            # Sort the final dataset
-            self.logger.info("Sorting output data...")
-            df = df.sort_values(
-                by=['TRAN_DATE', 'SYMBOL'],
-                ascending=[True, True]
-            ).reset_index(drop=True)
-            
-            print("\nSaving processed data...", end='', flush=True)
-            with tqdm(total=1, desc="Saving outputs") as pbar:
-                # Save Parquet
-                df.to_parquet(self.parquet_path, index=False)
-                pbar.update(1)
-            
-            # Log summary
-            self.logger.info(
-                f"{'Created new' if is_fresh_save else 'Updated'} output files\n"
-                f"Total records: {len(df):,}\n"
-                f"Date range: {df['TRAN_DATE'].min():%Y-%m-%d} to "
-                f"{df['TRAN_DATE'].max():%Y-%m-%d}\n"
-                f"Files:\n{self.parquet_path}"
-            )
-            
-            # Print summary
-            print(f"\nData {'created' if is_fresh_save else 'updated'} successfully")
-            print(f"Records: {len(df):,}")
-            print(f"Date range: {df['TRAN_DATE'].min():%Y-%m-%d} to "
-                f"{df['TRAN_DATE'].max():%Y-%m-%d}")
-            
-            # Return final DataFrame
-            return df
-            
-        except Exception as e:
-            self.logger.error(f"Failed to save output: {str(e)}")
-            raise
-            return None
-
-    def process_all_files(self):
-        """
-        Process files with simplified progress display,
-        and optionally apply corporate actions for full processing runs.
-        """
-        self.metrics['start_time'] = time.time()
-        
-        # Determine if this is a full processing run
-        latest_date = self.get_latest_processed_date()
-        is_full_processing = (latest_date is None)
-        
-        # Find and sort ZIP files
-        zip_files = []
-        for zip_path in self.downloads_dir.rglob("*.zip"):
-            try:
-                date_str = self.extract_date_from_filename(zip_path.name)
-                if date_str:
-                    file_date = pd.to_datetime(date_str)
-                    if latest_date is None or file_date > latest_date:
-                        zip_files.append((file_date, zip_path))
-            except Exception:
-                # Log the error but continue if a single file has issues
-                self.logger.warning(f"Could not extract date or process filename for {zip_path.name}. Skipping.")
-                continue
-        
-        zip_files.sort()
-        self.new_dates = [d for d, _ in zip_files]
-        zip_files = [p for _, p in zip_files]
-        
-        if not zip_files:
-            print("No new files to process.")
-            if is_full_processing: # If no files and it's a full run, maybe apply corporate actions to existing data
-                print("However, performing corporate action check on existing data (if any).")
-                try:
-                    # Load existing data if available to apply corporate actions
-                    if self.parquet_path.exists():
-                        final_df = pd.read_parquet(self.parquet_path)
-                        if not pd.api.types.is_datetime64_any_dtype(final_df['TRAN_DATE']):
-                            final_df['TRAN_DATE'] = pd.to_datetime(final_df['TRAN_DATE'])
-                        final_df, applied_count = self.apply_corporate_actions(final_df)
-                        self.metrics['corporate_actions_applied_count'] = applied_count
-                        if applied_count > 0:
-                            print(f"\nApplied {applied_count} corporate actions to existing data.")
-                            self.save_processed_data(final_df) # Save updated data
-                        else:
-                            print("No new corporate actions applied to existing data.")
-                    else:
-                        print("No existing data found for corporate action application.")
-                except Exception as e:
-                    self.logger.error(f"Error during corporate action application on existing data: {str(e)}")
-                    print(f"Error applying corporate actions to existing data: {str(e)}")
-            # Check if extract file exists, if not then try to load data
-            if not self.extract_file.exists():
-                print("No extract file found, generating now.")
-                try:
-                    self.load_data()
-                except Exception as e:
-                    self.logger.error(f"Error loading extract data: {str(e)}")
-                    print(f"Error loading extract data: {str(e)}")
-            elif getattr(self, 'ti_only', False):
-                print("Recalculating technical indicators for the complete data sets...")
-                try:
-                    self.load_data()
-                except Exception as e:
-                    self.logger.error(f"Error recalculating technical indicators: {str(e)}")
-                    print(f"Error recalculating technical indicators: {str(e)}")
-            return
-            
-        print(f"Processing {len(zip_files)} files...", end='', flush=True)
-        
-        # Process in batches
-        batch_size = 10
-        processed_batches = []
-
-        with tqdm(total=len(zip_files), desc="Processing ZIP files", bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt}') as pbar:
-            for i in range(0, len(zip_files), batch_size):
-                batch = zip_files[i:i + batch_size]
-                batch_data = []
-
-                for zip_file in batch:
-                    try:
-                        df = self.process_zip_file(zip_file)
-                        if df is not None and not df.empty:
-                            batch_data.append(df)
-                            self.metrics['files_processed'] += 1
-                    except Exception as e:
-                        logging.warning(f"Failed to process {zip_file}: {e}")
-                    finally:
-                        pbar.update(1)
-
-                if batch_data:
-                    batch_df = pd.concat(batch_data, ignore_index=True)
-                    processed_batches.append(batch_df)
-                    gc.collect()
-        
-        if processed_batches:
-            print("\nMerging data...", end='', flush=True)
-            final_df = pd.concat(processed_batches, ignore_index=True)
-            # Sorting will happen inside save_processed_data as well if merging with existing.
-            # However, for consistency before corporate actions, we can sort here too.
-            final_df = final_df.sort_values(['TRAN_DATE', 'SYMBOL'])
-            print("Done")
-
-            # Apply corporate actions if it's a full processing run
-            if is_full_processing:
-                print("It's a full processing run. Checking for corporate actions...")
-                final_df, applied_count = self.apply_corporate_actions(final_df)
-                self.metrics['corporate_actions_applied_count'] = applied_count
-                if applied_count > 0:
-                    print(f"Successfully applied {applied_count} corporate actions.")
-                    self.logger.info(f"Applied {applied_count} corporate actions during full processing.")
-                else:
-                    print("No corporate actions found or applied.")
-                    self.logger.info("No corporate actions found or applied during full processing.")
-
-            # Save processed data
-            df = self.save_processed_data(final_df)
-
-            # Reload data to ensure consistency
-            self.load_data()
-
-            """
-            # Update self.df and create features/target
-            if df is not None:
-                self.df = df
-                self.create_features()
-                self.create_target()
-                self.save_extract_file()
-            else:
-                print("No data to save after processing.")
-                self.logger.warning("No data to save after processing.")
-            """
-            
-        duration = time.time() - self.metrics['start_time']
-        self.metrics['duration'] = duration
-        self.metrics['end_datetime'] = datetime.now()
-        start_dt = datetime.fromtimestamp(self.metrics['start_time']).strftime('%Y-%m-%d %H:%M:%S')
-
-        print(f"\n--- Processing Summary ---")
-        print(f"Recalculate Technical Indicators (-ti): {'Yes (Complete Datasets)' if getattr(self, 'ti_only', False) else ('Yes (Extract File Missing)' if not self.extract_file.exists() else 'No (Skipped)')}")
-        print(f"Started: {start_dt}")
-        print(f"Ended: {self.metrics['end_datetime'].strftime('%Y-%m-%d %H:%M:%S')}")
-        print(f"Duration: {self.metrics['duration']:.1f}s")
-        print(f"Files Found: {len(zip_files)}") # This is the count of files considered for processing
-        print(f"Files Processed (new/updated): {self.metrics['files_processed']}")
-        print(f"Files Failed: {self.metrics['files_failed']}")
-        print(f"Rows Processed: {self.metrics['rows_processed']:,}")
-        if is_full_processing:
-            print(f"Corporate Actions Applied: {self.metrics['corporate_actions_applied_count']}")
-        print(f"--------------------------")
-
-
-def main():
-    import argparse
-    parser = argparse.ArgumentParser(description="NSE Data Processor with Technical Indicators switch")
-    parser.add_argument("-ti", "--ti", action="store_true", help="Recalculate technical indicators only for the current data file")
-    args = parser.parse_args()
-
-    try:
-        processor = NSEDataProcessor()
-        processor.ti_only = args.ti
-        processor.process_all_files()
-    except KeyboardInterrupt:
-        print("\nProcessing interrupted by user.")
-    except Exception as e:
-        print(f"\nAn unhandled error occurred: {str(e)}")
-        logging.exception("An unhandled error occurred in main execution block.")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()
-
-

nsereco/__init__.py

@@ -1,22 +0,0 @@
-import sys
-import warnings
-sys.original_warn = warnings.warn
-
-from flask import Flask
-
-# Configure TensorFlow to only use CPU to avoid CUDA compatibility issues with compute capability 12.0
-try:
-    import tensorflow as tf
-    tf.config.set_visible_devices([], 'GPU')
-except Exception:
-    pass
-
-def create_app():
-    app = Flask(__name__)
-    
-    # Import and register routes
-    from . import routes
-    app.register_blueprint(routes.bp)
-    
-    return app
-

nsereco/agent_workflow.py

@@ -1,229 +0,0 @@
-import os
-import json
-import logging
-import asyncio
-from typing import Dict, TypedDict, Any
-from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
-
-from langchain_core.language_models.fake import FakeListLLM
-from langchain_core.tools import tool
-from crewai import Agent, Task, Crew, Process
-from langgraph.graph import StateGraph, START, END
-
-# Import the managers
-from nsereco.routes import get_managers
-
-# Define the state for LangGraph
-class WorkflowState(TypedDict):
-    task_type: str
-    num_symbols: int
-    worker_threads: int
-    results: str
-    status: str
-
-# Create a Fake LLM for CrewAI to run offline
-fake_llm = FakeListLLM(responses=["I will now execute the python function."])
-
-# ==========================================
-# CrewAI Agents & Tools Definition
-# ==========================================
-
-@tool
-def execute_optuna_tuning_tool(num_symbols: int, max_workers: int) -> str:
-    """Tool to execute Optuna tuning prioritizing NIFTY stocks."""
-    sm, rec = get_managers()
-    tracker_path = sm.data_dir / 'symbol_master_tracker.json'
-    
-    # Load tracker
-    if not tracker_path.exists():
-        return "symbol_master_tracker.json not found."
-    
-    with open(tracker_path, 'r') as f:
-        tracker = json.load(f)
-        
-    # Filter pending tuning
-    pending = [sym for sym, data in tracker.items() if data.get("tuning_status") == "PENDING"]
-    # Prioritize nifty stocks
-    nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is True]
-    non_nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is False]
-    
-    target_symbols = (nifty_pending + non_nifty_pending)[:num_symbols]
-    
-    if not target_symbols:
-        return "No pending symbols for tuning."
-    
-    logging.info(f"Agent executing Optuna tuning for {len(target_symbols)} symbols: {target_symbols}")
-    
-    from nsereco.task_manager import task_manager
-    task_manager.update_progress(0, len(target_symbols))
-    
-    # Run tuning (this runs sequentially or using its internal threads)
-    for idx, sym in enumerate(target_symbols):
-        try:
-            tracker[sym]["tuning_status"] = "IN_PROGRESS"
-            with open(tracker_path, 'w') as f:
-                json.dump(tracker, f, indent=4)
-            # Recommender function for tuning actually tunes all passed symbols, but we can pass just one at a time, or all.
-            # wait, run_optuna_tuning takes num_symbols. It will pick from NIFTY stocks.
-            # To force it to tune our target_symbols, we need to adapt it, but since we are controlling it, we can just use train_symbol or run_optuna_tuning.
-            # Actually, run_optuna_tuning does randomly pick from NIFTY. Let's just update run_optuna_tuning to accept a list of symbols if we wanted, or we just let it run.
-            rec.run_optuna_tuning(sm, num_trials=20, num_symbols=1, max_workers=max_workers, tuning_symbols=[sym])
-            task_manager.update_progress(idx + 1, len(target_symbols))
-        except Exception as e:
-            logging.error(f"Error tuning {sym}: {e}")
-            tracker[sym]["tuning_status"] = "FAILED"
-            with open(tracker_path, 'w') as f:
-                json.dump(tracker, f, indent=4)
-            task_manager.update_progress(idx + 1, len(target_symbols))
-                
-    return f"Optuna tuning completed for {target_symbols}"
-
-@tool
-def execute_model_training_tool(num_symbols: int, max_workers: int) -> str:
-    """Tool to execute Model training using best hyperparameters prioritizing NIFTY stocks."""
-    sm, rec = get_managers()
-    tracker_path = sm.data_dir / 'symbol_master_tracker.json'
-    
-    if not tracker_path.exists():
-        return "symbol_master_tracker.json not found."
-    
-    with open(tracker_path, 'r') as f:
-        tracker = json.load(f)
-        
-    # Filter tuning COMPLETED and training PENDING
-    pending = [sym for sym, data in tracker.items() 
-               if data.get("tuning_status") == "COMPLETED" and data.get("training_status") == "PENDING"]
-    
-    # Prioritize nifty stocks
-    nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is True]
-    non_nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is False]
-    
-    target_symbols = (nifty_pending + non_nifty_pending)[:num_symbols]
-    
-    if not target_symbols:
-        return "No pending symbols for training."
-        
-    logging.info(f"Agent executing Model training for {len(target_symbols)} symbols: {target_symbols}")
-    
-    from nsereco.task_manager import task_manager
-    task_manager.update_progress(0, len(target_symbols))
-    
-    for idx, sym in enumerate(target_symbols):
-        try:
-            tracker[sym]["training_status"] = "IN_PROGRESS"
-            with open(tracker_path, 'w') as f:
-                json.dump(tracker, f, indent=4)
-            
-            # Fetch best parameters
-            best_params = tracker[sym].get("optuna_best_parameters")
-            if not best_params:
-                best_params = rec.best_hyperparameters
-                
-            # We use the train_symbol or run_model_training_all_stocks logic here
-            sym_df = sm.df[sm.df['SYMBOL'] == sym].copy()
-            rec.train_model(sym, sym_df, sm, best_params, "agent_training")
-            
-            # Mark as completed
-            tracker[sym]["training_status"] = "COMPLETED"
-            with open(tracker_path, 'w') as f:
-                json.dump(tracker, f, indent=4)
-            task_manager.update_progress(idx + 1, len(target_symbols))
-                
-        except Exception as e:
-            logging.error(f"Error training {sym}: {e}")
-            tracker[sym]["training_status"] = "FAILED"
-            with open(tracker_path, 'w') as f:
-                json.dump(tracker, f, indent=4)
-            task_manager.update_progress(idx + 1, len(target_symbols))
- 
-    return f"Model training completed for {target_symbols}"
-
-# Define Agents
-tuning_agent = Agent(
-    role='Hyperparameter Tuning Optimizer',
-    goal='Optimize hyperparameters for stock prediction models.',
-    backstory='You are an expert AI responsible for tuning stock models to perfection.',
-    tools=[],
-    allow_delegation=False,
-    llm="fake"
-)
-
-training_agent = Agent(
-    role='Model Trainer',
-    goal='Train predictive models using the best hyperparameters.',
-    backstory='You are an expert AI data scientist responsible for training robust models.',
-    tools=[],
-    allow_delegation=False,
-    llm="fake"
-)
-
-# ==========================================
-# LangGraph Workflow Definition
-# ==========================================
-
-async def optuna_tuning_node(state: WorkflowState) -> WorkflowState:
-    num_symbols = state['num_symbols']
-    worker_threads = state['worker_threads']
-    
-    # Execute the tool directly
-    result = execute_optuna_tuning_tool.invoke({"num_symbols": num_symbols, "max_workers": worker_threads})
-    
-    return {"results": result, "status": "COMPLETED"}
-
-async def model_training_node(state: WorkflowState) -> WorkflowState:
-    num_symbols = state['num_symbols']
-    worker_threads = state['worker_threads']
-    
-    result = execute_model_training_tool.invoke({"num_symbols": num_symbols, "max_workers": worker_threads})
-    
-    return {"results": result, "status": "COMPLETED"}
-
-def route_task(state: WorkflowState):
-    if state['task_type'] == 'tune':
-        return 'optuna_tuning_node'
-    elif state['task_type'] == 'train':
-        return 'model_training_node'
-    return END
-
-# Build LangGraph
-workflow = StateGraph(WorkflowState)
-workflow.add_node("optuna_tuning_node", optuna_tuning_node)
-workflow.add_node("model_training_node", model_training_node)
-workflow.add_conditional_edges(START, route_task)
-workflow.add_edge("optuna_tuning_node", END)
-workflow.add_edge("model_training_node", END)
-
-app = workflow.compile()
-
-# ==========================================
-# Orchestrator Function
-# ==========================================
-
-def run_agent_workflow(task_type: str, num_symbols: int, worker_threads: int):
-    """
-    Entry point to trigger the workflow. Runs asynchronously.
-    """
-    import asyncio
-    
-    async def _run():
-        initial_state = {
-            "task_type": task_type,
-            "num_symbols": num_symbols,
-            "worker_threads": worker_threads,
-            "results": "",
-            "status": "STARTED"
-        }
-        
-        logging.info(f"Starting Agent Workflow: Task={task_type}, Symbols={num_symbols}")
-        final_state = await app.ainvoke(initial_state)
-        logging.info(f"Workflow Finished. Result: {final_state['results']}")
-        
-    try:
-        loop = asyncio.get_running_loop()
-        # If we are in an already running loop (like within a web worker thread sometimes), we can create a task
-        loop.create_task(_run())
-    except RuntimeError:
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        loop.run_until_complete(_run())

nsereco/data.py

@@ -1,749 +0,0 @@
-import pandas as pd
-import logging
-from pathlib import Path
-from datetime import datetime, timedelta
-import json
-import gc
-import warnings
-import numpy as np
-import tensorflow as tf
-from tensorflow.keras.models import Sequential, load_model
-from tensorflow.keras.optimizers import Adam
-from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint
-from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate
-from tensorflow.keras.models import Model
-from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback
-import tensorflow.keras.backend as K
-import optuna
-from optuna.integration import TFKerasPruningCallback
-from rich.table import Table
-from rich.console import Console
-from rich import box
-import matplotlib.pyplot as plt
-import seaborn as sns
-import threading
-
-matplotlib_lock = threading.Lock()
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.utils.class_weight import compute_class_weight
-from sklearn.model_selection import train_test_split
-import time
-import sys
-import os
-import functools
-from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
-import threading
-from multiprocessing import Pool
-from itertools import product
-import signal
-import talib
-import pandas_ta_classic as ta
-import fastparquet
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.utils.data import DataLoader, TensorDataset
-from torch.optim import Adam
-from torch.nn import HuberLoss
-
-
-def _calculate_indicators_for_group(group_data_tuple):
-    symbol, group, col_mapping = group_data_tuple
-    df = group.copy()
-    
-    # Rename columns to match pandas_ta expectations
-    df = df.rename(columns=col_mapping)
-    
-    # ===== MOMENTUM INDICATORS (14) =====
-    df.ta.rsi(length=14, append=True)
-    df.ta.macd(fast=12, slow=26, signal=9, append=True)
-    stoch = df.ta.stoch(length=14, k=3, d=3, append=True)
-    df.ta.willr(length=14, append=True)
-    df.ta.cmo(length=14, append=True)
-    df.ta.cci(length=20, append=True)
-    df.ta.roc(length=12, append=True)
-    df.ta.mfi(length=14, append=True)
-    df.ta.fisher(length=9, append=True)
-    df.ta.kst(append=True)
-    df.ta.mom(length=10, append=True)
-    df.ta.tsi(long=25, short=13, append=True)
-    df.ta.uo(fast=7, medium=14, slow=28, append=True)
-    df.ta.ao(append=True)
-    
-    # ===== TREND INDICATORS (18) =====
-    df.ta.sma(length=20, append=True)
-    df.ta.sma(length=50, append=True)
-    df.ta.sma(length=200, append=True)
-    df.ta.ema(length=12, append=True)
-    df.ta.ema(length=26, append=True)
-    df.ta.ema(length=50, append=True)
-    df.ta.ema(length=200, append=True)
-    df.ta.adx(length=14, append=True)
-    df.ta.aroon(length=14, append=True)
-    df.ta.supertrend(length=10, multiplier=3, append=True)
-    df.ta.psar(af0=0.02, af=0.2, append=True)
-    df.ta.alma(length=9, append=True)
-    df.ta.dema(length=10, append=True)
-    df.ta.tema(length=10, append=True)
-    df.ta.hma(length=9, append=True)
-    df.ta.kama(length=10, append=True)
-    df.ta.linreg(length=25, append=True)
-    
-    # ===== VOLATILITY INDICATORS (14) =====
-    df['True_Range'] = np.maximum(
-        np.maximum(
-            df['High'] - df['Low'],
-            np.abs(df['High'] - df['Close'].shift())
-        ),
-        np.abs(df['Low'] - df['Close'].shift())
-    )
-    
-    bbands = df.ta.bbands(length=20, std=2, append=True)
-    df.ta.atr(length=14, append=True)
-    df.ta.natr(length=14, append=True)
-    df.ta.kc(length=20, scalar=2, append=True)
-    df.ta.stdev(length=20, append=True)
-    
-    df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252)
-    df['Donchian_High'] = df['High'].rolling(20).max()
-    df['Donchian_Low'] = df['Low'].rolling(20).min()
-    df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2
-    
-    hl_range = df['High'] - df['Low']
-    mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum()
-    df['Mass_Index'] = mass_index
-    
-    df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-    
-    atr_sum = df['True_Range'].rolling(14).sum()
-    high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9))
-    df['CHOP'] = 100 * high_low / np.log10(14)
-    
-    # ===== VOLUME INDICATORS (13) =====
-    df.ta.obv(append=True)
-    df.ta.ad(append=True)
-    df.ta.adosc(fast=3, slow=10, append=True)
-    df.ta.cmf(length=20, append=True)
-    
-    df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum()
-    df['Volume_SMA_20'] = df['Volume'].rolling(20).mean()
-    
-    klinger_fast = df['Volume'].ewm(span=34).mean()
-    klinger_slow = df['Volume'].ewm(span=55).mean()
-    df['KO'] = klinger_fast - klinger_slow
-    
-    distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2
-    box_height = df['High'] - df['Low']
-    eom = distance_moved / (box_height / df['Volume'])
-    df['EOM'] = eom.rolling(14).mean()
-    
-    df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume']
-    df['EFI'] = df['EFI'].ewm(span=13).mean()
-    
-    tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2)
-    vm_plus = np.abs(df['High'] - df['Low'].shift(1))
-    vm_minus = np.abs(df['Low'] - df['High'].shift(1))
-    df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum()
-    df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum()
-    
-    # ===== OSCILLATORS & ADDITIONAL =====
-    df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['Median_Price'] = (df['High'] + df['Low']) / 2
-    df['HL2'] = (df['High'] + df['Low']) / 2
-    df['OC2'] = (df['Open'] + df['Close']) / 2
-    df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4
-    
-    df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3
-    df['R1'] = 2 * df['Pivot'] - df['Low']
-    df['S1'] = 2 * df['Pivot'] - df['High']
-    df['R2'] = df['Pivot'] + (df['High'] - df['Low'])
-    df['S2'] = df['Pivot'] - (df['High'] - df['Low'])
-    
-    weights = np.arange(1, 21)
-    df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights))
-    df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean()
-    df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum()
-    
-    df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-    df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20']
-    
-    df['Price_vs_SMA20'] = df['Close'] - df['SMA_20']
-    df['Price_vs_EMA12'] = df['Close'] - df['EMA_12']
-    df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26']
-    df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200']
-    
-    df['RSI_Divergence'] = df['RSI_14'].diff()
-    df['Volume_Divergence'] = df['Volume'].diff()
-    
-    df['Momentum_Strength'] = df['RSI_14'].rolling(5).std()
-    df['Trend_Strength'] = df['ADX_14']
-    
-    df['Close_Change'] = df['Close'].pct_change()
-    df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close']
-    df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open']
-    df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean()
-    
-    # Revert column names to uppercase
-    inv_col_mapping = {v: k for k, v in col_mapping.items()}
-    df = df.rename(columns=inv_col_mapping)
-    
-    # Add sentiment & news features (4 features)
-    df['Sentiment'] = 0.0
-    df['News_Count'] = 0.0
-    df['News_Sentiment'] = 0.0
-    df['News_Impact'] = 0.0
-    
-    # Add hybrid strategy indicators (4 features)
-    df['VIX'] = 15.0
-    df['Vol_20MA'] = df['NET_TRDQTY'].rolling(window=20).mean().bfill().fillna(0.0)
-    df['Prev_High'] = df['HIGH_PRICE'].shift(1).bfill().fillna(0.0)
-    df['Breakout_Signal'] = ((df['CLOSE_PRICE'] > df['Prev_High']) & (df['NET_TRDQTY'] > df['Vol_20MA'])).astype(float)
-    
-    return df
-
-class StockData:
-    """Manages loading, preprocessing, and feature engineering of stock data."""
-    def __init__(self, base_dir=None):
-        self.base_dir = Path(base_dir or Path.cwd())
-        self.setup_paths()
-        self.setup_logging()
-        self.df = pd.DataFrame() # Main DataFrame for engineered features
-        self.profit_df = pd.DataFrame() # DataFrame used for profit maximizer backtesting
-        self.load_data(label_creator_type = 'simple') # Load and preprocess data on initialization
-        self.init_symbol_master_tracker() # Initialize unified tracking JSON
-
-    def init_symbol_master_tracker(self):
-        """Initializes the symbol_master_tracker.json file if it doesn't exist."""
-        from .utils import NIFTY_STOCKS, load_json_file, save_json_file
-        tracker_path = self.data_dir / 'symbol_master_tracker.json'
-        
-        existing_data = load_json_file(tracker_path, {})
-        all_symbols = self.get_all_symbols_from_df()
-        
-        updated = False
-        for sym in all_symbols:
-            if sym not in existing_data:
-                existing_data[sym] = {
-                    "nifty_indicator": sym in NIFTY_STOCKS,
-                    "optuna_best_parameters": None,
-                    "tuning_status": "PENDING",
-                    "training_status": "PENDING"
-                }
-                updated = True
-        
-        if updated:
-            save_json_file(tracker_path, existing_data)
-            logging.info(f"Initialized/Updated symbol_master_tracker.json with {len(existing_data)} symbols.")
-        self.symbol_master_tracker_path = tracker_path
-
-    def _create_forward_target(self, group, lookahead_days=1):
-        """
-        Creates a forward-looking regression target: the next closing price after `lookahead_days`.
-        Optionally allows for volatility computation or future annotation.
-        Returns a DataFrame with 'TARGET_PRICES' column containing single future prices.
-        """
-        # Ensure chronological order
-        group = group.sort_values(by='TRAN_DATE').copy()
-
-        # Extract future single-step targets
-        close_prices = group['CLOSE_PRICE'].values
-        future_targets = []
-
-        for i in range(len(close_prices)):
-            if i + lookahead_days < len(close_prices):
-                future_targets.append(close_prices[i + lookahead_days])
-            else:
-                future_targets.append(np.nan)
-
-        group['TARGET_PRICES'] = future_targets
-
-        # Drop rows where TARGET_PRICES is NaN
-        group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True)
-
-        return group
-
-    def _create_simple_target(self, group, lookahead_days=1):
-        """
-        Creates a regression target for each row: the next closing price.
-        Returns a DataFrame with an added column 'TARGET_PRICES' containing a single future price.
-        """
-        # Ensure chronological order
-        group = group.sort_values(by='TRAN_DATE').copy()
-
-        # Create single-step future target
-        future_targets = []
-        close_prices = group['CLOSE_PRICE'].values
-
-        for i in range(len(close_prices)):
-            if i + lookahead_days < len(close_prices):
-                future_targets.append(close_prices[i + lookahead_days])
-            else:
-                future_targets.append(np.nan)
-
-        group['TARGET_PRICES'] = future_targets
-
-        # Drop rows where TARGET_PRICES is NaN
-        group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True)
-
-        return group
-
-    def setup_paths(self):
-        """Sets up directory paths for data, models, and logs."""
-        self.data_dir = self.base_dir / 'processed'
-        self.recommender_dir = self.data_dir / 'recommender'
-        self.log_dir = self.base_dir / 'downloads'
-        self.watchlist_dir = self.data_dir / 'watchlists' 
-        self.input_file = self.data_dir / 'nse_processed_data.csv'
-        self.extract_file = self.data_dir / 'nse_extract_data.parquet'
-        self.extract_csvfile = self.data_dir / 'nse_extract_data.csv'
-        self.recommender_dir.mkdir(parents=True, exist_ok=True)
-        self.data_dir.mkdir(parents=True, exist_ok=True)
-        self.log_dir.mkdir(parents=True, exist_ok=True)
-
-    def setup_logging(self):
-        """Configures the logging system for console and file output."""
-        log_file_path = self.log_dir / 'nse_data_recommender.log'
-        # Remove existing handlers to prevent duplicate logs
-        for handler in logging.root.handlers[:]:
-            logging.root.removeHandler(handler)
-
-        logging.basicConfig(
-            level=logging.INFO,
-            format='%(asctime)s - %(levelname)s - %(message)s',
-            handlers=[
-                logging.FileHandler(log_file_path),
-                logging.StreamHandler(sys.stdout)
-            ]
-        )
-        # Suppress TensorFlow warnings for cleaner logs
-        tf.get_logger().setLevel('ERROR')
-        logging.getLogger('tensorflow').setLevel(logging.ERROR)
-
-    def get_all_symbols_from_df(self):
-        """Returns a list of all unique stock symbols in the loaded DataFrame."""
-        if hasattr(self, 'df') and not self.df.empty:
-            return self.df['SYMBOL'].unique().tolist()
-        return []
-    
-    @staticmethod
-    def run_correlation_analysis(df):
-        try:
-            logging.info("Running correlation analysis on engineered features...")
-            numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-
-            # Drop rows with NaNs to ensure clean correlation
-            clean_df = df[numeric_cols].dropna()
-
-            # Compute correlation matrix
-            corr_matrix = clean_df.corr()
-
-            # Focus on correlation with CLOSE_PRICE
-            target_corr = corr_matrix['CLOSE_PRICE'].sort_values(ascending=False)
-
-            print("\n📈 Correlation with CLOSE_PRICE:")
-            print(target_corr.to_string(float_format=lambda x: f"{x:.4f}"))
-
-            # Optionally, show top 5 positively and negatively correlated features
-            print("\n⬆️ Top 5 positively correlated features:")
-            print(target_corr.drop('CLOSE_PRICE').head(5).to_string(float_format=lambda x: f"{x:.4f}"))
-
-            print("\n⬇️ Top 5 negatively correlated features:")
-            print(target_corr.drop('CLOSE_PRICE').tail(5).to_string(float_format=lambda x: f"{x:.4f}"))
-
-            """
-            # Mutual Information Analysis
-            X = clean_df.drop(columns=['CLOSE_PRICE'], errors='ignore')
-            y = clean_df['CLOSE_PRICE']
-            scaler = MinMaxScaler()
-            X_scaled = scaler.fit_transform(X)
-
-            mi_scores = mutual_info_regression(X_scaled, y)
-            mi_series = pd.Series(mi_scores, index=X.columns).sort_values(ascending=False)
-
-            print("\n📊 Mutual Information Scores:")
-            print(mi_series.to_string(float_format=lambda x: f"{x:.4f}"))
-            """
-
-        except Exception as e:
-            logging.error(f"Correlation analysis failed: {e}")
-
-    def calculate_mlbench_indicators(self, df):
-        df = df.copy()
-        
-        # ===== MOMENTUM INDICATORS (14) =====
-        df.ta.rsi(length=14, append=True)
-        df.ta.macd(fast=12, slow=26, signal=9, append=True)
-        stoch = df.ta.stoch(length=14, k=3, d=3, append=True)
-        df.ta.willr(length=14, append=True)
-        df.ta.cmo(length=14, append=True)
-        df.ta.cci(length=20, append=True)
-        df.ta.roc(length=12, append=True)
-        df.ta.mfi(length=14, append=True)
-        df.ta.fisher(length=9, append=True)
-        df.ta.kst(append=True)
-        df.ta.mom(length=10, append=True)
-        df.ta.tsi(long=25, short=13, append=True)
-        df.ta.uo(fast=7, medium=14, slow=28, append=True)
-        df.ta.ao(append=True)
-        
-        # ===== TREND INDICATORS (18) =====
-        df.ta.sma(length=20, append=True)
-        df.ta.sma(length=50, append=True)
-        df.ta.sma(length=200, append=True)
-        df.ta.ema(length=12, append=True)
-        df.ta.ema(length=26, append=True)
-        df.ta.ema(length=50, append=True)
-        df.ta.ema(length=200, append=True)
-        df.ta.adx(length=14, append=True)
-        df.ta.aroon(length=14, append=True)
-        df.ta.supertrend(length=10, multiplier=3, append=True)
-        df.ta.psar(af0=0.02, af=0.2, append=True)
-        df.ta.alma(length=9, append=True)
-        df.ta.dema(length=10, append=True)
-        df.ta.tema(length=10, append=True)
-        df.ta.hma(length=9, append=True)
-        df.ta.kama(length=10, append=True)
-        df.ta.linreg(length=25, append=True)
-        
-        # ===== VOLATILITY INDICATORS (14) =====
-        df['True_Range'] = np.maximum(
-            np.maximum(
-                df['High'] - df['Low'],
-                np.abs(df['High'] - df['Close'].shift())
-            ),
-            np.abs(df['Low'] - df['Close'].shift())
-        )
-        
-        bbands = df.ta.bbands(length=20, std=2, append=True)
-        df.ta.atr(length=14, append=True)
-        df.ta.natr(length=14, append=True)
-        df.ta.kc(length=20, scalar=2, append=True)
-        df.ta.stdev(length=20, append=True)
-        
-        df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252)
-        df['Donchian_High'] = df['High'].rolling(20).max()
-        df['Donchian_Low'] = df['Low'].rolling(20).min()
-        df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2
-        
-        hl_range = df['High'] - df['Low']
-        mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum()
-        df['Mass_Index'] = mass_index
-        
-        df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-        
-        atr_sum = df['True_Range'].rolling(14).sum()
-        high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9))
-        df['CHOP'] = 100 * high_low / np.log10(14)
-        
-        # ===== VOLUME INDICATORS (13) =====
-        df.ta.obv(append=True)
-        df.ta.ad(append=True)
-        df.ta.adosc(fast=3, slow=10, append=True)
-        df.ta.cmf(length=20, append=True)
-        
-        df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum()
-        df['Volume_SMA_20'] = df['Volume'].rolling(20).mean()
-        
-        klinger_fast = df['Volume'].ewm(span=34).mean()
-        klinger_slow = df['Volume'].ewm(span=55).mean()
-        df['KO'] = klinger_fast - klinger_slow
-        
-        distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2
-        box_height = df['High'] - df['Low']
-        eom = distance_moved / (box_height / df['Volume'])
-        df['EOM'] = eom.rolling(14).mean()
-        
-        df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume']
-        df['EFI'] = df['EFI'].ewm(span=13).mean()
-        
-        tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2)
-        vm_plus = np.abs(df['High'] - df['Low'].shift(1))
-        vm_minus = np.abs(df['Low'] - df['High'].shift(1))
-        df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum()
-        df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum()
-        
-        # ===== OSCILLATORS & ADDITIONAL =====
-        df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['Median_Price'] = (df['High'] + df['Low']) / 2
-        df['HL2'] = (df['High'] + df['Low']) / 2
-        df['OC2'] = (df['Open'] + df['Close']) / 2
-        df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4
-        
-        df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3
-        df['R1'] = 2 * df['Pivot'] - df['Low']
-        df['S1'] = 2 * df['Pivot'] - df['High']
-        df['R2'] = df['Pivot'] + (df['High'] - df['Low'])
-        df['S2'] = df['Pivot'] - (df['High'] - df['Low'])
-        
-        weights = np.arange(1, 21)
-        df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights))
-        df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean()
-        df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum()
-        
-        df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9)
-        df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20']
-        
-        df['Price_vs_SMA20'] = df['Close'] - df['SMA_20']
-        df['Price_vs_EMA12'] = df['Close'] - df['EMA_12']
-        df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26']
-        df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200']
-        
-        df['RSI_Divergence'] = df['RSI_14'].diff()
-        df['Volume_Divergence'] = df['Volume'].diff()
-        
-        df['Momentum_Strength'] = df['RSI_14'].rolling(5).std()
-        df['Trend_Strength'] = df['ADX_14']
-        
-        df['Close_Change'] = df['Close'].pct_change()
-        df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close']
-        df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open']
-        df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean()
-        
-        return df
-
-    def create_features(self):
-        logging.info("Calculating technical indicators (105 total) and other features...")
-        
-        col_mapping = {
-            'OPEN_PRICE': 'Open',
-            'HIGH_PRICE': 'High',
-            'LOW_PRICE': 'Low',
-            'CLOSE_PRICE': 'Close',
-            'NET_TRDQTY': 'Volume'
-        }
-        
-        # Ensure correct column types
-        float_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'NET_TRDQTY', 'PREV_CL_PR']
-        for col in float_columns:
-            if col in self.df.columns:
-                self.df[col] = self.df[col].astype('float64')
-
-        grouped = self.df.groupby('SYMBOL', observed=False)
-        tasks = [(symbol, group, col_mapping) for symbol, group in grouped if len(group) >= 200]
-        
-        import multiprocessing
-        # Use available CPU cores minus 2 for efficiency and system responsiveness
-        num_cores = max(1, multiprocessing.cpu_count() - 2)
-        logging.info(f"Processing indicators in parallel using {num_cores} cores...")
-        
-        from tqdm import tqdm
-        processed_groups = []
-        with multiprocessing.Pool(processes=num_cores) as pool:
-            for group_res in tqdm(pool.imap(_calculate_indicators_for_group, tasks), total=len(tasks), desc="Calculating indicators per symbol"):
-                processed_groups.append(group_res)
-                
-        self.df = pd.concat(processed_groups, ignore_index=True)
-        
-        # Add NIFTY50 Index features (2 features)
-        logging.info("Checking for NIFTY50 index data...")
-        if 'NIFTY50' in self.df['SYMBOL'].values:
-            logging.info("Calculating NIFTY50 index features...")
-            nifty_df = self.df[self.df['SYMBOL'] == 'NIFTY50'].copy()
-            nifty_df = nifty_df[['TRAN_DATE', 'CLOSE_PRICE']].rename(columns={'CLOSE_PRICE': 'NIFTY50_CLOSE'})
-            self.df = self.df.merge(nifty_df, on='TRAN_DATE', how='left')
-            self.df['NIFTY50_RETURN'] = self.df.groupby('SYMBOL', observed=False)['NIFTY50_CLOSE'].pct_change() * 100
-            self.df['NIFTY50_RETURN'] = self.df['NIFTY50_RETURN'].fillna(0.0)
-        else:
-            self.df['NIFTY50_CLOSE'] = 0.0
-            self.df['NIFTY50_RETURN'] = 0.0
-
-        # Replace infinity and NaNs
-        self.df.replace([np.inf, -np.inf], np.nan, inplace=True)
-        # Safe fillna/bfill for numeric columns
-        numeric_cols = self.df.select_dtypes(include=[np.number]).columns
-        self.df[numeric_cols] = self.df[numeric_cols].bfill().fillna(0.0)
-
-    def create_target(self):
-        # Add a column for the symbol name
-        self.df['SYMBOL'] = self.df['SYMBOL'].astype('category')
-        # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE
-        self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True)
-        self.df.reset_index(drop=True, inplace=True)
-        
-        # Calculate and populate RECOMMEND column using the chosen strategy
-        # Options: 'strategy' (Ultra Profit Maximizer), 'forward' (future price movement), 'simple' (basic up/down/flat)
-        target_creator_type = 'simple' # Options: 'forward', 'simple'
-        logging.info(f"Generating Type: {target_creator_type} target price for model training...")
-        
-        if target_creator_type == 'forward':
-            self.df = self.df.groupby('SYMBOL').apply(
-                self._create_forward_target, 
-                lookahead_days=1
-            ).reset_index(drop=True)
-        elif target_creator_type == 'simple':
-            self.df = self.df.groupby('SYMBOL').apply(
-                self._create_simple_target, 
-                lookahead_days=1
-            ).reset_index(drop=True)
-
-        # Verify if TARGET_PRICES column is created
-        if 'TARGET_PRICES' not in self.df.columns:
-            logging.error("TARGET_PRICES column not found after target creation...")
-
-    def save_extract_file(self):
-        # Save the processed DataFrame to a CSV for future use
-        # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE
-        # Drop any rows with NaN in any column to ensure clean data
-        self.df.dropna(inplace=True)
-        self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True)
-        self.df.reset_index(drop=True, inplace=True)
-        self.df.to_parquet(self.extract_file, index=False, compression='snappy')
-        # self.df.to_csv(self.extract_csvfile, index=False)
-        logging.info(f"Processed data saved to {self.extract_file}")
-
-    def load_data(self, label_creator_type = 'simple'):
-        """
-        Loads stock data from CSV, performs date filtering, and calculates
-        various technical indicators and lag features.
-        """
-        logging.info("Loading stock data... This may take a moment.")
-        try:
-            # Define data types for efficient memory usage and accurate loading
-            dtypes = {
-                'SYMBOL': 'category', 'TRAN_DATE': str, 'OPEN_PRICE': 'float32',
-                'HIGH_PRICE': 'float32', 'LOW_PRICE': 'float32', 'PREV_CL_PR': 'float32',
-                'CLOSE_PRICE': 'float32', 'NET_TRDVAL': 'float32', 'NET_TRDQTY': 'float32',
-                'TRADES': 'float32', 'HI_52_WK': 'float32', 'LO_52_WK': 'float32',
-                'HI_DIFF': 'float32', 'LO_DIFF': 'float32'
-            }
-
-            """
-            # Try to read the CSV file with specified dtypes
-            try:
-                self.df = pd.read_csv(
-                    self.input_file,
-                    usecols=list(dtypes.keys()),
-                    dtype=dtypes
-                )
-            except Exception as e:
-                logging.warning(f"Initial read_csv failed: {e}. Retrying without parse_dates...")
-            
-            self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce')
-            self.df.dropna(subset=['TRAN_DATE'], inplace=True)
-
-            self.first_date = self.df['TRAN_DATE'].min()
-            self.last_date = self.df['TRAN_DATE'].max()
-
-            # Filter data for the last two years
-            scale_data = False
-            training_days = 365 * 2 # Years of data to consider for scaling
-            if scale_data:
-                self.scaled_date = self.last_date - timedelta(days=training_days)
-                logging.info(f"Processing transaction dates and filtering data from: {self.scaled_date.strftime('%Y-%m-%d')}...")
-                mask = self.df['TRAN_DATE'] >= self.scaled_date
-                latest_data = self.df[mask].copy()
-                self.df = latest_data
-            else:
-                logging.info(f"Processing transaction dates and filtering data from: {self.first_date.strftime('%Y-%m-%d')}...")
-       
-            self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True)
-            """
-
-            # If extract file already exists, load it to avoid recomputing
-            loaded_from_parquet = False
-            if self.extract_file.exists():
-                logging.info(f"Extract file {self.extract_file} already exists. Loading existing data...")
-                try:
-                    self.df = pd.read_parquet(self.extract_file)
-                    # Verify required columns exist
-                    required_cols = ['HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'OPEN_PRICE', 'NET_TRDQTY']
-                    missing_cols = [col for col in required_cols if col not in self.df.columns]
-                    if missing_cols:
-                        logging.warning(f"Parquet file missing required columns: {missing_cols}. Falling back to CSV.")
-                        self.df = pd.DataFrame()
-                    else:
-                        loaded_from_parquet = True
-                except Exception as e:
-                    logging.warning(f"Initial read_parquet failed: {e}. Retrying to load csv...")
-            
-            if not loaded_from_parquet:
-                parquet_src = self.data_dir / 'nse_processed_data.parquet'
-                if parquet_src.exists():
-                    logging.info(f"Loading data from Parquet source: {parquet_src}")
-                    try:
-                        self.df = pd.read_parquet(parquet_src)
-                        try:
-                            self.save_extract_file()
-                        except Exception as e:
-                            logging.error(f"Failed to save updated extract file: {e}")
-                    except Exception as e:
-                        logging.warning(f"Initial read_parquet from source failed: {e}. Retrying with CSV...")
-                
-                if self.df.empty:
-                    if self.input_file.exists():
-                        logging.info(f"Loading data from CSV: {self.input_file}")
-                        try:
-                            self.df = pd.read_csv(self.input_file)
-                            try:
-                                self.save_extract_file()
-                            except Exception as e:
-                                logging.error(f"Failed to save updated extract file: {e}")
-                        except Exception as e:
-                            logging.warning(f"Initial read_csv failed: {e}. Exiting...")
-                            sys.exit(1)
-                    else:
-                        logging.error(f"Input file parquet and csv does not exist. Cannot load data exiting...")
-                        sys.exit(1)
-
-            self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce')
-            self.df.dropna(subset=['TRAN_DATE'], inplace=True)
-            # Sort on SYMBOL and TRAN_DATE and reset index to ensure contiguous arrays for each symbol group (prevents talib core dumps)
-            self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True)
-            self.first_date = self.df['TRAN_DATE'].min()
-            self.last_date = self.df['TRAN_DATE'].max()
-
-            logging.info("Data loading and preprocessing completed successfully...")
-            # Show correlation analysis results
-            # self.run_correlation_analysis(self.df)
-            # Log the final DataFrame statistics
-            logging.info(f"Data loaded with {len(self.df)} rows and {len(self.df.columns)} columns.")
-            # Show all the column from dataframe
-            logging.info(f"Data columns: {', '.join(self.df.columns)}")
-            logging.info(f"Data contains {self.df['SYMBOL'].nunique()} unique symbols.")
-            logging.info(f"Data last date: {self.last_date.strftime('%Y-%m-%d')}, Data first date: {self.first_date.strftime('%Y-%m-%d')}")
-            logging.info(f"Data contains {self.df['TRAN_DATE'].nunique()} unique transaction dates.")  
-            # Show the first and lst few rows of the DataFrame for verification
-            #logging.info(f"First 5 rows of the DataFrame:\n{self.df.head()}")
-            #logging.info(f"Last 5 rows of the DataFrame:\n{self.df.tail()}")
-
-            gc.collect() # Explicit garbage collection to free up memory
-
-        except FileNotFoundError:
-            logging.error(f"Error: Input file not found at {self.input_file}. Please ensure 'nse_processed_data.csv' exists.")
-            sys.exit(1) # Exit if the critical input file is missing
-        except Exception as e:
-            logging.error(f"Failed to load and preprocess data: {e}")
-            sys.exit(1) # Exit on general data processing failure
-
-    def select_features(self, corr_threshold=0.9999):
-        """
-        Selects relevant features by removing highly correlated ones.
-        Mutual Information (MI) filtering is not explicitly applied here based on the original code,
-        but the function is retained for clarity of purpose.
-        """
-        logging.info(f"Running feature selection using correlation. Number of features before selection: {len(self.df.columns)}")
-
-        numeric_cols = self.df.select_dtypes(include=[np.number]).columns.tolist()
-        
-        clean_df = self.df[numeric_cols].dropna()
-
-        # Drop highly correlated features based on absolute correlation
-        corr_matrix = clean_df.corr().abs()
-        # Select upper triangle of correlation matrix to avoid duplicates and self-correlation
-        upper_tri = corr_matrix.where(
-            pd.DataFrame(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool),
-                        columns=corr_matrix.columns, index=corr_matrix.index)
-        )
-        # Find features with correlation greater than threshold
-        to_drop_corr = [col for col in upper_tri.columns if any(upper_tri[col] > corr_threshold)]
-        reduced_df = clean_df.drop(columns=to_drop_corr)
-
-        selected_features = reduced_df.columns.tolist()
-        
-        logging.info(f"Selected {len(selected_features)} features after correlation filtering.")
-        return selected_features, to_drop_corr
-
-# --- SymbolRecommender Class (for model training and tuning) ---

nsereco/models.py

@@ -1,274 +0,0 @@
-import pandas as pd
-import logging
-from pathlib import Path
-from datetime import datetime, timedelta
-import json
-import gc
-import warnings
-import numpy as np
-import tensorflow as tf
-from tensorflow.keras.models import Sequential, load_model
-from tensorflow.keras.optimizers import Adam
-from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint
-from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate
-from tensorflow.keras.models import Model
-from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback
-import tensorflow.keras.backend as K
-import optuna
-from optuna.integration import TFKerasPruningCallback
-from rich.table import Table
-from rich.console import Console
-from rich import box
-import matplotlib.pyplot as plt
-import seaborn as sns
-import threading
-
-matplotlib_lock = threading.Lock()
-from sklearn.preprocessing import MinMaxScaler
-from sklearn.utils.class_weight import compute_class_weight
-from sklearn.model_selection import train_test_split
-import time
-import sys
-import os
-import functools
-from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
-import threading
-from multiprocessing import Pool
-from itertools import product
-import signal
-import talib
-import fastparquet
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.utils.data import DataLoader, TensorDataset
-from torch.optim import Adam
-from torch.nn import HuberLoss
-
-
-class FunctionalAttentionModel(nn.Module):
-    def __init__(self, input_dim, params):
-        super(FunctionalAttentionModel, self).__init__()
-        self.conv1d = nn.Conv1d(in_channels=input_dim, out_channels=64, kernel_size=3, padding=2)  # causal padding manually
-        self.bn_c1 = nn.BatchNorm1d(64)
-        self.dropout_c1 = nn.Dropout(0.2)
-
-        self.bilstm1 = nn.LSTM(input_size=64, hidden_size=params["lstm_units_1"], batch_first=True,
-                               bidirectional=True)
-        self.bn_lstm1 = nn.BatchNorm1d(2 * params["lstm_units_1"])
-        self.dropout_lstm1 = nn.Dropout(params["dropout_1"])
-
-        self.attention = nn.MultiheadAttention(embed_dim=2 * params["lstm_units_1"],
-                                               num_heads=1,
-                                               batch_first=True)
-
-        self.bilstm2 = nn.LSTM(input_size=4 * params["lstm_units_1"], hidden_size=params["lstm_units_1"],
-                               batch_first=True, bidirectional=True)
-        self.bn_lstm2 = nn.BatchNorm1d(2 * params["lstm_units_1"])
-        self.dropout_lstm2 = nn.Dropout(params["dropout_2"])
-
-        self.dense1 = nn.Linear(2 * params["lstm_units_1"], params["dense_units_1"])
-        self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"])
-        self.dropout_d1 = nn.Dropout(params["dropout_3"])
-
-        self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"])
-        self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"])
-        self.dropout_d2 = nn.Dropout(params["dropout_4"])
-
-        self.output = nn.Linear(params["dense_units_2"], 1)
-
-    def forward(self, x):
-        # x: (batch, seq_len, input_dim)
-        x = x.transpose(1, 2)  # (batch, input_dim, seq_len) for Conv1d
-        x = self.conv1d(x)[..., :-2]  # causal padding: remove last 2 timesteps
-        x = self.bn_c1(x)
-        x = self.dropout_c1(x)
-        x = x.transpose(1, 2)  # back to (batch, seq_len, channels)
-
-        lstm1_out, _ = self.bilstm1(x)
-        lstm1_out_bn = self.bn_lstm1(lstm1_out.transpose(1, 2)).transpose(1, 2)
-        lstm1_out_drop = self.dropout_lstm1(lstm1_out_bn)
-
-        attn_output, _ = self.attention(lstm1_out_drop, lstm1_out_drop, lstm1_out_drop)
-        concat = torch.cat([lstm1_out_drop, attn_output], dim=-1)
-
-        lstm2_out, _ = self.bilstm2(concat)
-        lstm2_out_bn = self.bn_lstm2(lstm2_out.transpose(1, 2)).transpose(1, 2)
-        lstm2_out = self.dropout_lstm2(lstm2_out_bn)
-
-        # Take last timestep output
-        last_output = lstm2_out[:, -1, :]
-
-        x = self.dense1(last_output)
-        x = self.bn_d1(x)
-        x = F.relu(x)
-        x = self.dropout_d1(x)
-
-        x = self.dense2(x)
-        x = self.bn_d2(x)
-        x = F.relu(x)
-        x = self.dropout_d2(x)
-
-        return self.output(x)
-    
-
-class LSTMModel(nn.Module):
-    def __init__(self, input_dim, params):
-        super(LSTMModel, self).__init__()
-        self.lstm_units = params["lstm_units_1"]
-
-        # Bidirectional LSTM Layer 1
-        self.bilstm1 = nn.LSTM(input_size=input_dim,
-                               hidden_size=self.lstm_units,
-                               batch_first=True,
-                               bidirectional=True)
-        self.bn_lstm1 = nn.BatchNorm1d(2 * self.lstm_units)
-        self.dropout_lstm1 = nn.Dropout(params["dropout_1"])
-
-        # Bidirectional LSTM Layer 2
-        self.bilstm2 = nn.LSTM(input_size=2 * self.lstm_units,
-                               hidden_size=self.lstm_units,
-                               batch_first=True,
-                               bidirectional=True)
-        self.bn_lstm2 = nn.BatchNorm1d(2 * self.lstm_units)
-        self.dropout_lstm2 = nn.Dropout(params["dropout_2"])
-
-        # Dense layers
-        self.dense1 = nn.Linear(2 * self.lstm_units, params["dense_units_1"])
-        self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"])
-        self.dropout_d1 = nn.Dropout(params["dropout_3"])
-
-        self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"])
-        self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"])
-        self.dropout_d2 = nn.Dropout(params["dropout_4"])
-
-        # Output layer
-        self.output = nn.Linear(params["dense_units_2"], 1)
-
-    def forward(self, x):
-        # x: (batch, seq_len, input_dim)
-        lstm1_out, _ = self.bilstm1(x)
-        lstm1_out_bn = self.bn_lstm1(lstm1_out.transpose(1, 2)).transpose(1, 2)
-        lstm1_out_drop = self.dropout_lstm1(lstm1_out_bn)
-
-        lstm2_out, _ = self.bilstm2(lstm1_out_drop)
-        lstm2_out_bn = self.bn_lstm2(lstm2_out.transpose(1, 2)).transpose(1, 2)
-        lstm2_out_drop = self.dropout_lstm2(lstm2_out_bn)
-
-        # Take last timestep
-        last_output = lstm2_out_drop[:, -1, :]
-
-        x = self.dense1(last_output)
-        x = self.bn_d1(x)
-        x = F.relu(x)
-        x = self.dropout_d1(x)
-
-        x = self.dense2(x)
-        x = self.bn_d2(x)
-        x = F.relu(x)
-        x = self.dropout_d2(x)
-
-        return self.output(x)
-
-class GRUAttentionModel(nn.Module):
-    def __init__(self, input_dim, params):
-        super(GRUAttentionModel, self).__init__()
-        self.conv1d = nn.Conv1d(in_channels=input_dim, out_channels=64, kernel_size=3, padding=2)  # causal padding
-        self.bn_c1 = nn.BatchNorm1d(64)
-        self.dropout_c1 = nn.Dropout(0.2)
-
-        self.bi_gru1 = nn.GRU(input_size=64, hidden_size=params["lstm_units_1"],
-                              batch_first=True, bidirectional=True)
-        self.bn_gru1 = nn.BatchNorm1d(2 * params["lstm_units_1"])
-        self.dropout_gru1 = nn.Dropout(params["dropout_1"])
-
-        self.attention = nn.MultiheadAttention(embed_dim=2 * params["lstm_units_1"],
-                                               num_heads=1,
-                                               batch_first=True)
-
-        self.bi_gru2 = nn.GRU(input_size=4 * params["lstm_units_1"], hidden_size=params["lstm_units_1"],
-                              batch_first=True, bidirectional=True)
-        self.bn_gru2 = nn.BatchNorm1d(2 * params["lstm_units_1"])
-        self.dropout_gru2 = nn.Dropout(params["dropout_2"])
-
-        self.dense1 = nn.Linear(2 * params["lstm_units_1"], params["dense_units_1"])
-        self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"])
-        self.dropout_d1 = nn.Dropout(params["dropout_3"])
-
-        self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"])
-        self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"])
-        self.dropout_d2 = nn.Dropout(params["dropout_4"])
-
-        self.output = nn.Linear(params["dense_units_2"], 1)
-
-    def forward(self, x):
-        # x: (batch, seq_len, input_dim)
-        x = x.transpose(1, 2)  # (batch, input_dim, seq_len) for Conv1d
-        x = self.conv1d(x)[..., :-2]  # causal padding: remove last 2 timesteps
-        x = self.bn_c1(x)
-        x = self.dropout_c1(x)
-        x = x.transpose(1, 2)  # back to (batch, seq_len, channels)
-
-        gru1_out, _ = self.bi_gru1(x)
-        gru1_out_bn = self.bn_gru1(gru1_out.transpose(1, 2)).transpose(1, 2)
-        gru1_out_drop = self.dropout_gru1(gru1_out_bn)
-
-        attn_output, _ = self.attention(gru1_out_drop, gru1_out_drop, gru1_out_drop)
-        concat = torch.cat([gru1_out_drop, attn_output], dim=-1)
-
-        gru2_out, _ = self.bi_gru2(concat)
-        gru2_out_bn = self.bn_gru2(gru2_out.transpose(1, 2)).transpose(1, 2)
-        gru2_out_drop = self.dropout_gru2(gru2_out_bn)
-
-        last_output = gru2_out_drop[:, -1, :]  # final timestep
-
-        x = self.dense1(last_output)
-        x = self.bn_d1(x)
-        x = F.relu(x)
-        x = self.dropout_d1(x)
-
-        x = self.dense2(x)
-        x = self.bn_d2(x)
-        x = F.relu(x)
-        x = self.dropout_d2(x)
-
-        return self.output(x)
-
-
-
-class HalveOnImprovement(Callback):
-    def __init__(self, monitor='val_loss', improve_count=3, reduction_factor=0.75, patience=5, verbose=1, min_lr=1e-6):
-        super().__init__()
-        self.monitor = monitor
-        self.patience = patience
-        self.verbose = verbose
-        self.min_lr = min_lr
-        self.best = float('inf')
-        self.epoch_counter = 0
-        self.wait = 0  # Tracks if any improvement occurred
-        self.improve_count = improve_count
-        self.reduction_factor = reduction_factor
-
-    def on_epoch_end(self, epoch, logs=None):
-        current = logs.get(self.monitor)
-        if current is None:
-            return
-
-        self.epoch_counter += 1
-
-        if current < self.best:
-            self.best = current
-            self.wait += 1  # Mark that improvement occurred
-
-        if self.epoch_counter >= self.patience:
-            if self.wait >= self.improve_count:
-                old_lr = float(K.get_value(self.model.optimizer.lr))
-                new_lr = max(old_lr * self.reduction_factor, self.min_lr)
-                K.set_value(self.model.optimizer.lr, new_lr)
-                print(f"\n[HalveOnImprovement] Epoch {epoch}: LR reduced from {old_lr:.6f} to {new_lr:.6f} after {self.epoch_counter} epochs with {self.wait} improvements.")
-                # Reset counters
-                self.epoch_counter = 0
-                self.wait = 0
-
-# --- Main Application Logic (Menu Driven) ---

nsereco/routes.py

@@ -1,260 +0,0 @@
-from flask import Blueprint, render_template, jsonify, request
-import logging
-import gc
-
-from .data import StockData
-from .recommender import SymbolRecommender
-from .task_manager import task_manager
-
-bp = Blueprint('main', __name__)
-
-# Global instances (initialized lazily or at startup)
-_stock_data_manager = None
-_recommender = None
-
-def get_managers():
-    global _stock_data_manager, _recommender
-    if _stock_data_manager is None:
-        logging.info("Initializing StockData...")
-        _stock_data_manager = StockData()
-        
-        # Check if features need to be created/loaded
-        if not SymbolRecommender.DL_FEATURES:
-            selected_features, dropped_features = _stock_data_manager.select_features()
-            SymbolRecommender.DL_FEATURES = selected_features
-            SymbolRecommender.dropped_features = dropped_features
-
-        expected_features = SymbolRecommender.DL_FEATURES
-        if 'NIFTY50' not in _stock_data_manager.df['SYMBOL'].values:
-            expected_features = [f for f in expected_features if not f.startswith('NIFTY50')]
-            
-        missing_features = [f for f in expected_features if f not in _stock_data_manager.df.columns]
-        if missing_features or 'TARGET_PRICES' not in _stock_data_manager.df.columns:
-            logging.info("Features or target prices missing from loaded data. Generating features and targets...")
-            _stock_data_manager.create_features()
-            _stock_data_manager.create_target()
-            try:
-                _stock_data_manager.save_extract_file()
-            except Exception as e:
-                logging.error(f"Failed to save updated extract file: {e}")
-
-        # Filter DL_FEATURES to only include columns that were successfully generated/loaded
-        SymbolRecommender.DL_FEATURES = [f for f in SymbolRecommender.DL_FEATURES if f in _stock_data_manager.df.columns]
-        
-        logging.info("Initializing SymbolRecommender...")
-        _recommender = SymbolRecommender(
-            recommender_dir=_stock_data_manager.recommender_dir,
-            training_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_recommender.json',
-            tuning_parameters_path=_stock_data_manager.data_dir / 'nsereco_parameters.json',
-            nifty_profit_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_nifty_profit.json',
-            all_profit_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_all_profit.json'
-        )
-    return _stock_data_manager, _recommender
-
-
-@bp.route('/')
-def index():
-    return render_template('index.html')
-
-@bp.route('/api/status')
-def status():
-    return jsonify(task_manager.get_status())
-
-@bp.route('/api/initialize', methods=['POST'])
-def initialize():
-    success, msg = task_manager.start_task("System Initialization", get_managers)
-    return jsonify({"success": success, "message": msg})
-
-@bp.route('/api/stop', methods=['POST'])
-def stop():
-    success, msg = task_manager.stop_task()
-    return jsonify({"success": success, "message": msg})
-
-@bp.route('/api/kill', methods=['POST'])
-def kill():
-    # Hard kill the backend process
-    import threading
-    import os
-    def shutdown():
-        import time
-        import logging
-        import os
-        import signal
-        time.sleep(1.5)
-        logging.warning("Graceful shutdown message due to UI kill switch")
-        os.kill(os.getpid(), signal.SIGINT)
-    threading.Thread(target=shutdown, daemon=True).start()
-    return jsonify({"success": True, "message": "Application shutting down."})
-
-import os
-from flask import send_from_directory
-
-@bp.route('/api/graphs')
-def list_graphs():
-    sm, _ = get_managers()
-    if not sm or not sm.recommender_dir:
-        return jsonify([])
-    recommender_dir = sm.recommender_dir
-    graphs = []
-    if os.path.exists(recommender_dir):
-        graphs = [f for f in os.listdir(recommender_dir) if f.startswith('nsereco_backtest_') and f.endswith('.png')]
-    return jsonify(sorted(graphs))
-
-@bp.route('/api/graphs/<filename>')
-def serve_graph(filename):
-    sm, _ = get_managers()
-    recommender_dir = sm.recommender_dir
-    return send_from_directory(recommender_dir, filename)
-
-@bp.route('/api/tune_selected', methods=['POST'])
-def tune_selected():
-    data = request.json
-    num_stocks = int(data.get('num_stocks', 20))
-    worker_threads = int(data.get('worker_threads', 8))
-    
-    def task():
-        from nsereco.agent_workflow import run_agent_workflow
-        run_agent_workflow('tune', num_stocks, worker_threads)
-        import gc
-        gc.collect()
-
-    success, msg = task_manager.start_task(f"Agentic Hyperparameter tuning ({num_stocks} stocks, {worker_threads} threads)", task)
-    return jsonify({"success": success, "message": msg})
-
-@bp.route('/api/train_all', methods=['POST'])
-def train_all():
-    data = request.json or {}
-    num_stocks = int(data.get('num_stocks', 20))
-    worker_threads = int(data.get('worker_threads', 8))
-
-    def task():
-        from nsereco.agent_workflow import run_agent_workflow
-        run_agent_workflow('train', num_stocks, worker_threads)
-        import gc
-        gc.collect()
-
-    success, msg = task_manager.start_task(f"Agentic Model training ({num_stocks} stocks, {worker_threads} threads)", task)
-    return jsonify({"success": success, "message": msg})
-
-@bp.route('/api/optuna_finder', methods=['POST'])
-def optuna_finder():
-    data = request.json
-    num_trials = int(data.get('num_trials', 30))
-    num_symbols = int(data.get('num_symbols', 1))
-    
-    def task():
-        sm, rec = get_managers()
-        rec.run_optuna_tuning(sm, num_trials=num_trials, num_symbols=num_symbols)
-        gc.collect()
-
-    success, msg = task_manager.start_task("Optuna hyperparameters finder", task)
-    return jsonify({"success": success, "message": msg})
-
-@bp.route('/api/analyze', methods=['POST'])
-def analyze():
-    data = request.json
-    symbol = data.get('symbol', 'SBIN')
-    
-    def task():
-        sm, rec = get_managers()
-        rec.analyze_tuner_data(symbol=symbol)
-
-    success, msg = task_manager.start_task(f"Tuner data analysis for {symbol}", task)
-    return jsonify({"success": success, "message": msg})
-
-
-@bp.route('/api/metrics')
-def get_metrics():
-    import json
-    import re
-    import os
-    from pathlib import Path
-    
-    # 1. Total symbols
-    global _stock_data_manager
-    total_symbols_list = []
-    if _stock_data_manager is not None and _stock_data_manager.df is not None:
-        total_symbols_list = sorted(_stock_data_manager.df['SYMBOL'].dropna().unique().astype(str).tolist())
-    else:
-        try:
-            data_dir = Path("/source/python/code/nsedata/analysis/processed")
-            symbol_list_path = data_dir / 'nse_dashboard_symbollist.json'
-            if symbol_list_path.exists():
-                with open(symbol_list_path, 'r') as f:
-                    symbol_dict = json.load(f)
-                    total_symbols_list = sorted(list(symbol_dict.keys()))
-        except Exception:
-            pass
-            
-    # 2. Tuned symbols
-    tuned_symbols = set()
-    for d in ["recommender", "recommender_torch"]:
-        path = Path("/source/python/code/nsedata/analysis/processed") / d
-        if path.exists():
-            for f in os.listdir(path):
-                if f.startswith('nsereco_tuner_') and f.endswith('.json'):
-                    sym = f[len('nsereco_tuner_'):-len('.json')]
-                    tuned_symbols.add(sym)
-    tuned_symbols_list = sorted(list(tuned_symbols))
-    
-    # 3. Trained symbols
-    trained_symbols_list = []
-    global _recommender
-    if _recommender is not None:
-        trained_symbols_list = [sym for sym, val in _recommender.training_tracker_data.items() if val.get("training", False)]
-    else:
-        try:
-            recommender_json_path = Path("/source/python/code/nsedata/analysis/processed/nse_dashboard_recommender.json")
-            if recommender_json_path.exists():
-                with open(recommender_json_path, 'r') as f:
-                    rec_data = json.load(f) or {}
-                    trained_symbols_list = [sym for sym, val in rec_data.items() if isinstance(val, dict) and val.get("training", False)]
-        except Exception:
-            pass
-    trained_symbols_list = sorted(trained_symbols_list)
-    
-    # 4. Backtest graphs symbols
-    backtest_symbols = set()
-    for d in ["recommender", "recommender_torch"]:
-        path = Path("/source/python/code/nsedata/analysis/processed") / d
-        if path.exists():
-            for f in os.listdir(path):
-                if f.startswith('nsereco_backtest_') and f.endswith('.png'):
-                    name = f[len('nsereco_backtest_'):-len('.png')]
-                    match = re.match(r'^([A-Za-z0-9\-&]+)(?:_v\d+)?$', name)
-                    if match:
-                        backtest_symbols.add(match.group(1))
-                    else:
-                        backtest_symbols.add(name.split('_')[0])
-    backtest_symbols_list = sorted(list(backtest_symbols))
-    
-    # 5. Tuning / Training in progress
-    status_data = task_manager.get_status()
-    is_running = status_data["is_running"]
-    status_str = status_data["status"].lower()
-    tuning_in_progress = is_running and ("tune" in status_str or "optuna" in status_str or "finder" in status_str)
-    training_in_progress = is_running and ("train" in status_str)
-    
-    return jsonify({
-        "total_symbols": {
-            "count": len(total_symbols_list),
-            "symbols": total_symbols_list
-        },
-        "tuned_symbols": {
-            "count": len(tuned_symbols_list),
-            "symbols": tuned_symbols_list
-        },
-        "trained_symbols": {
-            "count": len(trained_symbols_list),
-            "symbols": trained_symbols_list
-        },
-        "backtest_graphs": {
-            "count": len(backtest_symbols_list),
-            "symbols": backtest_symbols_list
-        },
-        "tuning_in_progress": tuning_in_progress,
-        "training_in_progress": training_in_progress,
-        "completed_count": status_data["completed_count"],
-        "total_count": status_data["total_count"]
-    })
-

nsereco/static/css/style.css

@@ -1,377 +0,0 @@
-:root {
-    --bg-dark: #0f172a;
-    --card-bg: rgba(30, 41, 59, 0.7);
-    --primary-color: #3b82f6;
-    --primary-hover: #2563eb;
-    --text-main: #f8fafc;
-    --text-muted: #94a3b8;
-    --success: #10b981;
-    --warning: #f59e0b;
-    --danger: #ef4444;
-}
-
-* {
-    box-sizing: border-box;
-    margin: 0;
-    padding: 0;
-}
-
-body {
-    font-family: 'Inter', sans-serif;
-    background-color: var(--bg-dark);
-    color: var(--text-main);
-    line-height: 1.6;
-    min-height: 100vh;
-    background: linear-gradient(135deg, #0f172a 0%, #1e1b4b 100%);
-}
-
-.app-container {
-    max-width: 1400px;
-    margin: 0 auto;
-    padding: 2rem;
-    display: flex;
-    flex-direction: column;
-    gap: 2rem;
-    height: 100vh;
-}
-
-.app-header {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-    border-bottom: 1px solid rgba(255, 255, 255, 0.1);
-    padding-bottom: 1rem;
-}
-
-.app-header h1 {
-    font-weight: 600;
-    letter-spacing: -0.5px;
-    background: -webkit-linear-gradient(#60a5fa, #a78bfa);
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-}
-
-.status-indicator {
-    display: flex;
-    align-items: center;
-    gap: 0.5rem;
-    font-weight: 600;
-    background: rgba(0, 0, 0, 0.3);
-    padding: 0.5rem 1rem;
-    border-radius: 20px;
-}
-
-.status-dot {
-    width: 10px;
-    height: 10px;
-    border-radius: 50%;
-    background-color: var(--success);
-    box-shadow: 0 0 8px var(--success);
-    transition: background-color 0.3s ease;
-}
-
-.status-dot.running {
-    background-color: var(--warning);
-    box-shadow: 0 0 8px var(--warning);
-    animation: pulse 1.5s infinite;
-}
-
-@keyframes pulse {
-    0% { transform: scale(0.95); opacity: 0.7; }
-    50% { transform: scale(1.1); opacity: 1; }
-    100% { transform: scale(0.95); opacity: 0.7; }
-}
-
-.dashboard-grid {
-    display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
-    gap: 1.5rem;
-}
-
-.glassmorphism {
-    background: var(--card-bg);
-    backdrop-filter: blur(12px);
-    -webkit-backdrop-filter: blur(12px);
-    border: 1px solid rgba(255, 255, 255, 0.08);
-    border-radius: 12px;
-    padding: 1.5rem;
-    box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2);
-    transition: transform 0.2s ease, box-shadow 0.2s ease;
-}
-
-.glassmorphism:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 15px 35px rgba(0, 0, 0, 0.3);
-}
-
-.card h2 {
-    font-size: 1.25rem;
-    margin-bottom: 0.75rem;
-    color: #e2e8f0;
-}
-
-.card p {
-    color: var(--text-muted);
-    font-size: 0.9rem;
-    margin-bottom: 1.5rem;
-}
-
-.form-group {
-    margin-bottom: 1rem;
-}
-
-.form-group label {
-    display: block;
-    font-size: 0.85rem;
-    color: var(--text-muted);
-    margin-bottom: 0.25rem;
-}
-
-.form-group input {
-    width: 100%;
-    background: rgba(0, 0, 0, 0.2);
-    border: 1px solid rgba(255, 255, 255, 0.1);
-    color: var(--text-main);
-    padding: 0.5rem 0.75rem;
-    border-radius: 6px;
-    font-family: inherit;
-}
-
-.form-group input:focus {
-    outline: none;
-    border-color: var(--primary-color);
-}
-
-.btn {
-    width: 100%;
-    padding: 0.75rem;
-    border: none;
-    border-radius: 6px;
-    font-family: inherit;
-    font-weight: 600;
-    cursor: pointer;
-    transition: all 0.2s ease;
-}
-
-.btn.primary {
-    background-color: var(--primary-color);
-    color: white;
-}
-
-.btn.primary:hover {
-    background-color: var(--primary-hover);
-}
-
-.btn:disabled {
-    opacity: 0.5;
-    cursor: not-allowed;
-}
-
-.console-section {
-    flex-grow: 1;
-    display: flex;
-    flex-direction: column;
-    min-height: 300px;
-    background: rgba(0, 0, 0, 0.5);
-    border: 1px solid rgba(255, 255, 255, 0.1);
-    border-radius: 12px;
-    overflow: hidden;
-}
-
-.console-header {
-    background: rgba(255, 255, 255, 0.05);
-    padding: 0.75rem 1.5rem;
-    border-bottom: 1px solid rgba(255, 255, 255, 0.1);
-}
-
-.console-header h3 {
-    font-size: 1rem;
-    font-weight: 600;
-}
-
-.console-window {
-    padding: 1rem 1.5rem;
-    overflow-y: auto;
-    font-family: 'Consolas', 'Courier New', monospace;
-    font-size: 0.85rem;
-    color: #a3e635; /* Terminal green */
-    line-height: 1.5;
-    flex-grow: 1;
-}
-
-.console-line {
-    margin-bottom: 2px;
-    word-wrap: break-word;
-}
-
-.console-actions {
-    display: flex;
-    gap: 0.5rem;
-}
-
-.btn.secondary {
-    background-color: #64748b;
-    color: white;
-}
-
-.btn.secondary:hover {
-    background-color: #475569;
-}
-
-.btn.warning {
-    background-color: var(--warning);
-    color: white;
-}
-
-.btn.warning:hover {
-    background-color: #d97706;
-}
-
-.btn.danger {
-    background-color: var(--danger);
-    color: white;
-}
-
-.btn.danger:hover {
-    background-color: #b91c1c;
-}
-
-/* Modal CSS */
-.modal {
-    display: none; 
-    position: fixed; 
-    z-index: 1000; 
-    left: 0;
-    top: 0;
-    width: 100%; 
-    height: 100%; 
-    overflow: auto; 
-    background-color: rgba(0,0,0,0.6); 
-    backdrop-filter: blur(5px);
-}
-
-.modal-content {
-    background-color: var(--bg-dark);
-    margin: 5% auto; 
-    padding: 20px;
-    border: 1px solid rgba(255, 255, 255, 0.1);
-    width: 80%;
-    max-width: 900px;
-    border-radius: 12px;
-    position: relative;
-}
-
-.close-btn {
-    color: var(--text-muted);
-    float: right;
-    font-size: 28px;
-    font-weight: bold;
-    cursor: pointer;
-}
-
-.close-btn:hover,
-.close-btn:focus {
-    color: white;
-    text-decoration: none;
-}
-
-select#graph-select {
-    width: 100%;
-    background: rgba(0, 0, 0, 0.2);
-    border: 1px solid rgba(255, 255, 255, 0.1);
-    color: var(--text-main);
-    padding: 0.5rem 0.75rem;
-    border-radius: 6px;
-    font-family: inherit;
-    margin-bottom: 1rem;
-}
-
-/* Metrics Summary Grid and Cards */
-.metrics-grid {
-    display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
-    gap: 1rem;
-    margin-bottom: 1rem;
-}
-
-.metric-card {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    justify-content: center;
-    padding: 1rem;
-    text-align: center;
-}
-
-.metric-label {
-    font-size: 0.8rem;
-    color: var(--text-muted);
-    font-weight: 600;
-    text-transform: uppercase;
-    letter-spacing: 0.5px;
-    margin-bottom: 0.5rem;
-}
-
-.metric-value {
-    font-size: 1.8rem;
-    font-weight: 600;
-    color: #3b82f6;
-    background: -webkit-linear-gradient(#60a5fa, #3b82f6);
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-}
-
-.metric-value.clickable {
-    cursor: pointer;
-    text-decoration: underline dotted;
-    text-underline-offset: 4px;
-    transition: transform 0.2s ease, opacity 0.2s ease;
-}
-
-.metric-value.clickable:hover {
-    transform: scale(1.05);
-    opacity: 0.85;
-}
-
-.metric-value.active-status {
-    color: var(--warning);
-    background: -webkit-linear-gradient(#fcd34d, #f59e0b);
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-    animation: statusPulse 2s infinite;
-}
-
-@keyframes statusPulse {
-    0% { opacity: 0.7; }
-    50% { opacity: 1; }
-    100% { opacity: 0.7; }
-}
-
-/* Symbols Grid in Modal */
-.symbols-list-grid {
-    display: grid;
-    grid-template-columns: repeat(auto-fill, minmax(100px, 1fr));
-    gap: 0.75rem;
-    margin-top: 1.5rem;
-    max-height: 400px;
-    overflow-y: auto;
-    padding-right: 0.5rem;
-}
-
-.symbol-badge {
-    background: rgba(255, 255, 255, 0.05);
-    border: 1px solid rgba(255, 255, 255, 0.1);
-    border-radius: 6px;
-    padding: 0.35rem 0.5rem;
-    text-align: center;
-    font-family: monospace;
-    font-size: 0.85rem;
-    color: var(--text-main);
-    transition: background 0.2s ease, border-color 0.2s ease;
-}
-
-.symbol-badge:hover {
-    background: rgba(59, 130, 246, 0.2);
-    border-color: var(--primary-color);
-}
-

nsereco/static/js/app.js

@@ -1,261 +0,0 @@
-document.addEventListener('DOMContentLoaded', () => {
-    const statusText = document.getElementById('system-status');
-    const statusDot = document.querySelector('.status-dot');
-    const consoleOutput = document.getElementById('console-output');
-    
-    let isTaskRunning = false;
-    let autoScroll = true;
-
-    // Detect if user scrolls up to disable autoscroll
-    consoleOutput.addEventListener('scroll', () => {
-        const isAtBottom = consoleOutput.scrollHeight - consoleOutput.scrollTop <= consoleOutput.clientHeight + 10;
-        autoScroll = isAtBottom;
-    });
-
-    function setButtonsDisabled(disabled) {
-        document.querySelectorAll('.btn:not(#btn-stop):not(#btn-view-graphs):not(#btn-kill)').forEach(btn => {
-            btn.disabled = disabled;
-        });
-        document.getElementById('btn-stop').disabled = !disabled;
-    }
-
-    function appendLogs(logs) {
-        if (!logs || logs.length === 0) return;
-        
-        // We only append new logs in a real app, but for simplicity we'll just rewrite it if it's small, 
-        // or since task_manager clears it on start, we just set innerHTML.
-        consoleOutput.innerHTML = logs.map(line => `<div class="console-line">${line}</div>`).join('');
-        
-        if (autoScroll) {
-            consoleOutput.scrollTop = consoleOutput.scrollHeight;
-        }
-    }
-
-    async function checkStatus() {
-        try {
-            const res = await fetch('/api/status');
-            const data = await res.json();
-            
-            isTaskRunning = data.is_running;
-            statusText.textContent = data.status;
-            
-            if (isTaskRunning) {
-                statusDot.classList.add('running');
-                setButtonsDisabled(true);
-            } else {
-                statusDot.classList.remove('running');
-                setButtonsDisabled(false);
-            }
-            
-            appendLogs(data.logs);
-        } catch (err) {
-            console.error('Error fetching status:', err);
-            statusText.textContent = 'Connection Error';
-            statusDot.classList.remove('running');
-        }
-    }
-
-    // Poll status every 2 seconds
-    setInterval(checkStatus, 2000);
-    checkStatus();
-
-    async function postAction(endpoint, payload = {}) {
-        if (isTaskRunning) return;
-        
-        try {
-            const res = await fetch(endpoint, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify(payload)
-            });
-            const data = await res.json();
-            if (!data.success) {
-                alert(data.message);
-            } else {
-                checkStatus();
-            }
-        } catch (err) {
-            alert('Failed to start task: ' + err.message);
-        }
-    }
-
-    document.getElementById('btn-init').addEventListener('click', () => {
-        postAction('/api/initialize');
-    });
-
-    document.getElementById('btn-tune').addEventListener('click', () => {
-        const numStocks = parseInt(document.getElementById('tune-stocks').value, 10);
-        const numThreads = parseInt(document.getElementById('tune-threads').value, 10);
-        postAction('/api/tune_selected', { num_stocks: numStocks, worker_threads: numThreads });
-    });
-
-    document.getElementById('btn-train').addEventListener('click', () => {
-        const numStocks = parseInt(document.getElementById('train-stocks').value, 10);
-        const numThreads = parseInt(document.getElementById('train-threads').value, 10);
-        postAction('/api/train_all', { num_stocks: numStocks, worker_threads: numThreads });
-    });
-
-
-    document.getElementById('btn-stop').addEventListener('click', async () => {
-        try {
-            const res = await fetch('/api/stop', { method: 'POST' });
-            const data = await res.json();
-            alert(data.message);
-        } catch (err) {
-            console.error(err);
-        }
-    });
-
-    document.getElementById('btn-kill').addEventListener('click', () => {
-        if (confirm("Are you sure you want to completely shut down the application?")) {
-            fetch('/api/kill', { method: 'POST' }).catch(err => console.log('Shutdown connection drop:', err));
-            alert("Application shutdown initiated.");
-            document.body.innerHTML = "<div style='text-align: center; margin-top: 100px;'><h1>Application Terminated</h1><p>You can close this window.</p></div>";
-        }
-    });
-
-    // Modal Logic
-    const modal = document.getElementById('graph-modal');
-    const closeBtn = document.querySelector('.close-btn');
-    const selectGraph = document.getElementById('graph-select');
-    const graphImg = document.getElementById('graph-img');
-
-    document.getElementById('btn-view-graphs').addEventListener('click', async () => {
-        modal.style.display = 'block';
-        try {
-            const res = await fetch('/api/graphs');
-            const graphs = await res.json();
-            selectGraph.innerHTML = '<option value="">Select a graph...</option>';
-            if (graphs.length === 0) {
-                selectGraph.innerHTML = '<option value="">No graphs available</option>';
-            } else {
-                graphs.forEach(g => {
-                    const option = document.createElement('option');
-                    option.value = g;
-                    option.textContent = g;
-                    selectGraph.appendChild(option);
-                });
-            }
-            graphImg.style.display = 'none';
-        } catch (err) {
-            console.error(err);
-        }
-    });
-
-    closeBtn.addEventListener('click', () => {
-        modal.style.display = 'none';
-    });
-
-    window.addEventListener('click', (e) => {
-        if (e.target === modal) {
-            modal.style.display = 'none';
-        }
-    });
-
-    selectGraph.addEventListener('change', (e) => {
-        const filename = e.target.value;
-        if (filename) {
-            // Add a timestamp query param to bypass browser cache
-            graphImg.src = `/api/graphs/${filename}?t=${new Date().getTime()}`;
-            graphImg.style.display = 'block';
-        } else {
-            graphImg.style.display = 'none';
-        }
-    });
-
-    // Metrics and Modal management
-    let currentMetrics = {
-        total_symbols: { count: 0, symbols: [] },
-        tuned_symbols: { count: 0, symbols: [] },
-        trained_symbols: { count: 0, symbols: [] },
-        backtest_graphs: { count: 0, symbols: [] }
-    };
-
-    async function fetchMetrics() {
-        try {
-            const res = await fetch('/api/metrics');
-            const data = await res.json();
-            currentMetrics = data;
-            
-            // Update UI elements
-            document.getElementById('metric-total-symbols').textContent = data.total_symbols.count;
-            document.getElementById('metric-tuned-symbols').textContent = data.tuned_symbols.count;
-            document.getElementById('metric-trained-symbols').textContent = data.trained_symbols.count;
-            document.getElementById('metric-backtest-graphs').textContent = data.backtest_graphs.count;
-            
-            // Tuning status
-            const tuningStatusEl = document.getElementById('metric-tuning-status');
-            if (data.tuning_in_progress) {
-                tuningStatusEl.textContent = `Running (${data.completed_count}/${data.total_count})`;
-                tuningStatusEl.classList.add('active-status');
-            } else {
-                tuningStatusEl.textContent = 'Idle';
-                tuningStatusEl.classList.remove('active-status');
-            }
-            
-            // Training status
-            const trainingStatusEl = document.getElementById('metric-training-status');
-            if (data.training_in_progress) {
-                trainingStatusEl.textContent = `Running (${data.completed_count}/${data.total_count})`;
-                trainingStatusEl.classList.add('active-status');
-            } else {
-                trainingStatusEl.textContent = 'Idle';
-                trainingStatusEl.classList.remove('active-status');
-            }
-        } catch (err) {
-            console.error('Error fetching metrics:', err);
-        }
-    }
-
-    // Call fetchMetrics inside status loop
-    const originalCheckStatus = checkStatus;
-    checkStatus = async function() {
-        await originalCheckStatus();
-        await fetchMetrics();
-    };
-
-    // Generic Symbols Modal
-    const symbolsModal = document.getElementById('symbols-modal');
-    const closeSymbolsBtn = document.getElementById('close-symbols-btn');
-    const symbolsTitle = document.getElementById('symbols-modal-title');
-    const symbolsContainer = document.getElementById('symbols-list-container');
-
-    function showSymbolsList(title, symbols) {
-        symbolsTitle.textContent = title;
-        symbolsContainer.innerHTML = '';
-        if (!symbols || symbols.length === 0) {
-            symbolsContainer.innerHTML = '<div style="grid-column: 1/-1; text-align: center; color: var(--text-muted); padding: 2rem;">No symbols available</div>';
-        } else {
-            symbols.forEach(sym => {
-                const badge = document.createElement('div');
-                badge.className = 'symbol-badge';
-                badge.textContent = sym;
-                symbolsContainer.appendChild(badge);
-            });
-        }
-        symbolsModal.style.display = 'block';
-    }
-
-    document.getElementById('metric-total-symbols').addEventListener('click', () => {
-        showSymbolsList('Total Symbols List', currentMetrics.total_symbols.symbols);
-    });
-    document.getElementById('metric-tuned-symbols').addEventListener('click', () => {
-        showSymbolsList('Tuned Symbols List', currentMetrics.tuned_symbols.symbols);
-    });
-    document.getElementById('metric-trained-symbols').addEventListener('click', () => {
-        showSymbolsList('Trained Symbols List', currentMetrics.trained_symbols.symbols);
-    });
-    document.getElementById('metric-backtest-graphs').addEventListener('click', () => {
-        showSymbolsList('Backtest Graphs Available', currentMetrics.backtest_graphs.symbols);
-    });
-
-    closeSymbolsBtn.addEventListener('click', () => {
-        symbolsModal.style.display = 'none';
-    });
-
-    window.addEventListener('click', (e) => {
-        if (e.target === symbolsModal) {
-            symbolsModal.style.display = 'none';
-        }
-    });
-});

nsereco/task_manager.py

@@ -1,96 +0,0 @@
-import threading
-import logging
-from collections import deque
-import traceback
-
-class TaskManager:
-    def __init__(self):
-        self.current_task = None
-        self.status = "Idle"
-        self.completed_count = 0
-        self.total_count = 0
-        self.logs = deque(maxlen=500)
-        self.lock = threading.Lock()
-        self.setup_custom_logger()
-
-    def update_progress(self, completed, total, status=None):
-        with self.lock:
-            self.completed_count = completed
-            self.total_count = total
-            if status is not None:
-                self.status = status
-
-    def increment_progress(self):
-        with self.lock:
-            self.completed_count += 1
-        
-    def setup_custom_logger(self):
-        class DequeHandler(logging.Handler):
-            def __init__(self, deque_obj):
-                super().__init__()
-                self.deque = deque_obj
-                
-            def emit(self, record):
-                try:
-                    msg = self.format(record)
-                    self.deque.append(msg)
-                except Exception:
-                    self.handleError(record)
-                    
-        self.handler = DequeHandler(self.logs)
-        self.handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
-        logging.getLogger().addHandler(self.handler)
-        # Ensure base logging level is INFO
-        logging.getLogger().setLevel(logging.INFO)
-
-    def start_task(self, name, func, *args, **kwargs):
-        with self.lock:
-            if self.current_task and self.current_task.is_alive():
-                return False, "A task is already running."
-            
-            self.status = f"Running: {name}"
-            self.completed_count = 0
-            self.total_count = 0
-            self.logs.clear()
-            logging.info(f"--- Started Task: {name} ---")
-            
-            import nsereco.utils as utils
-            utils.interrupted = False
-            
-            def task_wrapper():
-                try:
-                    func(*args, **kwargs)
-                    logging.info(f"--- Task {name} Completed Successfully ---")
-                except Exception as e:
-                    logging.error(f"Task Failed: {e}")
-                    logging.error(traceback.format_exc())
-                finally:
-                    with self.lock:
-                        self.status = "Idle"
-                        self.completed_count = 0
-                        self.total_count = 0
-                        
-            self.current_task = threading.Thread(target=task_wrapper, daemon=True)
-            self.current_task.start()
-            return True, "Task started successfully."
-
-    def stop_task(self):
-        with self.lock:
-            if self.current_task and self.current_task.is_alive():
-                import nsereco.utils as utils
-                utils.interrupted = True
-                logging.info("--- Cancel Requested. Waiting for graceful shutdown... ---")
-                return True, "Stop signal sent. Task is shutting down."
-            return False, "No task is currently running."
-
-    def get_status(self):
-        with self.lock:
-            return {
-                "status": self.status,
-                "is_running": self.current_task is not None and self.current_task.is_alive(),
-                "completed_count": self.completed_count,
-                "total_count": self.total_count,
-                "logs": list(self.logs)
-            }
-
-task_manager = TaskManager()

nsereco/templates/index.html

@@ -1,134 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>NSEReco Dashboard</title>
-    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
-    <!-- Google Fonts for modern typography -->
-    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap" rel="stylesheet">
-</head>
-<body>
-    <div class="app-container">
-        <header class="app-header">
-            <h1>NSEReco Dashboard</h1>
-            <div class="status-indicator">
-                <span class="status-dot"></span>
-                <span id="system-status">Checking status...</span>
-            </div>
-        </header>
-
-        <!-- Metrics Grid -->
-        <section class="metrics-grid">
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Total Symbols</div>
-                <div class="metric-value clickable" id="metric-total-symbols">-</div>
-            </div>
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Tuned Symbols</div>
-                <div class="metric-value clickable" id="metric-tuned-symbols">-</div>
-            </div>
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Trained Symbols</div>
-                <div class="metric-value clickable" id="metric-trained-symbols">-</div>
-            </div>
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Backtest Graphs</div>
-                <div class="metric-value clickable" id="metric-backtest-graphs">-</div>
-            </div>
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Tuning Status</div>
-                <div class="metric-value" id="metric-tuning-status">Idle</div>
-            </div>
-            <div class="metric-card glassmorphism">
-                <div class="metric-label">Training Status</div>
-                <div class="metric-value" id="metric-training-status">Idle</div>
-            </div>
-        </section>
-
-        <div class="top-actions" style="display: flex; gap: 1rem; margin-bottom: 1rem; justify-content: flex-end;">
-            <button id="btn-view-graphs" class="btn secondary" style="width: auto; padding: 0.5rem 1.5rem;">View Graphs</button>
-            <button id="btn-stop" class="btn warning" style="width: auto; padding: 0.5rem 1.5rem;" disabled>Stop Task</button>
-            <button id="btn-kill" class="btn danger" style="width: auto; padding: 0.5rem 1.5rem;">Kill App</button>
-        </div>
-
-        <main class="dashboard-grid">
-            <!-- Card 1: Initialization -->
-            <div class="card glassmorphism">
-                <h2>System Setup</h2>
-                <p>Initialize the data manager, load datasets, and engineer 50+ features. This must be done before running models.</p>
-                <button id="btn-init" class="btn primary">Initialize System</button>
-            </div>
-
-            <!-- Card 2: Hyperparameter Tuning -->
-            <div class="card glassmorphism">
-                <h2>1. Optuna tuning</h2>
-                <p>Check list of stock for pending tuning, prioritize nifty stocks, trigger number of symbols / threads for tuning via the Agent Workflow.</p>
-                <div class="form-group">
-                    <label for="tune-stocks">Number of Stocks:</label>
-                    <input type="number" id="tune-stocks" value="20" min="1" max="50">
-                </div>
-                <div class="form-group">
-                    <label for="tune-threads">Worker Threads:</label>
-                    <input type="number" id="tune-threads" value="8" min="1" max="16">
-                </div>
-                <button id="btn-tune" class="btn primary">Start Tuning</button>
-            </div>
-
-            <!-- Card 3: Model Training -->
-            <div class="card glassmorphism">
-                <h2>2. Model training</h2>
-                <p>Check for tuning completed but pending training, get best hyperparameters, prioritize nifty stocks, trigger training via the Agent Workflow.</p>
-                <div class="form-group">
-                    <label for="train-stocks">Number of Stocks:</label>
-                    <input type="number" id="train-stocks" value="20" min="1" max="50">
-                </div>
-                <div class="form-group">
-                    <label for="train-threads">Worker Threads:</label>
-                    <input type="number" id="train-threads" value="8" min="1" max="16">
-                </div>
-                <button id="btn-train" class="btn primary">Start Training</button>
-            </div>
-        </main>
-
-        <section class="console-section">
-            <div class="console-header">
-                <h3>Live Console Output</h3>
-            </div>
-            <div id="console-output" class="console-window">
-                <!-- Logs will be appended here -->
-            </div>
-        </section>
-    </div>
-
-    <!-- Graph Modal -->
-    <div id="graph-modal" class="modal">
-        <div class="modal-content glassmorphism">
-            <span class="close-btn">&times;</span>
-            <h2>Backtest Graphs</h2>
-            <div class="form-group">
-                <label for="graph-select">Select a graph:</label>
-                <select id="graph-select">
-                    <option value="">No graphs available</option>
-                </select>
-            </div>
-            <div class="graph-container">
-                <img id="graph-img" src="" alt="Backtest Graph" style="display: none; width: 100%; border-radius: 8px;">
-            </div>
-        </div>
-    </div>
-
-    <!-- Symbols List Modal -->
-    <div id="symbols-modal" class="modal">
-        <div class="modal-content glassmorphism">
-            <span class="close-btn" id="close-symbols-btn">&times;</span>
-            <h2 id="symbols-modal-title">Symbols List</h2>
-            <div id="symbols-list-container" class="symbols-list-grid">
-                <!-- Symbols will be populated here -->
-            </div>
-        </div>
-    </div>
-
-    <script src="{{ url_for('static', filename='js/app.js') }}"></script>
-</body>
-</html>