diff --git a/README.md b/README.md deleted file mode 100644 index 14537264aa77854a8a59f6dc60705d949cb211f3..0000000000000000000000000000000000000000 --- a/README.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Nsedash -emoji: 🌖 -colorFrom: yellow -colorTo: yellow -sdk: docker -pinned: false -short_description: NSE Symbols information in a Dashboard ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/app.py b/app.py index 8461b35d77c59bf7b5712025a5250349757f279a..378d268f0cdbeaab0fcf640c58d0aa7e5eca287a 100644 --- a/app.py +++ b/app.py @@ -44,7 +44,7 @@ def _silent_loader(): if not app_script.exists(): app_script = Path("nsedash_app.py") - args = [sys.executable, str(app_script), "-hf"] + args = [sys.executable, str(app_script), "--hf"] try: os.execv(sys.executable, args) except Exception: diff --git a/nsecorpact.py b/nsecorpact.py deleted file mode 100644 index 50fd6b05c0b9a217b30ea0db203bdd61c154993f..0000000000000000000000000000000000000000 --- a/nsecorpact.py +++ /dev/null @@ -1,284 +0,0 @@ -# This script processes corporate action data from the NSE and generates a JSON file -# containing the necessary information for dashboard integration. - -import pandas as pd -import re -import requests -import os -from datetime import datetime, timedelta - -# --- Configuration and Setup --- -# Directory to save processed files (final destination) -processed_dir = './processed' -# Temporary directory for downloads -temp_dir = './tempfiles' - -# Ensure both directories exist -os.makedirs(processed_dir, exist_ok=True) -os.makedirs(temp_dir, exist_ok=True) - -# Output JSON file path -output_file = os.path.join(processed_dir, 'nse_dashboard_corpact.json') - -# Base URL for downloading corporate actions data -DOWNLOAD_URL_TEMPLATE = "https://www.nseindia.com/api/corporates-corporateActions?index=equities&from_date=01-01-2020&to_date={}&csv=true" - -# URL for the main corporate actions page (to get session cookies) -NSE_CORPORATE_ACTIONS_PAGE = "https://www.nseindia.com/companies-listing/corporate-filings-actions" - -# --- Helper Functions for Date and Download --- - -def get_last_nse_working_day(): - """ - Determines the last NSE working day (i.e., the most recent weekday). - NSE is typically closed on Saturdays and Sundays. - """ - today = datetime.now() - # Subtract days until a weekday is found (Monday=0, Tuesday=1, ..., Sunday=6) - # If today is Saturday (5), subtract 1 to get Friday. - # If today is Sunday (6), subtract 2 to get Friday. - # If today is Monday-Friday, use today. - while today.weekday() > 4: # 5 is Saturday, 6 is Sunday - today -= timedelta(days=1) - return today.strftime('%d-%m-%Y') - -def download_nse_corporate_actions_file(): - """ - Downloads the corporate actions CSV file from NSE to the tempfiles directory. - It returns the path to the downloaded file in the temporary directory. - Returns None if download fails. - """ - last_working_day = get_last_nse_working_day() - download_url = DOWNLOAD_URL_TEMPLATE.format(last_working_day) - - # Construct the temporary file name and path - temp_file_name = f"CF-CA-equities-01-01-2020-to-{last_working_day}.csv" - temp_file_path = os.path.join(temp_dir, temp_file_name) - - # Note: We always attempt to download the latest to temp, no skipping here. - # The check for existing file is done at the processed_dir level later. - - print(f"Attempting to download corporate actions data to temporary directory.") - - # Create a session to persist cookies and headers - session = requests.Session() - - try: - # Step 1: Visit the main corporate actions page to get necessary cookies - print(f"Fetching cookies.") - session_headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'Accept-Language': 'en-US,en;q=0.9', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - } - session.get(NSE_CORPORATE_ACTIONS_PAGE, headers=session_headers) - - # Step 2: Use the same session to download the CSV file - print(f"Downloading CSV using established session.") - download_headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'text/csv', # Specifically request CSV - 'Accept-Language': 'en-US,en;q=0.9', - 'Connection': 'keep-alive', - 'Referer': NSE_CORPORATE_ACTIONS_PAGE, # Refer to the main page visited earlier - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'same-origin', - 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', - } - response = session.get(download_url, headers=download_headers) - response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx) - - # Get the raw binary content - raw_bytes_content = response.content - - # --- BOM Preprocessing --- - # Detect and remove UTF-8 BOM if present (EF BB BF) - if raw_bytes_content.startswith(b'\xef\xbb\xbf'): - # print("Detected and removed UTF-8 BOM from the file content.") - raw_bytes_content = raw_bytes_content[3:] - # --- End BOM Preprocessing --- - - # Decode to string for further preprocessing (only line stripping, no .strip('"')) - raw_content_string = raw_bytes_content.decode('latin1', errors='ignore') - - # Preprocessing: Only strip leading/trailing whitespace from each line - cleaned_lines = [line.strip() for line in raw_content_string.splitlines()] - cleaned_content = "\n".join(cleaned_lines) - - # Save the cleaned content to the temporary file path - with open(temp_file_path, 'w', encoding='latin1') as f: - f.write(cleaned_content) - - print(f"Successfully downloaded and preprocessed to temporary file.") - return temp_file_path - except requests.exceptions.RequestException as e: - print(f"Error downloading file: {e}") - print("This often indicates an issue with the NSE API (e.g., authentication required, rate limiting, or changed endpoint).") - print("You may still need to manually download the file if this persists.") - return None - -# --- Existing Functions (Copied from original prompt) --- - -# Function to calculate adjustment factor for bonuses -def calculate_bonus_adj_factor(purpose): - """Extracts and calculates the adjustment factor for bonus shares.""" - if "Bonus" in purpose: - try: - # Ratio format example: "1:4" in "Bonus 1:4" - # It's safer to use regex to extract the ratio explicitly - match = re.search(r"Bonus (\d+:\d+)", purpose) - if match: - ratio = match.group(1) - bonus, base = map(int, ratio.split(":")) - return f"{bonus}:{base}", "Bonus" - except ValueError: - print(f"Could not parse bonus ratio from: {purpose}") - return None, None - -# Function to calculate adjustment factor for stock splits -def calculate_split_adj_factor(purpose): - """Extracts and calculates the adjustment factor for stock splits.""" - if "Face Value Split" in purpose: - try: - # Use regex to find the 'From' and 'To' values - # Example: "Face Value Split (Sub-Division) - From Rs 10/- Per Share To Re 1/- Per Share" - match = re.search(r"From Rs (\d+(?:\.\d+)?)/?-? Per Share To (?:Re|Rs) (\d+(?:\.\d+)?)/?-? Per Share", purpose) - if match: - from_value_str = match.group(1) - to_value_str = match.group(2) - - # Convert to float to handle potential decimal values - from_value = float(from_value_str) - to_value = float(to_value_str) - - # Return ratio as 'new_face_value:old_face_value' - # For a split from 10 to 1, the ratio is 1:10 - return f"{int(to_value)}:{int(from_value)}", "Split" - else: - pass # No match found - except ValueError: - print(f"Could not parse split values from: {purpose}") - return None, None - -def main(): - temp_downloaded_file_path = None # Path of the file in tempfiles - input_file = None # Final path of the file in processed_dir - df = None # DataFrame for processing - - # Step 1: Attempt to download and preprocess the input file to tempfiles - temp_downloaded_file_path = download_nse_corporate_actions_file() - - # Define the final target file path in the processed directory - final_processed_file_name = 'Corporate_action.parquet' - final_processed_file_path = os.path.join(processed_dir, final_processed_file_name) - - if temp_downloaded_file_path and os.path.exists(temp_downloaded_file_path): - try: - # Read downloaded CSV - df_temp = pd.read_csv(temp_downloaded_file_path, encoding='latin1') - # Save to Parquet - df_temp.to_parquet(final_processed_file_path, index=False) - # Remove temp CSV - os.remove(temp_downloaded_file_path) - # Also clean up any legacy Corporate_action.csv in the processed dir - legacy_csv = os.path.join(processed_dir, 'Corporate_action.csv') - if os.path.exists(legacy_csv): - os.remove(legacy_csv) - input_file = final_processed_file_path - print("CSV downloaded file converted to Parquet and saved successfully.") - except Exception as e: - print(f"Error processing downloaded CSV file: {e}") - input_file = final_processed_file_path if os.path.exists(final_processed_file_path) else None - else: - print("\nAutomatic download to temporary directory failed or file not found in tempfiles.") - # Fallback: Check for the specifically named file in processed_dir - print(f"Checking for '{final_processed_file_name}' in '{processed_dir}' as a fallback...") - if os.path.exists(final_processed_file_path): - input_file = final_processed_file_path - print(f"Using existing file as input: '{input_file}'") - else: - # Check if there is an existing Corporate_action.csv to convert - fallback_csv = os.path.join(processed_dir, 'Corporate_action.csv') - if os.path.exists(fallback_csv): - try: - df_temp = pd.read_csv(fallback_csv, encoding='latin1') - df_temp.to_parquet(final_processed_file_path, index=False) - os.remove(fallback_csv) - input_file = final_processed_file_path - print(f"Auto-converted existing '{fallback_csv}' to Parquet.") - except Exception as e: - print(f"Failed to auto-convert existing CSV fallback: {e}") - - if not input_file: - print(f"No '{final_processed_file_name}' found in '{processed_dir}'. Cannot proceed without an input file.") - print(f"Please manually download the latest CSV from the following URL and place it in the '{temp_dir}' directory, then rerun the script:") - print(f" {DOWNLOAD_URL_TEMPLATE.format(get_last_nse_working_day())}") - return # Exit if no input file is available - - # Step 4: Use the determined input file as the dataframe source - if input_file and os.path.exists(input_file): # Ensure input_file exists before reading - try: - df = pd.read_parquet(input_file) - print(f"Successfully loaded data from '{input_file}'.") - except Exception as e: - print(f"Error reading input Parquet file '{input_file}': {e}") - return # Exit if reading the Parquet fails - else: - print("No valid input file determined or file does not exist. Exiting without processing.") - return - - # Initialize an output DataFrame - output_data = [] - - # Only proceed if df was successfully loaded - if df is not None: - for _, row in df.iterrows(): - symbol = row['SYMBOL'] - purpose = str(row['PURPOSE']) # Ensure purpose is a string - ex_date = row['EX-DATE'] - - ratio, corp_action = None, None - - # Calculate adjustment factor and corp action type - if "Bonus" in purpose: - ratio, corp_action = calculate_bonus_adj_factor(purpose) - elif "Face Value Split" in purpose: - ratio, corp_action = calculate_split_adj_factor(purpose) - - # Only add data if a valid corporate action and ratio were identified - if ratio and corp_action: - try: - # Convert date to YYYY-MM-DD format, handling potential errors - formatted_date = pd.to_datetime(ex_date, format='%d-%b-%Y').strftime('%Y-%m-%d') - output_data.append({ - "symbol": symbol, - "type": corp_action, - "date": formatted_date, - "ratio": ratio - }) - except ValueError as ve: - print(f"Error parsing date '{ex_date}' for symbol '{symbol}': {ve}") - except Exception as e: - print(f"An unexpected error occurred processing row for symbol '{symbol}': {e}") - - - # Convert output data to a DataFrame - output_df = pd.DataFrame(output_data) - - # Save to the target file in JSON format - output_df.to_json(output_file, orient='records', indent=4) - - print(f"\nCorporate action data processed and saved to {output_file}") - print(f"Total records processed: {len(df)}") - print(f"Total valid corporate actions extracted: {len(output_df)}") - else: - print("Dataframe 'df' was not created due to previous errors. No processing performed.") - - -if __name__ == "__main__": - main() - diff --git a/nsedash/__init__.py b/nsedash/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/nsedash/config.py b/nsedash/config.py deleted file mode 100644 index a6a02868fde19c4f8d9d2abafac8f3204f6b3613..0000000000000000000000000000000000000000 --- a/nsedash/config.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -nsedash.config -============== -Central configuration: directory layout, file paths, constants, index -constituent lists, and the Recommendation enum. - -All paths derive from a single base directory (default: current working -directory) and preserve the exact directory contract of nsedash.py v4.5: - - / - processed/ - nse_processed_data.csv - nse_summary_data.parquet - nse_dashboard_etflist.json - nse_dashboard_mclink.json - nse_dashboard_contents.json - nse_dashboard_config.json - recommender/ - watchlists/ - PortFolioEqtSummary.csv (optional) - SecurityMaster.zip (optional) - downloads/ - nse_dashboard.log - nse_dashboard_metrics.log -""" - -from __future__ import annotations - -import os -from enum import Enum -from pathlib import Path - -# --------------------------------------------------------------------------- -# Runtime mode flags (set by CLI switches or HF Space environment) -# --------------------------------------------------------------------------- - -#: True when running inside a Hugging Face Space (SPACE_ID is set) or -#: when the ``-hf`` CLI flag was passed. -HF_MODE = bool(int(os.environ.get("HF_MODE", "0"))) - -#: False when the ``-noadmin`` CLI flag was passed. Disables admin-only -#: actions (script runner, raw data tools, etc.). -ADMIN_MODE = bool(int(os.environ.get("ADMIN_MODE", "1"))) - -#: HuggingFace dataset repo used for two-way data persistence. -HF_DATASET_REPO = os.environ.get("NSEDASH_HF_DATASET", "Sam-max1/nsedash_data") - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -DATA_RETENTION_DAYS = 730 # 2-year historical data window -DEFAULT_REFRESH_RATE_MIN = 5 # live data polling interval (minutes) -DEFAULT_VOLUME_CUTOFF = 10_000 # minimum average volume for volume analysis -WATCHLIST_SLOTS = 20 # named watchlist slots -GRID_COLUMNS = 5 # dashboard grid columns (UI hint) -AUTO_QTY_BUDGET = 100_000 # ₹ budget for the auto-quantity heuristic -ANALYSIS_LIMIT = 50 # top-N rows for every analysis screen -HTTP_TIMEOUT_SEC = 10 # network timeout (AUDIT FIX C-1: original had none) -FETCH_MAX_WORKERS = 8 # concurrent symbol fetches (AUDIT FIX P-1) - -MARKET_TZ = "Asia/Kolkata" # NSE market timezone (AUDIT FIX H-8) -MARKET_OPEN = (9, 15) -MARKET_CLOSE = (15, 30) - -REQUEST_HEADERS = { - 'User-Agent': ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/125.0.0.0 Safari/537.36' - ), - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - 'Accept': ( - 'text/html,application/xhtml+xml,application/xml;' - 'q=0.9,image/webp,image/apng,*/*;q=0.8,' - 'application/signed-exchange;v=b3;q=0.7' - ), - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', -} - -CSV_COLUMNS = [ - 'SYMBOL', 'SECURITY', 'TRAN_DATE', - 'OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', - 'PREV_CL_PR', 'CLOSE_PRICE', 'NET_TRDVAL', - 'NET_TRDQTY', 'TRADES', 'HI_52_WK', 'LO_52_WK', - 'HI_DIFF', 'LO_DIFF' -] - -CSV_DTYPES = { - 'SYMBOL': 'category', - 'SECURITY': 'category', - 'OPEN_PRICE': 'float32', - 'HIGH_PRICE': 'float32', - 'LOW_PRICE': 'float32', - 'PREV_CL_PR': 'float32', - 'CLOSE_PRICE': 'float32', - 'NET_TRDVAL': 'float32', - 'NET_TRDQTY': 'float32', - 'TRADES': 'float32', - 'HI_52_WK': 'float32', - 'LO_52_WK': 'float32', - 'HI_DIFF': 'float32', - 'LO_DIFF': 'float32', -} - - -class Recommendation(str, Enum): - """LSTM model recommendation values.""" - BUY = "BUY" - SELL = "SELL" - HOLD = "HOLD" - NA = "N/A" - - -# --------------------------------------------------------------------------- -# Index Constituent Lists (retained verbatim from v4.5) -# --------------------------------------------------------------------------- - -SENSEX_STOCKS = [ - "ASIANPAINT", "AXISBANK", "BAJAJFINSV", "BAJFINANCE", "BHARTIARTL", - "BPCL", "DRREDDY", "HDFC", "HDFCBANK", "HEROMOTOCO", - "HINDUNILVR", "HCLTECH", "ICICIBANK", "INDUSINDBK", "INFY", - "ITC", "KOTAKBANK", "LT", "M&M", "MARUTI", - "NESTLEIND", "NTPC", "ONGC", "POWERGRID", "RELIANCE", - "SBIN", "SUNPHARMA", "TCS", "TATAMOTORS", "ULTRACEMCO" -] - -NIFTY_STOCKS = [ - "ADANIENT", "ADANIPORTS", "APOLLOHOSP", "ASIANPAINT", "AXISBANK", - "BAJAJ-AUTO", "BAJFINANCE", "BAJAJFINSV", "BEL", "BHARTIARTL", - "BPCL", "BRITANNIA", "CIPLA", "COALINDIA", "DRREDDY", - "EICHERMOT", "GRASIM", "HCLTECH", "HDFCLIFE", "HDFCBANK", - "HEROMOTOCO", "HINDALCO", "HINDUNILVR", "ICICIBANK", "INDUSINDBK", - "INFY", "ITC", "JIOFIN", "JSWSTEEL", "KOTAKBANK", - "LT", "M&M", "MARUTI", "NESTLEIND", "NTPC", - "ONGC", "POWERGRID", "RELIANCE", "SBIN", "SBILIFE", - "SHRIRAMFIN", "SUNPHARMA", "TATACONSUM", "TATAMOTORS", "TATASTEEL", - "TCS", "TECHM", "TITAN", "TRENT", "ULTRACEMCO", "WIPRO" -] - -UTILITY_SCRIPTS = [ - ("Download Daily Data", "nsedown.py"), - ("Process Corporate Actions", "nsecorpact.py"), - ("Process Daily Data", "nseprocessor.py"), - ("Process Max / Min", "nsesymhist.py"), - ("Download Live Data", "nsemaxlive.py"), -] - - - -class Paths: - """All filesystem paths for a given base directory.""" - - def __init__(self, base_dir: str | os.PathLike | None = None, session_token: str | None = None): - self.base_dir = Path(base_dir or os.environ.get("NSEDASH_BASE_DIR") or Path.cwd()) - self.session_token = session_token - self.data_dir = self.base_dir / 'processed' - - # Isolate session files - if session_token and session_token != "admin": - self.watchlist_dir = self.data_dir / 'watchlists' / f"session_{session_token}" - self.state_file = self.data_dir / f"nse_dashboard_contents_{session_token}.json" - self.config_file = self.data_dir / f"nse_dashboard_config_{session_token}.json" - self.portfolio_file = self.data_dir / f"PortFolioEqtSummary_{session_token}.parquet" - else: - self.watchlist_dir = self.data_dir / 'watchlists' - self.state_file = self.data_dir / 'nse_dashboard_contents.json' - self.config_file = self.data_dir / 'nse_dashboard_config.json' - self.portfolio_file = self.data_dir / 'PortFolioEqtSummary.parquet' - - self.log_dir = self.base_dir / 'downloads' - self.recommender_dir = self.data_dir / 'recommender' - self.aianalysis_dir = self.data_dir / 'aianalysis' - - self.input_file = self.data_dir / 'nse_processed_data.parquet' - self.extract_file = self.data_dir / 'nse_summary_data.parquet' - self.etf_list_file = self.data_dir / 'nse_dashboard_etflist.json' - self.mc_link_file = self.data_dir / 'nse_dashboard_mclink.json' - self.log_file = self.log_dir / 'nse_dashboard.log' - self.metrics_file = self.log_dir / 'nse_dashboard_metrics.log' - self.security_master_zip = self.data_dir / 'SecurityMaster.zip' - - wl = self.watchlist_dir - # Live analysis watchlists - self.max_vol_watchlist_file = wl / 'nse_dashboard_maxvol50.json' - self.max_price_watchlist_file = wl / 'nse_dashboard_maxprice50.json' - self.max_fall_watchlist_file = wl / 'nse_dashboard_maxfall50.json' - self.max_score_watchlist_file = wl / 'nse_dashboard_maxscore50.json' - self.max_return_files = { - "1W": wl / 'nse_dashboard_maxreturn1w50.json', - "1M": wl / 'nse_dashboard_maxreturn1m50.json', - "3M": wl / 'nse_dashboard_maxreturn3m50.json', - } - # Historical max/min price analysis watchlists - self.hist_price_files = { - ("1D", "Max"): wl / 'nse_dashboard_hist_maxprice1d50.json', - ("1D", "Min"): wl / 'nse_dashboard_hist_minprice1d50.json', - ("1W", "Max"): wl / 'nse_dashboard_hist_maxprice1w50.json', - ("1W", "Min"): wl / 'nse_dashboard_hist_minprice1w50.json', - ("1M", "Max"): wl / 'nse_dashboard_hist_maxprice1m50.json', - ("1M", "Min"): wl / 'nse_dashboard_hist_minprice1m50.json', - ("3M", "Max"): wl / 'nse_dashboard_hist_maxprice3m50.json', - ("3M", "Min"): wl / 'nse_dashboard_hist_minprice3m50.json', - } - # Historical volume analysis watchlists - self.hist_vol_files = { - "1D": wl / 'nse_dashboard_hist_maxvol1d50.json', - "1W": wl / 'nse_dashboard_hist_maxvol1w50.json', - "1M": wl / 'nse_dashboard_hist_maxvol1m50.json', - "3M": wl / 'nse_dashboard_hist_maxvol3m50.json', - } - - def watchlist_slot(self, n: int) -> Path: - """Path for numbered watchlist slot n (1..WATCHLIST_SLOTS).""" - return self.watchlist_dir / f"nse_dashboard_watchlist{n}.json" - - def ensure_dirs(self) -> None: - for d in (self.data_dir, self.data_dir / 'watchlists', self.watchlist_dir, self.log_dir, self.recommender_dir, self.aianalysis_dir): - d.mkdir(parents=True, exist_ok=True) diff --git a/nsedash/core/__init__.py b/nsedash/core/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/nsedash/core/dashboard_service.py b/nsedash/core/dashboard_service.py deleted file mode 100644 index ad4254ac524a17f20fc0a78ab795133c00ecb953..0000000000000000000000000000000000000000 --- a/nsedash/core/dashboard_service.py +++ /dev/null @@ -1,887 +0,0 @@ -""" -nsedash.core.dashboard_service -============================== -Server-side state controller — the headless successor to v4.5's -``StockDashboard``. All Tkinter dialogs become structured return payloads -(``{"ok":..., "message":..., ...}``) the frontend renders as toasts/modals; -all card widgets become JSON card payloads rendered by the browser. - -State parity with v4.5: - cells (ordered stock cards with Quantity/Price/current_cmp/selection), - stocks universe keyed "SYMBOL - SECURITY", selected_date, - current_watchlist_name, watchlist_modified, real_time_active, - live_data_source(+preference), live_refresh_rate, - volume_cutoff_threshold, selected_symbols_for_watchlist. -""" - -from __future__ import annotations - -import logging -import threading -from datetime import datetime, timedelta -from pathlib import Path -from zoneinfo import ZoneInfo - -import pandas as pd - -from ..config import (AUTO_QTY_BUDGET, DEFAULT_REFRESH_RATE_MIN, - DEFAULT_VOLUME_CUTOFF, GRID_COLUMNS, MARKET_CLOSE, - MARKET_OPEN, MARKET_TZ, NIFTY_STOCKS, SENSEX_STOCKS, - WATCHLIST_SLOTS, Paths) -from .portfolio import compute_portfolio_summary -from .stock_data import StockData -from .watchlists import WatchlistStore - -CHART_WINDOW = 90 # points displayed, parity with v4.5 update_graph -DMA_WINDOWS = (9, 20, 50, 200) - - -class Cell: - """One stock card's server-side state.""" - - __slots__ = ("symbol", "quantity", "price", "current_cmp", "prev_close", - "selected", "live", "recommendation") - - def __init__(self, symbol: str, quantity: float = 0.0, price: float = 0.0): - self.symbol = symbol - self.quantity = float(quantity) - self.price = float(price) - self.current_cmp = 0.0 - self.prev_close = 0.0 - self.selected = False - self.live: dict | None = None - self.recommendation: dict | None = None - - def get_data(self) -> dict: - """Parity with StockCell.get_data().""" - return {"SYMBOL": self.symbol, "Quantity": self.quantity, - "Price": self.price} - - -class DashboardService: - - def __init__(self, paths: Paths, hf_sync=None): - self.paths = paths - self._lock = threading.RLock() - self.stock_data = StockData(paths) - self.watchlists = WatchlistStore(paths, hf_sync=hf_sync) - - self.cells: list[Cell] = [] - self.stocks: dict[str, pd.Series] = {} - self.COLUMNS = GRID_COLUMNS - self.real_time_active = False - self.last_refresh = None - self.live_data_source = "N/A" - self.live_data_source_preference: str | None = None - self.live_refresh_rate = DEFAULT_REFRESH_RATE_MIN - self.watchlist_modified = False - self.volume_cutoff_threshold = DEFAULT_VOLUME_CUTOFF - self.selected_date = self.stock_data.last_date - self.watchlist_names = self.watchlists.load_watchlist_names() - - cfg = self.watchlists.load_config() - self.current_watchlist_name = cfg.get( - 'current_watchlist_name', "Default List" - ) - self._mc_links_cache: dict | None = None - self.load_universe() - self.load_dashboard_state_by_name(self.current_watchlist_name) - - # ================================================================== - # Universe / date data - # ================================================================== - - def load_universe(self) -> None: - """Build the stocks dict for the selected date (v4.5 load_data).""" - data = self.stock_data.get_data_for_date(self.selected_date) - with self._lock: - self.stocks = { - f"{row['SYMBOL']} - {row['SECURITY']}": row - for _, row in data.iterrows() - } - # AUDIT FIX P-8: symbol → key index removes the O(cells×stocks) - # linear prefix scans v4.5 performed throughout. - self._symbol_key_index = { - str(row['SYMBOL']): key for key, row in self.stocks.items() - } - self._mc_links_cache = None - - def key_for_symbol(self, symbol: str) -> str | None: - return self._symbol_key_index.get(symbol) - - def mc_links(self) -> dict: - if self._mc_links_cache is None: - self._mc_links_cache = self.stock_data.load_mc_link_data() - return self._mc_links_cache - - # ================================================================== - # Card payloads - # ================================================================== - - @staticmethod - def _chart_payload(price_history, volume_history) -> dict | None: - """ - 90-day chart series with 9/20/50/200 DMA computed over the full - history (parity with StockCell.update_graph). Sent to the browser - for client-side rendering. - """ - if not price_history or len(price_history) < 2: - return None - dates = [x[0] for x in price_history] - prices = [float(x[1]) for x in price_history] - volumes = [float(x[1]) for x in (volume_history or [])] - prices_series = pd.Series(prices) - start = max(0, len(dates) - CHART_WINDOW) - - def fmt_date(d): - if hasattr(d, 'strftime'): - return d.strftime('%Y-%m-%d') - return str(d)[:10] - - payload = { - "dates": [fmt_date(d) for d in dates[start:]], - "prices": prices[start:], - "volumes": volumes[start:] if volumes else [], - } - for w in DMA_WINDOWS: - if len(prices_series) >= w: - dma = prices_series.rolling(window=w).mean() - payload[f"dma_{w}"] = [ - None if pd.isna(v) else round(float(v), 2) - for v in dma.iloc[start:] - ] - else: - payload[f"dma_{w}"] = None - return payload - - def _cell_payload(self, cell: Cell, include_chart: bool = True) -> dict: - """Full card payload (historic + live + MC + P&L inputs).""" - key = self.key_for_symbol(cell.symbol) - row = self.stocks.get(key) if key else None - mc = self.mc_links().get(cell.symbol, {}) - payload = { - "symbol": cell.symbol, - "quantity": cell.quantity, - "price": cell.price, - "current_cmp": cell.current_cmp, - "selected": cell.selected, - "live": cell.live, - "recommendation": cell.recommendation, - "mc": { - "mcap": mc.get('MC_MCAP', 0.0), - "fv": mc.get('MC_FV', 0.0), - "score": mc.get('MC_SCORE', 0.0), - "ath": mc.get('MC_ATH', 0.0), - "atl": mc.get('MC_ATL', 0.0), - "link": mc.get('MC_LINK'), - }, - } - if row is not None: - prev_price = float(row['PREV_CL_PR']) - curr_price = float(row['CLOSE_PRICE']) - try: - daily_change = ( - (curr_price / prev_price - 1) * 100 if prev_price != 0 else 0.0 - ) - except (ValueError, TypeError): - daily_change = 0.0 - vc = row.get('VOLUME_CHANGES') or [] - padded = ([0.0] * (5 - len(vc)) + list(vc)) if len(vc) < 5 else list(vc)[:5] - payload.update({ - "security": str(row['SECURITY']), - "tran_date": row['TRAN_DATE'].strftime('%d %b %Y'), - "prev_close": prev_price, - "close_price": curr_price, - "hi_52wk": float(row['HI_52_WK']), - "hi_diff": float(row['HI_DIFF']), - "lo_52wk": float(row['LO_52_WK']), - "lo_diff": float(row['LO_DIFF']), - "changes": { - "1D": daily_change, - "1W": float(row.get('WEEK_CHANGE', 0) or 0), - "1M": float(row.get('MONTH_CHANGE', 0) or 0), - "3M": float(row.get('THREE_MONTH_CHANGE', 0) or 0), - }, - # Display order parity: reversed padded list, labelled D1..D5 - "volume_changes": padded[::-1], - }) - if include_chart: - payload["chart"] = self._chart_payload( - row.get('PRICE_HISTORY'), row.get('VOLUME_HISTORY') - ) - return payload - - def board_payload(self, include_charts: bool = True) -> dict: - """Complete dashboard snapshot for the frontend.""" - with self._lock: - return { - "banner": { - "mode": "live" if self.real_time_active else "historic", - "source": self.live_data_source, - "watchlist": self.current_watchlist_name, - "refresh_rate": self.live_refresh_rate, - }, - "selected_date": str(self.selected_date)[:10], - "last_data_date": str(self.stock_data.last_date)[:10], - "watchlist_modified": self.watchlist_modified, - "watchlist_names": self.watchlist_names, - "volume_cutoff": self.volume_cutoff_threshold, - "columns": self.COLUMNS, - "cells": [ - self._cell_payload(c, include_charts) for c in self.cells - ], - "portfolio": self.portfolio_summary(), - "search_keys": sorted(self.stocks.keys()), - "trading_hour": self.is_trading_hour(), - "indices": self.get_index_quotes(), - } - - # ================================================================== - # Cell management (add/remove/delete-all/inputs/selection) - # ================================================================== - - def add_stock(self, selected: str | None = None, quantity: float = 0.0, - price: float = 0.0, save_state: bool = True) -> Cell | None: - with self._lock: - if not selected or selected not in self.stocks: - logging.warning( - f"Skipping addition: stock '{selected}' not found in data." - ) - return None - row = self.stocks[selected] - symbol = str(row['SYMBOL']) - # Auto-quantity heuristic (v4.5 M-8): only on addition. - if float(quantity) == 0.0 and float(price) == 0.0: - curr_price = float(row.get('CLOSE_PRICE', 0) or 0) - if curr_price > 0: - price = curr_price - quantity = int(AUTO_QTY_BUDGET / curr_price) - cell = Cell(symbol, quantity, price) - cell.current_cmp = float(row.get('CLOSE_PRICE', 0) or 0) - cell.prev_close = float(row.get('PREV_CL_PR', 0) or 0) - self.cells.append(cell) - if save_state: - self.save_dashboard_state() - return cell - - def _find_cell(self, symbol: str) -> Cell | None: - return next((c for c in self.cells if c.symbol == symbol), None) - - def remove_stock(self, symbol: str) -> bool: - with self._lock: - cell = self._find_cell(symbol) - if not cell: - return False - self.cells.remove(cell) - self.save_dashboard_state() - return True - - def delete_all_stocks(self, mark_modified: bool = True) -> None: - with self._lock: - self.cells.clear() - if mark_modified: - self.watchlist_modified = True - self.save_dashboard_state() - - def update_cell_inputs(self, symbol: str, quantity: float, - price: float) -> dict: - """Quantity/Price entry change (v4.5 _on_entry_change).""" - with self._lock: - cell = self._find_cell(symbol) - if not cell: - return {"ok": False, - "message": f"Stock {symbol} not on the board."} - try: - cell.quantity = float(quantity) - cell.price = float(price) - except (TypeError, ValueError): - cell.quantity = 0.0 - cell.price = 0.0 - return {"ok": False, - "message": "Please enter valid numbers for " - "Quantity and Price."} - self.watchlist_modified = True - return {"ok": True, "portfolio": self.portfolio_summary()} - - def toggle_selection(self, symbol: str, selected: bool) -> None: - with self._lock: - cell = self._find_cell(symbol) - if cell: - cell.selected = bool(selected) - - @property - def selected_symbols_for_watchlist(self) -> list[str]: - return [c.symbol for c in self.cells if c.selected] - - # ================================================================== - # Live data - # ================================================================== - - def is_trading_hour(self) -> bool: - """True within NSE market hours (IST). AUDIT FIX H-8: v4.5 used the - server's local clock; this uses Asia/Kolkata explicitly.""" - now = datetime.now(ZoneInfo(MARKET_TZ)) - if not (0 <= now.weekday() <= 4): - return False - market_open = now.replace(hour=MARKET_OPEN[0], minute=MARKET_OPEN[1], - second=0, microsecond=0) - market_close = now.replace(hour=MARKET_CLOSE[0], minute=MARKET_CLOSE[1], - second=0, microsecond=0) - return market_open <= now <= market_close - - def apply_live_quotes(self, quotes: list[dict], source: str) -> int: - """Apply fetched quotes to matching cells; returns update count.""" - updated = 0 - with self._lock: - self.live_data_source = source - by_symbol = {q["SYMBOL"]: q for q in quotes} - for cell in self.cells: - q = by_symbol.get(cell.symbol) - if not q: - continue - curr = float(q['CLOSE_PRICE']) - prev = float(q['PREV_CL_PR']) - curr_vol = float(q.get('CURR_VOL', 0) or 0) - prev_vol = float(q.get('PREV_VOL', 0) or 0) - cell.current_cmp = curr - cell.live = { - "price": curr, - "pct_change": (curr / prev - 1) * 100 if prev else 0, - "day_high": float(q.get('DAY_HIGH', 0) or 0), - "day_low": float(q.get('DAY_LOW', 0) or 0), - "vol_change": ( - (curr_vol / prev_vol - 1) * 100 if prev_vol else 0.0 - ), - "updated_at": datetime.now(ZoneInfo(MARKET_TZ)).strftime('%d %b %Y | %H:%M:%S IST'), - } - updated += 1 - self.last_refresh = datetime.now(ZoneInfo(MARKET_TZ)) - return updated - - def get_index_quotes(self) -> dict[str, dict]: - """Fetch live index quotes for NIFTY and SENSEX with 15s cache.""" - import time - now = time.time() - if hasattr(self, '_index_cache') and self._index_cache and (now - getattr(self, '_index_cache_time', 0) < 15): - return self._index_cache - try: - fetcher = LiveFetcher(session=self.session) - quotes = fetcher.fetch_index_quotes() - if quotes: - self._index_cache = quotes - self._index_cache_time = now - return quotes - except Exception as e: - logging.warning(f"Error fetching index quotes: {e}") - return getattr(self, '_index_cache', {}) - - def load_live_data(self) -> dict: - """One live refresh cycle for all displayed stocks, with post-market caching.""" - symbols = [c.symbol for c in self.cells] - if not symbols: - return {"ok": True, "updated": 0, "source": self.live_data_source} - - now = datetime.now(ZoneInfo(MARKET_TZ)) - cache_file = self.paths.data_dir / 'nse_live_quotes_cache.json' - is_live = self.is_trading_hour() - - # If market is closed, check cache first - if not is_live: - # Helper to calculate last session date - def get_last_session_date(dt: datetime) -> datetime.date: - curr = dt - if curr.weekday() < 5: - if curr.hour < 15 or (curr.hour == 15 and curr.minute < 30): - curr -= timedelta(days=1) - while curr.weekday() >= 5: - curr -= timedelta(days=1) - return curr.date() - - last_session_now = get_last_session_date(now) - - if cache_file.exists(): - try: - import json - with open(cache_file, 'r') as f: - cache_data = json.load(f) - cache_time = datetime.fromisoformat(cache_data["timestamp"]).astimezone(ZoneInfo(MARKET_TZ)) - last_session_cache = get_last_session_date(cache_time) - - # Cache is valid if it's from the same last trading session and was pulled at/after 15:30 - is_same_session = last_session_now == last_session_cache - is_post_close = cache_time.hour > 15 or (cache_time.hour == 15 and cache_time.minute >= 30) - - if is_same_session and is_post_close: - logging.info("Loading live data from post-market cache.") - quotes = cache_data["quotes"] - # Filter quotes to only keep those currently on the board - board_symbols = set(symbols) - filtered_quotes = [q for q in quotes if q["SYMBOL"] in board_symbols] - updated = self.apply_live_quotes(filtered_quotes, cache_data["source"]) - return { - "ok": True, - "updated": updated, - "source": cache_data["source"], - "cached": True, - "portfolio": self.portfolio_summary(), - "message": "Loaded live data from post-market cache." - } - except Exception as e: - logging.warning(f"Failed to read live quote cache: {e}") - - logging.info("No valid post-market cache found. Pulling live data once after market hours.") - - # If market is open, or cache is missing/invalid when closed: - sess_id = self.paths.state_file.stem.split('_')[-1] if '_' in self.paths.state_file.stem else 'admin' - logging.info("[LIVE] Starting live data pull cycle — %d symbol(s) (Source: %s, Session: %s)", - len(symbols), self.live_data_source_preference or 'Google', sess_id) - quotes, source = self.stock_data.fetch_real_time_data( - symbols, preferred_source=self.live_data_source_preference or "Google" - ) - if not quotes: - logging.error("Failed to fetch live data from any source.") - return {"ok": False, "updated": 0, "source": "None", - "message": "Failed to fetch live data from any source."} - - # Save to cache - try: - import json - cache_payload = { - "timestamp": now.isoformat(), - "source": source, - "quotes": quotes - } - with open(cache_file, 'w') as f: - json.dump(cache_payload, f, indent=4) - logging.info("Successfully updated live quote cache.") - except Exception as e: - logging.warning(f"Failed to write live quote cache: {e}") - - updated = self.apply_live_quotes(quotes, source) - return {"ok": True, "updated": updated, "source": source, - "portfolio": self.portfolio_summary()} - - def start_live(self, source: str, refresh_rate: int | None) -> dict: - """Enable live mode (v4.5 toggle_live_data + ask_live_data_source).""" - with self._lock: - if source not in ("Google", "Yahoo"): - self.real_time_active = False - return {"ok": False, - "message": "Live data not started as source " - "selection was cancelled."} - try: - rate = int(refresh_rate) if refresh_rate else DEFAULT_REFRESH_RATE_MIN - except (TypeError, ValueError): - rate = DEFAULT_REFRESH_RATE_MIN - self.live_refresh_rate = rate if rate > 0 else DEFAULT_REFRESH_RATE_MIN - self.live_data_source_preference = source - self.real_time_active = True - self.last_refresh = datetime.now() - sess_id = self.paths.state_file.stem.split('_')[-1] if '_' in self.paths.state_file.stem else 'admin' - print(f"[LIVE] Activating live data pull: source={source}, rate={self.live_refresh_rate}m (Session: {sess_id})", flush=True) - return {"ok": True, "refresh_rate": self.live_refresh_rate} - - def stop_live(self) -> dict: - """Disable live mode and restore historic data (v4.5 behaviour).""" - with self._lock: - self.real_time_active = False - self.last_refresh = None - for cell in self.cells: - cell.live = None - logging.info("Live data updates stopped. Displaying historic data.") - self.load_historic_data() - return {"ok": True} - - def load_historic_data(self) -> dict: - """Refresh all displayed cells from historic data for selected_date.""" - try: - data = self.stock_data.get_data_for_date(self.selected_date) - except Exception as e: - logging.error(f"Failed to refresh historic data: {e}") - return {"ok": False, "message": str(e)} - with self._lock: - historic = {str(r['SYMBOL']): r for _, r in data.iterrows()} - removed = [] - for cell in list(self.cells): - row = historic.get(cell.symbol) - if row is not None: - cell.current_cmp = float(row['CLOSE_PRICE']) - cell.prev_close = float(row['PREV_CL_PR']) - else: - logging.warning( - f"Historic data not found for {cell.symbol}. Removing cell." - ) - self.cells.remove(cell) - removed.append(cell.symbol) - if removed: - self.save_dashboard_state() - return {"ok": True, "removed": removed} - - # ================================================================== - # Portfolio summary - # ================================================================== - - def portfolio_summary(self) -> dict: - cells = [] - for c in self.cells: - key = self.key_for_symbol(c.symbol) - prev_close = 0.0 - if key and 'PREV_CL_PR' in self.stocks[key]: - prev_close = float(self.stocks[key]['PREV_CL_PR']) - elif c.current_cmp != 0.0: - prev_close = c.current_cmp - cells.append({ - "Quantity": c.quantity, "Price": c.price, - "current_cmp": c.current_cmp, "prev_close": prev_close, - }) - return compute_portfolio_summary(cells) - - # ================================================================== - # Watchlists - # ================================================================== - - def save_dashboard_state(self) -> None: - self.watchlists.save_dashboard_state([c.get_data() for c in self.cells]) - - def _load_cells_from_entries(self, entries: list[dict], - live_data_map: dict | None = None) -> int: - """Rebuild the board from watchlist entries; returns count loaded.""" - self.delete_all_stocks(mark_modified=False) - loaded = 0 - for stock_info in entries: - symbol = stock_info.get('SYMBOL') - key = self.key_for_symbol(symbol) - if key: - cell = self.add_stock( - key, - quantity=stock_info.get('Quantity', 0.0), - price=stock_info.get('Price', 0.0), - save_state=False, - ) - if cell and live_data_map and symbol in live_data_map: - self.apply_live_quotes( - [live_data_map[symbol]], self.live_data_source - ) - loaded += 1 if cell else 0 - return loaded - - def load_dashboard_state_by_name(self, watchlist_name: str) -> dict: - """Restore the dashboard from a named watchlist (v4.5 parity).""" - try: - file_path = None - if watchlist_name == "Default List": - file_path = self.paths.state_file - else: - for i, name in enumerate(self.watchlist_names): - if name == watchlist_name: - file_path = self.paths.watchlist_slot(i + 1) - break - if not file_path or not file_path.exists(): - logging.warning( - f"Watchlist file not found for '{watchlist_name}'." - ) - self.delete_all_stocks(mark_modified=False) - return {"ok": False, "message": "Watchlist file not found."} - _name, entries = self.watchlists.load_symbols_from_json(file_path) - self._load_cells_from_entries(entries) - with self._lock: - self.watchlist_modified = False - self.current_watchlist_name = watchlist_name - return {"ok": True, "loaded": len(self.cells)} - except Exception as e: - logging.error( - f"Failed to load dashboard state for '{watchlist_name}': {e}" - ) - self.delete_all_stocks(mark_modified=False) - self.current_watchlist_name = "Default List" - return {"ok": False, "message": f"Failed to load dashboard: {e}"} - - def load_watchlist(self, watchlist_num: int) -> dict: - """Load a numbered watchlist slot (v4.5 load_watchlist).""" - try: - file_path = self.paths.watchlist_slot(watchlist_num) - name, entries = self.watchlists.load_symbols_from_json(file_path) - if not entries: - self.delete_all_stocks(mark_modified=False) - self.current_watchlist_name = "Default List" - return {"ok": False, - "message": f"Watchlist {watchlist_num} is empty."} - if not name: - name = f"Watchlist {watchlist_num}" - self._load_cells_from_entries(entries) - with self._lock: - self.current_watchlist_name = name - self.watchlist_modified = False - self.save_dashboard_state() - return {"ok": True, "name": name, "loaded": len(self.cells)} - except Exception as e: - self.current_watchlist_name = "Default List" - return {"ok": False, "message": f"Failed to load watchlist: {e}"} - - def load_watchlist_from_file(self, file_path: Path) -> dict: - """Load a watchlist from an arbitrary JSON file (v4.5 parity, - including the live-quote prefetch when live mode is active).""" - try: - target_path = Path(file_path) - if not target_path.exists(): - global_path = self.paths.data_dir / 'watchlists' / target_path.name - if global_path.exists(): - target_path = global_path - - name, entries = self.watchlists.load_symbols_from_json(target_path) - if not entries: - return {"ok": False, - "message": f"The watchlist '{target_path.stem}' is empty or file not found."} - if not name: - name = target_path.stem - live_map: dict = {} - if self.real_time_active: - symbols = [e['SYMBOL'] for e in entries] - quotes, source = self.stock_data.fetch_real_time_data( - symbols, - preferred_source=self.live_data_source_preference or "Google", - ) - live_map = {q['SYMBOL']: q for q in quotes} - self.live_data_source = source - self._load_cells_from_entries(entries, live_map) - with self._lock: - self.current_watchlist_name = name - self.save_dashboard_state() - return {"ok": True, "name": name, "loaded": len(self.cells), - "message": f"Successfully loaded {len(self.cells)} " - f"stocks from '{name}'."} - except Exception as e: - logging.error(f"Error loading custom watchlist: {e}") - return {"ok": False, - "message": f"Failed to load custom watchlist: {e}"} - - def save_dashboard(self) -> dict: - """Save the board to the currently loaded named slot.""" - if not self.cells: - return {"ok": False, "message": "No stocks to save."} - try: - idx = self.watchlist_names.index(self.current_watchlist_name) - except ValueError: - return {"ok": False, "needs_save_as": True, - "message": "Current view is not a saved watchlist. " - "Use 'Save as Watchlist'."} - try: - self.watchlists.save_slot( - idx + 1, self.current_watchlist_name, - [c.get_data() for c in self.cells], - ) - self.watchlist_modified = False - self.watchlists.save_config(self.current_watchlist_name) - return {"ok": True, - "message": f"Dashboard saved to: " - f"{self.current_watchlist_name}"} - except Exception as e: - return {"ok": False, "message": f"Failed to save dashboard: {e}"} - - def save_as_watchlist(self, watchlist_num: int, name: str) -> dict: - """Save the board to a specific slot under a new name.""" - if not self.cells: - return {"ok": False, "message": "No stocks to save"} - if not (1 <= int(watchlist_num) <= WATCHLIST_SLOTS): - return {"ok": False, - "message": f"Slot must be 1..{WATCHLIST_SLOTS}."} - name = (name or "").strip() or f"Watchlist {watchlist_num}" - try: - self.watchlists.save_slot( - int(watchlist_num), name, [c.get_data() for c in self.cells] - ) - self.current_watchlist_name = name - self.watchlist_names = self.watchlists.load_watchlist_names() - self.watchlist_modified = False - self.watchlists.save_config(self.current_watchlist_name) - return {"ok": True, "message": f"Saved as {name}"} - except Exception as e: - return {"ok": False, "message": f"Failed to save watchlist: {e}"} - - def add_selected_to_watchlist(self, watchlist_num: int) -> dict: - """Merge checkbox-selected symbols into a slot (v4.5 parity, - including clearing the selections afterwards).""" - selected = self.selected_symbols_for_watchlist - if not selected: - return {"ok": False, - "message": "Please select at least one stock using " - "the checkboxes."} - try: - name, added = self.watchlists.add_symbols_to_slot( - int(watchlist_num), selected - ) - except Exception as e: - return {"ok": False, - "message": f"Failed to add stocks to watchlist: {e}"} - with self._lock: - for c in self.cells: - c.selected = False - if added > 0: - self.watchlist_modified = True - self.watchlist_names = self.watchlists.load_watchlist_names() - return {"ok": True, - "message": f"Added {added} stock(s) to '{name}'."} - return {"ok": True, - "message": "All selected stocks are already in the " - "chosen watchlist."} - - # ================================================================== - # Index / analysis board loads - # ================================================================== - - def load_index_stocks(self, index_name: str) -> dict: - stock_list = {"SENSEX": SENSEX_STOCKS, "NIFTY": NIFTY_STOCKS}.get(index_name) - if stock_list is None: - return {"ok": False, "message": f"Unknown index: {index_name}"} - self.delete_all_stocks(mark_modified=False) - for symbol in stock_list: - key = self.key_for_symbol(symbol) - if key: - self.add_stock(key, save_state=False) - self.current_watchlist_name = f"{index_name} Index" - self.save_dashboard_state() - return {"ok": True, "loaded": len(self.cells)} - - def display_analysis_result(self, symbols: list[str], - json_name: str, cached: bool) -> dict: - """Common tail of every analysis flow: rebuild board from symbols.""" - if not symbols: - return {"ok": False, "message": "No stock data available for filter."} - self.delete_all_stocks(mark_modified=False) - for symbol in symbols: - key = self.key_for_symbol(symbol) - if key: - self.add_stock(key, save_state=False) - self.save_dashboard_state() - self.current_watchlist_name = json_name.replace(" Portfolio", "") - message = ( - f"Loaded {len(self.cells)} stocks from saved file." - if cached else - f"Displayed {len(self.cells)} stocks after calculation." - ) - return {"ok": True, "message": message, "loaded": len(self.cells)} - - def load_portfolio_holdings(self, holdings: list[dict], - unmapped: list[str], broker_name: str = "ICICIDirect") -> dict: - """Board rebuild after portfolio import, automatically saving to Watchlist 2.""" - if not holdings: - return {"ok": False, "message": "No stocks mapped."} - with self._lock: - self.delete_all_stocks(mark_modified=False) - loaded = 0 - for h in holdings: - key = self.key_for_symbol(h['SYMBOL']) - if key: - self.add_stock(key, quantity=h['Quantity'], - price=h['Price'], save_state=False) - loaded += 1 - self.save_dashboard_state() - - # Auto-save to watchlist slot 2 and rename it - watchlist_name = f"{broker_name} Portfolio" - self.watchlists.save_slot( - 2, watchlist_name, [c.get_data() for c in self.cells] - ) - self.current_watchlist_name = watchlist_name - self.watchlist_names = self.watchlists.load_watchlist_names() - self.watchlist_modified = False - self.watchlists.persist_config() if hasattr(self.watchlists, 'persist_config') else self.watchlists.save_config(self.current_watchlist_name) - - msg = f"Successfully loaded {loaded} stocks from {broker_name}." - if unmapped: - msg += f" Could not map: {', '.join(str(u) for u in unmapped)}" - return {"ok": True, "loaded": loaded, "unmapped": unmapped, - "message": msg} - - # ================================================================== - # Date change / refresh / export / config - # ================================================================== - - def change_date(self, new_date_str: str) -> dict: - """v4.5 on_date_change: reload universe and update board cells.""" - try: - new_date = datetime.strptime(new_date_str, '%Y-%m-%d').date() - except ValueError: - return {"ok": False, "message": "Invalid date format (YYYY-MM-DD)."} - if new_date > datetime.now().date(): - return {"ok": False, "message": "Cannot select future dates."} - old_date = self.selected_date - self.selected_date = new_date - try: - self.load_universe() - except ValueError: - self.selected_date = old_date - return {"ok": False, - "message": f"No stock data available for {new_date}."} - except Exception as e: - self.selected_date = old_date - return {"ok": False, - "message": f"Failed to load data for {new_date}: {e}"} - with self._lock: - removed = [] - for cell in list(self.cells): - key = self.key_for_symbol(cell.symbol) - if key: - row = self.stocks[key] - cell.current_cmp = float(row['CLOSE_PRICE']) - cell.prev_close = float(row['PREV_CL_PR']) - else: - self.cells.remove(cell) - removed.append(cell.symbol) - if removed: - self.save_dashboard_state() - return {"ok": True, - "message": f"Dashboard updated for {self.selected_date}.", - "removed": removed} - - def refresh_dashboard(self) -> dict: - """v4.5 refresh_dashboard: reload CSV + rebuild universe + re-add.""" - stopped_live = False - if self.real_time_active: - self.stop_live() - stopped_live = True - try: - self.stock_data.load_data() - except Exception as e: - return {"ok": False, - "message": f"Failed to reload data: {e}\n" - "Dashboard might not be fully updated."} - current = {c.symbol: c.get_data() for c in self.cells} - try: - self.load_universe() - self.delete_all_stocks(mark_modified=False) - reloaded = 0 - for symbol, old in current.items(): - key = self.key_for_symbol(symbol) - if key: - self.add_stock(key, quantity=old['Quantity'], - price=old['Price'], save_state=False) - reloaded += 1 - self.save_dashboard_state() - msg = f"Data reloaded. {reloaded} stocks re-displayed." - if stopped_live: - msg = "Live data updates have been paused for dashboard " \ - "refresh. " + msg - return {"ok": True, "message": msg, "reloaded": reloaded} - except Exception as e: - return {"ok": False, - "message": f"An error occurred while updating the " - f"dashboard: {e}"} - - def export_to_csv(self) -> dict: - """Export cell data to a timestamped CSV in downloads/.""" - try: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = self.paths.log_dir / f"stock_data_{timestamp}.csv" - pd.DataFrame([c.get_data() for c in self.cells]).to_csv( - filename, index=False - ) - return {"ok": True, "file": str(filename), - "message": f"Data exported to: {filename}"} - except Exception as e: - logging.error(f"Failed to export data to CSV: {e}") - return {"ok": False, - "message": "Failed to export data. Check logs."} - - def persist_config(self) -> None: - self.watchlists.save_config(self.current_watchlist_name) diff --git a/nsedash/core/hf_sync.py b/nsedash/core/hf_sync.py deleted file mode 100644 index b96e645b9ae36db954de6ccfd77508c30d39075c..0000000000000000000000000000000000000000 --- a/nsedash/core/hf_sync.py +++ /dev/null @@ -1,338 +0,0 @@ -""" -nsedash.core.hf_sync -==================== -Two-way synchronisation between the local ``processed/`` directory and -the Hugging Face dataset ``Sam-max1/nsedash_data``. - -Active ONLY when ``SPACE_ID`` is set in the environment (i.e. inside a -Hugging Face Space). In local/dev mode this module is a complete no-op -so that local runs are unaffected. - -Key design decisions ---------------------- -* **Startup pull** — ``pull_on_startup()`` is called *before* Flask starts - and *before* ``DashboardService`` is created so that processed data is - fresh on every Space restart. It is synchronous (blocks startup) to - guarantee correctness. -* **Async push** — every local file write triggers ``push_file_async()`` - which queues the upload on a background daemon thread. The main request - path is never blocked. -* **3-second debounce** — a timer is reset on each push call for the same - path so that rapid sequential writes (e.g. saving all watchlist slots at - once) are coalesced into a single upload. -* **Session cleanup** — ``delete_session_files(token)`` is called when a - session expires so per-user files are removed from the dataset, keeping - the repo clean. -* **Read-only parquet guard** — the large ``nse_processed_data.parquet`` - is pulled on startup but never pushed at runtime (it is only updated via - ``push_to_hf.py``). -""" - -from __future__ import annotations - -import logging -import os -import threading -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - pass - -log = logging.getLogger(__name__) - -# Files that are pulled on startup but NEVER pushed at runtime (too large / -# managed by push_to_hf.py instead). -_READ_ONLY_FILES = frozenset({ - "nse_processed_data.parquet", - "nse_summary_data.parquet", -}) - -# Remote prefix inside the HF dataset repo for all processed files. -_REMOTE_PREFIX = "processed" - - -class HFDataSync: - """ - Two-way sync between ``processed/`` and ``Sam-max1/nsedash_data``. - - Parameters - ---------- - data_dir: - Local path to ``/processed/``. - repo_id: - HuggingFace dataset repo ID (default: ``Sam-max1/nsedash_data``). - token: - HF API token. If *None*, tries ``HF_TOKEN`` env var. - enabled: - Forced override — set to ``False`` to disable even in HF mode. - """ - - def __init__( - self, - data_dir: Path, - repo_id: str = "Sam-max1/nsedash_data", - token: str | None = None, - enabled: bool | None = None, - ) -> None: - self.data_dir = Path(data_dir) - self.repo_id = repo_id - - # Resolve token - self._token = token or os.environ.get("HF_TOKEN") or os.environ.get("HF_PRIVATE_TOKEN") - - # Enable only when running inside a HF Space or HF mode is set (unless forced) - if enabled is None: - self.enabled = "SPACE_ID" in os.environ or os.environ.get("HF_MODE") == "1" - else: - self.enabled = enabled - - self._api = None # lazy HfApi instance - self._lock = threading.Lock() - - # Debounce: path → (Timer, repo_path) - self._pending: dict[str, threading.Timer] = {} - self._debounce_sec = 3.0 - - if self.enabled: - if not self._token: - log.warning( - "[HFSync] HF_TOKEN not set — HF dataset sync is disabled." - ) - self.enabled = False - else: - log.info( - "[HFSync] Enabled — dataset: %s, data_dir: %s", - self.repo_id, self.data_dir, - ) - else: - log.info("[HFSync] Disabled (not in HF Space mode).") - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _get_api(self): - """Lazy-load HfApi (avoids import cost when sync is disabled).""" - if self._api is None: - from huggingface_hub import HfApi - self._api = HfApi(token=self._token) - return self._api - - def _ensure_repo(self) -> None: - """Create the dataset repo if it does not yet exist.""" - try: - api = self._get_api() - api.create_repo( - repo_id=self.repo_id, - repo_type="dataset", - private=False, - exist_ok=True, - ) - log.info("[HFSync] Dataset repo '%s' is ready.", self.repo_id) - except Exception as exc: - log.error("[HFSync] Could not ensure dataset repo exists: %s", exc) - - def _remote_path(self, local_path: Path) -> str: - """Compute the in-repo path for a local file under data_dir.""" - try: - rel = local_path.relative_to(self.data_dir) - except ValueError: - rel = Path(local_path.name) - return f"{_REMOTE_PREFIX}/{rel.as_posix()}" - - # ------------------------------------------------------------------ - # Startup pull - # ------------------------------------------------------------------ - - def pull_on_startup(self) -> None: - """ - Download all files from the HF dataset into ``processed/``. - - Runs synchronously during startup. Missing remote files are - silently skipped (first-run scenario). The local parquet is - never overwritten if it is larger than the remote copy (safety - guard for the large data file). - """ - if not self.enabled: - return - - log.info("[HFSync] Starting startup pull from %s …", self.repo_id) - try: - self._ensure_repo() - api = self._get_api() - remote_files = list(api.list_repo_files( - repo_id=self.repo_id, repo_type="dataset" - )) - except Exception as exc: - log.error("[HFSync] Could not list remote files: %s", exc) - return - - pulled, skipped, failed = 0, 0, 0 - for remote_path in remote_files: - # Skip HF metadata files - if remote_path.startswith("."): - continue - if not remote_path.startswith(f"{_REMOTE_PREFIX}/"): - skipped += 1 - continue - - rel = remote_path[len(_REMOTE_PREFIX) + 1:] # strip "processed/" - local_path = self.data_dir / rel - - # Ensure local file is removed before pulling to guarantee overwriting - if local_path.exists() and local_path.is_file(): - try: - local_path.unlink() - except Exception as e: - log.warning("[HFSync] Could not remove local file %s before pull: %s", local_path, e) - - local_path.parent.mkdir(parents=True, exist_ok=True) - - try: - from huggingface_hub import hf_hub_download - tmp = hf_hub_download( - repo_id=self.repo_id, - repo_type="dataset", - filename=remote_path, - token=self._token, - local_dir=str(self.data_dir.parent), # downloads into parent/processed/rel - local_dir_use_symlinks=False, - ) - # hf_hub_download places file at local_dir/ - # which resolves to data_dir.parent/processed/rel = data_dir/rel ✓ - pulled += 1 - log.info("[HFSync] Pulled: %s", rel) - except Exception as exc: - log.warning("[HFSync] Could not pull '%s': %s", remote_path, exc) - failed += 1 - - log.info( - "[HFSync] Startup pull complete — pulled=%d, skipped=%d, failed=%d", - pulled, skipped, failed, - ) - - # ------------------------------------------------------------------ - # Async push (write path) - # ------------------------------------------------------------------ - - def push_file_async(self, local_path: Path | str, - repo_path: str | None = None) -> None: - """ - Schedule an upload of *local_path* to the HF dataset. - - Calls within ``debounce_sec`` for the same path are coalesced — - only the last write is uploaded. This is a fire-and-forget - background thread; the caller is never blocked. - """ - if not self.enabled: - return - - local_path = Path(local_path) - if not local_path.exists(): - return - - # Never push read-only / large files at runtime - if local_path.name in _READ_ONLY_FILES: - return - - repo_path = repo_path or self._remote_path(local_path) - key = str(local_path) - - with self._lock: - # Cancel any pending timer for this path - existing = self._pending.pop(key, None) - if existing: - existing.cancel() - - def _upload(): - with self._lock: - self._pending.pop(key, None) - try: - api = self._get_api() - api.upload_file( - path_or_fileobj=str(local_path), - path_in_repo=repo_path, - repo_id=self.repo_id, - repo_type="dataset", - commit_message=f"auto: update {local_path.name}", - ) - log.info("[HFSync] Pushed: %s → %s", local_path.name, repo_path) - except Exception as exc: - log.warning( - "[HFSync] Push failed for '%s': %s", local_path.name, exc - ) - - timer = threading.Timer(self._debounce_sec, _upload) - timer.daemon = True - timer.start() - self._pending[key] = timer - - # ------------------------------------------------------------------ - # Session cleanup (delete per-user files from HF dataset) - # ------------------------------------------------------------------ - - def delete_session_files(self, token: str) -> None: - """ - Remove per-user files for *token* from the HF dataset. - - Called when a session expires or the user disconnects, to keep - the dataset clean. Runs in a background thread. - """ - if not self.enabled or not token or token in ("admin", "default"): - return - - def _delete(): - try: - api = self._get_api() - remote_files = list(api.list_repo_files( - repo_id=self.repo_id, repo_type="dataset" - )) - suffix = f"_{token}" - targets = [ - f for f in remote_files - if token in f and not f.startswith(".") - ] - for path in targets: - try: - api.delete_file( - path_in_repo=path, - repo_id=self.repo_id, - repo_type="dataset", - commit_message=f"cleanup: session {token[:8]} expired", - ) - log.info("[HFSync] Deleted session file: %s", path) - except Exception as exc: - log.warning( - "[HFSync] Could not delete '%s': %s", path, exc - ) - except Exception as exc: - log.error( - "[HFSync] Session cleanup failed for token %s: %s", - token[:8], exc, - ) - - threading.Thread(target=_delete, daemon=True, name=f"hfsync-cleanup-{token[:8]}").start() - - -# --------------------------------------------------------------------------- -# Module-level no-op singleton (used when sync is disabled) -# --------------------------------------------------------------------------- - -class _NoOpSync: - """Drop-in replacement when HF sync is disabled — all methods are no-ops.""" - enabled = False - - def pull_on_startup(self) -> None: pass - def push_file_async(self, *_a, **_kw) -> None: pass - def delete_session_files(self, *_a, **_kw) -> None: pass - - -def make_sync(data_dir: Path, repo_id: str, token: str | None = None) -> HFDataSync | _NoOpSync: - """ - Factory — returns a live ``HFDataSync`` when in HF mode, else a no-op. - """ - is_hf = "SPACE_ID" in os.environ or os.environ.get("HF_MODE") == "1" - if not is_hf: - return _NoOpSync() - return HFDataSync(data_dir=data_dir, repo_id=repo_id, token=token) diff --git a/nsedash/core/live_fetch.py b/nsedash/core/live_fetch.py deleted file mode 100644 index cc1ec5527914376eb3e3b27fe66a409b9ef13743..0000000000000000000000000000000000000000 --- a/nsedash/core/live_fetch.py +++ /dev/null @@ -1,361 +0,0 @@ -""" -nsedash.core.live_fetch -======================= -Real-time price/volume scraping from Google Finance and Yahoo Finance, -with per-symbol source failover — functional parity with v4.5's -``fetch_real_time_data_gf`` / ``fetch_real_time_data_yf`` / -``fetch_real_time_data``. - -Audit fixes applied here: - C-1 Every HTTP call now has an explicit timeout (original had none: a - single hung socket froze the refresh cycle indefinitely). - P-1 Symbols are fetched concurrently via ThreadPoolExecutor instead of - strictly sequentially (original: ~2 blocking requests per symbol, - serially — 50 symbols ≈ 100 serial round trips). - P-2 A shared ``requests.Session`` reuses TCP/TLS connections. - A-1 ``fetch_real_time_volume`` returned ``([], [])`` on error but - ``(float, float)`` on success; the type anomaly is fixed — - always returns ``(float, float)``. - P-3 The Google path no longer performs a second full Yahoo page fetch - just for volume when the Yahoo page was already fetched during - failover; volume scraping is still done for Google-sourced quotes - (behaviour retained) but through the shared session. -""" - -from __future__ import annotations - -import logging -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime -from typing import Callable, Iterable - -import json -import requests -from bs4 import BeautifulSoup - -from ..config import FETCH_MAX_WORKERS, HTTP_TIMEOUT_SEC, REQUEST_HEADERS - -PrevVolumeLookup = Callable[[str], float] - - -import threading - -_global_fetch_lock = threading.Lock() - - -class LiveFetcher: - """ - Scrapes live quotes. ``prev_volume_lookup`` is injected by the data - layer so this module stays free of DataFrame coupling. - """ - - def __init__(self, prev_volume_lookup: PrevVolumeLookup | None = None, - session: requests.Session | None = None): - self.prev_volume_lookup = prev_volume_lookup or (lambda _s: 0.0) - self.session = session or requests.Session() - self.session.headers.update(REQUEST_HEADERS) - - # ------------------------------------------------------------------ - # Volume (Yahoo) - # ------------------------------------------------------------------ - - def fetch_real_time_volume(self, symbol: str) -> tuple[float, float]: - """Return (current_volume, previous_volume) for a symbol.""" - current_volume, previous_volume = 0.0, 0.0 - yahoo_symbol = f'{symbol}.NS' - try: - response = self.session.get( - f'https://finance.yahoo.com/quote/{yahoo_symbol}', - timeout=HTTP_TIMEOUT_SEC, - ) - response.raise_for_status() - soup = BeautifulSoup(response.text, 'html.parser') - best_res = None - for s in soup.find_all('script'): - if s.string and 'quoteResponse' in s.string: - try: - data = json.loads(s.string) - body = json.loads(data['body']) - for r_item in body['quoteResponse']['result']: - if r_item.get('symbol') == yahoo_symbol: - if best_res is None or (r_item.get('regularMarketDayHigh') is not None and best_res.get('regularMarketDayHigh') is None): - best_res = r_item - except Exception: - pass - - if best_res: - vol_val = best_res.get('regularMarketVolume') - if isinstance(vol_val, dict): - current_volume = float(vol_val.get('raw', 0.0)) - elif vol_val is not None: - current_volume = float(vol_val) - else: - # Fallback to the old HTML parser - vol_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketVolume'}) - if vol_tag and vol_tag.text: - try: - current_volume = float(vol_tag.text.replace(',', '')) - except ValueError: - pass - except requests.exceptions.RequestException as req_err: - logging.error(f"HTTP Request error for {symbol} from Yahoo Finance: {req_err}") - return 0.0, 0.0 - except Exception as e: - logging.error(f"Error fetching volume for {symbol} from Yahoo Finance: {e}") - return 0.0, 0.0 - try: - previous_volume = float(self.prev_volume_lookup(symbol)) - except Exception as e: - logging.error(f"Error fetching previous volume for {symbol} from historic data: {e}") - return current_volume, previous_volume - - # ------------------------------------------------------------------ - # Google Finance - # ------------------------------------------------------------------ - - def fetch_symbol_gf(self, symbol: str) -> dict | None: - """Scrape one symbol from Google Finance. None on failure.""" - try: - response = self.session.get( - f'https://www.google.com/finance/quote/{symbol}:NSE', - headers={'User-Agent': 'python-requests/2.32.5'}, - timeout=HTTP_TIMEOUT_SEC, - ) - response.raise_for_status() - soup = BeautifulSoup(response.text, 'html.parser') - - price_element = soup.find('div', {'class': 'YMlKec fxKbKc'}) - current_price = ( - float(price_element.text[1:].replace(",", "")) - if price_element else 0.0 - ) - if current_price <= 0.0: - raise ValueError(f"Invalid or zero price scraped for {symbol}") - previous_price_element = soup.find('div', {'class': 'P6K39c'}) - previous_close_price = ( - float(previous_price_element.text[1:].replace(",", "")) - if previous_price_element else 0.0 - ) - all_p6k39c = soup.find_all('div', {'class': 'P6K39c'}) - day_high = day_low = 0.0 - if len(all_p6k39c) > 1: - range_text = all_p6k39c[1].text.replace('₹', '').replace(',', '') - if ' - ' in range_text: - low_str, high_str = range_text.split(' - ') - day_low, day_high = float(low_str), float(high_str) - if previous_close_price == 0.0: - previous_close_price = current_price - current_volume, previous_volume = self.fetch_real_time_volume(symbol) - return { - "SYMBOL": symbol, - "CLOSE_PRICE": current_price, - "PREV_CL_PR": previous_close_price, - "DAY_HIGH": day_high, - "DAY_LOW": day_low, - "CURR_VOL": current_volume, - "PREV_VOL": previous_volume, - } - except requests.exceptions.RequestException as req_err: - logging.error(f"HTTP Request error for {symbol} from Google Finance: {req_err}") - except AttributeError: - logging.error( - f"Parsing error for {symbol} from Google Finance. " - "HTML structure might have changed." - ) - except ValueError: - logging.error( - f"Data conversion error for {symbol} from Google Finance. " - "Could not convert price to float." - ) - except Exception as e: - logging.error(f"Error fetching data for {symbol} from Google Finance: {e}") - return None - - # ------------------------------------------------------------------ - # Yahoo Finance - # ------------------------------------------------------------------ - - def fetch_symbol_yf(self, symbol: str) -> dict | None: - """Scrape one symbol from Yahoo Finance. None on failure.""" - yahoo_symbol = f'{symbol}.NS' - try: - response = self.session.get( - f'https://finance.yahoo.com/quote/{yahoo_symbol}', - timeout=HTTP_TIMEOUT_SEC, - ) - response.raise_for_status() - soup = BeautifulSoup(response.text, 'html.parser') - - best_res = None - for s in soup.find_all('script'): - if s.string and 'quoteResponse' in s.string: - try: - data = json.loads(s.string) - body = json.loads(data['body']) - for r_item in body['quoteResponse']['result']: - if r_item.get('symbol') == yahoo_symbol: - if best_res is None or (r_item.get('regularMarketDayHigh') is not None and best_res.get('regularMarketDayHigh') is None): - best_res = r_item - except Exception: - pass - - if not best_res: - # Fallback to the old HTML tag parser (used by tests and old site structure) - try: - price_tag = soup.find(attrs={"data-testid": "qsp-price"}) - if not price_tag: - price_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketPrice'}) - - if not price_tag: - raise ValueError(f"No Svelte Sourced Quote data or HTML tags found for {symbol}") - - current_price = float(price_tag.text.replace(',', '')) - - prev_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketPreviousClose'}) - previous_close_price = float(prev_tag.text.replace(',', '')) if prev_tag else current_price - - day_high, day_low = 0.0, 0.0 - range_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketDayRange'}) - if range_tag and range_tag.text: - parts = [p.strip() for p in range_tag.text.split('-')] - if len(parts) == 2: - day_low = float(parts[0].replace(',', '')) - day_high = float(parts[1].replace(',', '')) - - vol_tag = soup.find('fin-streamer', {'data-symbol': yahoo_symbol, 'data-field': 'regularMarketVolume'}) - current_volume = float(vol_tag.text.replace(',', '')) if vol_tag else 0.0 - except Exception as fallback_err: - raise ValueError(f"No Svelte Sourced Quote data found for {symbol} (HTML fallback failed: {fallback_err})") - else: - def get_val(field): - val = best_res.get(field) - if isinstance(val, dict): - return float(val.get('raw', 0.0)) - return float(val) if val is not None else 0.0 - - current_price = get_val('regularMarketPrice') - if current_price <= 0.0: - raise ValueError(f"Invalid or zero price scraped for {symbol}") - previous_close_price = get_val('regularMarketPreviousClose') - day_high = get_val('regularMarketDayHigh') - day_low = get_val('regularMarketDayLow') - current_volume = get_val('regularMarketVolume') - previous_volume = 0.0 - try: - previous_volume = float(self.prev_volume_lookup(symbol)) - except Exception as e: - logging.error(f"Error fetching previous volume for {symbol}: {e}") - if previous_close_price == 0.0: - previous_close_price = current_price - return { - "SYMBOL": symbol, - "CLOSE_PRICE": current_price, - "PREV_CL_PR": previous_close_price, - "DAY_HIGH": day_high, - "DAY_LOW": day_low, - "CURR_VOL": current_volume, - "PREV_VOL": previous_volume, - } - except requests.exceptions.RequestException as req_err: - logging.error(f"HTTP Request error for {symbol} from Yahoo Finance: {req_err}") - except Exception as e: - logging.error(f"Error fetching data for {symbol} from Yahoo Finance: {e}") - return None - - # Batch wrappers preserved for API parity with v4.5 ------------------ - - def fetch_real_time_data_gf(self, symbols: Iterable[str]) -> list[dict]: - return [d for d in (self.fetch_symbol_gf(s) for s in symbols) if d] - - def fetch_real_time_data_yf(self, symbols: Iterable[str]) -> list[dict]: - return [d for d in (self.fetch_symbol_yf(s) for s in symbols) if d] - - # ------------------------------------------------------------------ - # Failover orchestration - # ------------------------------------------------------------------ - - def _fetch_one_with_failover(self, symbol: str, - preferred_source: str) -> tuple[dict | None, str]: - """Fetch one symbol trying preferred source first, then the other.""" - order = ( - [("Google", self.fetch_symbol_gf), ("Yahoo", self.fetch_symbol_yf)] - if preferred_source == "Google" - else [("Yahoo", self.fetch_symbol_yf), ("Google", self.fetch_symbol_gf)] - ) - primary_name = order[0][0] - for i, (name, fn) in enumerate(order): - data = fn(symbol) - if data: - return data, name - if i == 0: - logging.warning( - f"{primary_name} Finance failed for {symbol}. " - f"Falling back to {order[1][0]}." - ) - logging.error(f"Both sources failed for {symbol}.") - return None, "None" - - def fetch_real_time_data(self, symbols: list[str], - preferred_source: str = "Google" - ) -> tuple[list[dict], str]: - """ - Fetch real-time data for all symbols with per-symbol failover. - Returns (list of quote dicts, source_name). Source name reflects - the last successful source, matching v4.5 semantics. - """ - preferred_source = preferred_source or "Google" - data_source = "None" - real_time_data: list[dict] = [] - failed_symbols: list[str] = [] - symbols = list(symbols) - if not symbols: - return [], data_source - - with _global_fetch_lock: - with ThreadPoolExecutor(max_workers=min(FETCH_MAX_WORKERS, len(symbols))) as pool: - results = list(pool.map( - lambda s: (s, self._fetch_one_with_failover(s, preferred_source)), - symbols, - )) - for symbol, (data, source) in results: - if data: - real_time_data.append(data) - data_source = source - else: - failed_symbols.append(symbol) - if failed_symbols: - logging.error(f"Failed symbols: {', '.join(failed_symbols)}") - return real_time_data, data_source - - def fetch_index_quotes(self) -> dict[str, dict]: - """ - Scrape live market index quotes for NIFTY 50 (^NSEI) and SENSEX (^BSESN). - Returns dict keyed by index name ('NIFTY', 'SENSEX'). - """ - indices = { - 'NIFTY': '^NSEI', - 'SENSEX': '^BSESN', - } - res = {} - for name, sym in indices.items(): - try: - url = f"https://query1.finance.yahoo.com/v8/finance/chart/{sym}?interval=1m" - r = self.session.get(url, timeout=HTTP_TIMEOUT_SEC) - r.raise_for_status() - data = r.json() - meta = data['chart']['result'][0]['meta'] - price = float(meta.get('regularMarketPrice', 0.0)) - prev = float(meta.get('chartPreviousClose') or meta.get('previousClose') or price) - chg = price - prev - pct = (chg / prev * 100.0) if prev > 0 else 0.0 - res[name] = { - "name": name, - "price": round(price, 2), - "change": round(chg, 2), - "pct_change": round(pct, 2), - "formatted_price": f"{price:,.2f}", - "formatted_change": f"{chg:+,.2f} ({pct:+.2f}%)" - } - except Exception as e: - logging.warning(f"Could not fetch live index for {name} ({sym}): {e}") - return res diff --git a/nsedash/core/nvidia_llm.py b/nsedash/core/nvidia_llm.py deleted file mode 100644 index ce4885f83dd98e73b761b7eae66a7b4632cc6d1b..0000000000000000000000000000000000000000 --- a/nsedash/core/nvidia_llm.py +++ /dev/null @@ -1,933 +0,0 @@ -""" -NVIDIA NIM LLM Wrapper with Intelligent Routing -================================================ -Maintains a curated registry of all 121 NVIDIA NIM free-tier models, -categorized and benchmarked by purpose. Provides intelligent routing -based on the request type (coding, reasoning, chat, memory, judge, -finance, vision, safety, translation, embedding). - -Model Registry is stored in: utils/config/nvidia_model_registry.json -""" - -from typing import Any, Optional -import requests -import os -import json -import threading -import datetime -import time - -try: - from dotenv import load_dotenv, find_dotenv - env_file = find_dotenv(usecwd=True) - if env_file: - load_dotenv(env_file) - else: - load_dotenv() -except ImportError: - pass - -try: - import keyring -except ImportError: - keyring = None - -try: - from crewai.llms.base_llm import BaseLLM -except ImportError: - class BaseLLM: - def __init__(self, **kwargs): - self.model = kwargs.get("model") - self.api_key = kwargs.get("api_key") - -# --------------------------------------------------------------------------- -# Telemetry Token Tracking State -# --------------------------------------------------------------------------- -TOKEN_STATE_FILE = os.path.join(os.path.dirname(__file__), "token_state.json") -MODEL_REGISTRY_FILE = os.path.join(os.path.dirname(__file__), "nvidia_model_registry.json") -token_lock = threading.Lock() -_WARNED_MISSING_KEY = False - -def load_token_usage() -> dict: - with token_lock: - if os.path.exists(TOKEN_STATE_FILE): - try: - with open(TOKEN_STATE_FILE, "r") as f: - return json.load(f) - except Exception: - pass - return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - -def save_token_usage(usage: dict): - with token_lock: - try: - with open(TOKEN_STATE_FILE, "w") as f: - json.dump(usage, f, indent=4) - except Exception: - pass - -def add_tokens(prompt: int, completion: int, model_id: str = ""): - usage = load_token_usage() - usage["prompt_tokens"] += prompt - usage["completion_tokens"] += completion - usage["total_tokens"] += (prompt + completion) - # Per-model telemetry - per_model = usage.setdefault("per_model", {}) - entry = per_model.setdefault(model_id, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}) - entry["prompt_tokens"] += prompt - entry["completion_tokens"] += completion - entry["total_tokens"] += (prompt + completion) - save_token_usage(usage) - -def get_token_usage() -> dict: - return load_token_usage() - -def reset_token_usage(): - save_token_usage({"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}) - - -# --------------------------------------------------------------------------- -# Curated Model Registry — Built from deep research on 121 NVIDIA NIM models -# --------------------------------------------------------------------------- -# Categories: coding, reasoning, chat, memory, judge, finance, vision, -# safety, translation, embedding, creative, fast -# -# Ratings: 5=exceptional, 4=excellent, 3=good, 2=decent, 1=basic -# Source: NVIDIA NIM catalog + community benchmarks (HumanEval, MMLU, etc.) -# --------------------------------------------------------------------------- - -NVIDIA_MODEL_REGISTRY = { - # === CODING === - # Best for: code generation, debugging, refactoring, code review, self-improvement agents - "coding": { - "primary": "deepseek-ai/deepseek-v4-pro", - "fallback": ["deepseek-ai/deepseek-v4-flash", "poolside/laguna-xs-2.1", "abacusai/dracarys-llama-3.1-70b-instruct", "nvidia/nv-embedcode-7b-v1"], - "models": { - "deepseek-ai/deepseek-v4-pro": { - "rating": 5, "context_window": 163840, "description": "State-of-the-art coder. Outstanding performance on code generation and multi-step agent debugging.", "strengths": ["code_gen", "agentic", "debugging"] - }, - "deepseek-ai/deepseek-v4-flash": { - "rating": 4, "context_window": 163840, "description": "High-throughput coding model. Ideal for low-latency coding loops.", "strengths": ["code_gen", "fast", "agentic"] - }, - "poolside/laguna-xs-2.1": { - "rating": 5, "context_window": 32768, "description": "Poolside Laguna XS. Specially trained for agentic coding and long-horizon developer workflows.", "strengths": ["code_gen", "agentic", "planning"] - }, - "abacusai/dracarys-llama-3.1-70b-instruct": { - "rating": 4, "context_window": 131072, "description": "Abacus.AI fine-tune of Llama 3.1 70B, optimized for complex code instruction following.", "strengths": ["code_gen", "instruction"] - }, - "nvidia/nv-embedcode-7b-v1": { - "rating": 3, "context_window": 8192, "description": "NVIDIA specialized coding embedding model for semantic code search.", "strengths": ["embedding", "code_search"] - } - } - }, - - # === REASONING / COMPLEX TASKS === - # Best for: multi-step reasoning, mathematical analysis, strategy planning, agentic workflows - "reasoning": { - "primary": "google/diffusiongemma-26b-a4b-it", - "fallback": ["meta/llama-3.1-70b-instruct", "minimaxai/minimax-m3", "z-ai/glm-5.2", "nvidia/nemotron-3-ultra-550b-a55b", "thinkingmachines/inkling", "meta/llama-3.3-70b-instruct", "nvidia/llama-3.3-nemotron-super-49b-v1.5", "meta/llama-3.1-8b-instruct"], - "models": { - "minimaxai/minimax-m3": { - "rating": 5, "context_window": 128000, "description": "MiniMax-M3 SOTA reasoning model. Exceptional financial analysis and long context planning.", "strengths": ["reasoning", "finance", "instruction"] - }, - "google/diffusiongemma-26b-a4b-it": { - "rating": 5, "context_window": 128000, "description": "Google DiffusionGemma 26B A4B IT. Next-gen reasoning and fast inference.", "strengths": ["reasoning", "finance", "agentic"] - }, - "z-ai/glm-5.2": { - "rating": 5, "context_window": 128000, "description": "GLM-5.2 SOTA reasoning model. Outstanding multi-step planning and financial instruction following.", "strengths": ["reasoning", "finance", "instruction"] - }, - "nvidia/nemotron-3-ultra-550b-a55b": { - "rating": 5, "context_window": 128000, "description": "NVIDIA 550B reasoning model. Exceptional logical analysis.", "strengths": ["reasoning", "agentic", "math"] - }, - "thinkingmachines/inkling": { - "rating": 4, "context_window": 32768, "description": "Inkling 975B MoE reasoning model. Excels at complex analysis.", "strengths": ["reasoning", "moe"] - }, - "meta/llama-3.1-70b-instruct": { - "rating": 5, "context_window": 128000, "description": "Meta Llama 3.1 70B Instruct. High throughput, low latency.", "strengths": ["reasoning", "finance", "fast"] - }, - "meta/llama-3.3-70b-instruct": { - "rating": 5, "context_window": 128000, "description": "Meta Llama 3.3 70B Instruct.", "strengths": ["reasoning", "finance"] - }, - "nvidia/llama-3.3-nemotron-super-49b-v1.5": { - "rating": 4, "context_window": 128000, "description": "NVIDIA Nemotron Super 49B v1.5.", "strengths": ["reasoning", "agentic"] - }, - "meta/llama-3.1-8b-instruct": { - "rating": 4, "context_window": 128000, "description": "Llama 3.1 8B Instruct. Fast fallback.", "strengths": ["fast", "general"] - } - } - }, - - # === CHAT / GENERAL PURPOSE === - # Best for: conversational Q&A, summarization, instruction-following - "chat": { - "primary": "meta/llama-3.3-70b-instruct", - "fallback": ["meta/llama-3.1-70b-instruct", "meta/llama-3.1-8b-instruct", "minimaxai/minimax-m3", "mistralai/mistral-medium-3.5-128b", "google/gemma-4-31b-it", "mistralai/mistral-small-4-119b-2603", "qwen/qwen3.5-122b-a10b", "nvidia/nemotron-3-nano-30b-a3b", "mistralai/ministral-14b-instruct-2512", "bytedance/seed-oss-36b-instruct", "nvidia/nvidia-nemotron-nano-9b-v2", "openai/gpt-oss-20b", "openai/gpt-oss-120b", "sarvamai/sarvam-m", "google/gemma-3n-e4b-it", "google/gemma-3n-e2b-it", "mistralai/mistral-nemotron", "nvidia/llama-3.1-nemotron-nano-8b-v1", "upstage/solar-10.7b-instruct", "mistralai/mixtral-8x7b-instruct-v0.1", "minimaxai/minimax-m2.7", "stepfun-ai/step-3.7-flash", "stepfun-ai/step-3.5-flash"], - "models": { - "meta/llama-3.3-70b-instruct": { - "rating": 5, "context_window": 128000, "description": "Meta Llama 3.3 70B. High-quality instruction following, default conversational model.", "strengths": ["chat", "general", "instruction"] - }, - "meta/llama-3.1-70b-instruct": { - "rating": 4, "context_window": 128000, "description": "Meta Llama 3.1 70B. Excellent instruction execution and formatting.", "strengths": ["chat", "general"] - }, - "meta/llama-3.1-8b-instruct": { - "rating": 4, "context_window": 128000, "description": "Llama 3.1 8B. Fast, reliable conversational chat model.", "strengths": ["chat", "fast", "general"] - }, - "minimaxai/minimax-m3": { - "rating": 4, "context_window": 40960, "description": "MiniMax M3. Strong general assistant, optimized for long-form dialogue.", "strengths": ["chat", "dialogue"] - }, - "minimaxai/minimax-m2.7": { - "rating": 4, "context_window": 40960, "description": "MiniMax M2.7. Fast conversational LLM.", "strengths": ["chat", "fast"] - }, - "stepfun-ai/step-3.7-flash": { - "rating": 4, "context_window": 128000, "description": "StepFun 3.7 Flash. Ultra-fast high-concurrency conversational assistant.", "strengths": ["chat", "fast"] - }, - "stepfun-ai/step-3.5-flash": { - "rating": 3, "context_window": 128000, "description": "StepFun 3.5 Flash model.", "strengths": ["chat", "fast"] - }, - "mistralai/mistral-medium-3.5-128b": { - "rating": 4, "context_window": 128000, "description": "Mistral Medium 3.5. Balanced instruct model with great reasoning capacity.", "strengths": ["chat", "instruction"] - }, - "google/gemma-4-31b-it": { - "rating": 4, "context_window": 128000, "description": "Google Gemma 4 31B instruction-tuned model. Top scores in general chat.", "strengths": ["chat", "instruction"] - }, - "mistralai/mistral-small-4-119b-2603": { - "rating": 4, "context_window": 32768, "description": "Mistral Small 4 119B. High quality intermediate model for general instruct tasks.", "strengths": ["chat", "instruction"] - }, - "qwen/qwen3.5-122b-a10b": { - "rating": 4, "context_window": 32768, "description": "Qwen3.5 122B MoE model, highly conversational.", "strengths": ["chat", "moe"] - }, - "nvidia/nemotron-3-nano-30b-a3b": { - "rating": 3, "context_window": 32768, "description": "NVIDIA Nemotron 3 Nano.", "strengths": ["chat", "fast"] - }, - "mistralai/ministral-14b-instruct-2512": { - "rating": 3, "context_window": 32768, "description": "Ministral 14B instruction model.", "strengths": ["chat", "fast"] - }, - "bytedance/seed-oss-36b-instruct": { - "rating": 4, "context_window": 32768, "description": "ByteDance Seed OSS 36B.", "strengths": ["chat", "general"] - }, - "nvidia/nvidia-nemotron-nano-9b-v2": { - "rating": 3, "context_window": 32768, "description": "NVIDIA Nemotron Nano 9B v2.", "strengths": ["chat", "fast"] - }, - "openai/gpt-oss-20b": { - "rating": 4, "context_window": 32768, "description": "OpenAI GPT-OSS 20B model.", "strengths": ["chat", "general"] - }, - "openai/gpt-oss-120b": { - "rating": 4, "context_window": 32768, "description": "OpenAI GPT-OSS 120B model.", "strengths": ["chat", "general"] - }, - "sarvamai/sarvam-m": { - "rating": 3, "context_window": 8192, "description": "Sarvam-M multilingual chat model.", "strengths": ["chat", "multilingual"] - }, - "google/gemma-3n-e4b-it": { - "rating": 4, "context_window": 128000, "description": "Google Gemma 3N E4B model.", "strengths": ["chat", "general"] - }, - "google/gemma-3n-e2b-it": { - "rating": 4, "context_window": 128000, "description": "Google Gemma 3N E2B model.", "strengths": ["chat", "general"] - }, - "mistralai/mistral-nemotron": { - "rating": 4, "context_window": 128000, "description": "Mistral-Nemotron collaboration model.", "strengths": ["chat", "general"] - }, - "nvidia/llama-3.1-nemotron-nano-8b-v1": { - "rating": 3, "context_window": 128000, "description": "NVIDIA tuned Llama 3.1 Nano 8B.", "strengths": ["chat", "fast"] - }, - "upstage/solar-10.7b-instruct": { - "rating": 3, "context_window": 4096, "description": "Upstage Solar 10.7B instruction-tuned model.", "strengths": ["chat", "instruction"] - }, - "mistralai/mixtral-8x7b-instruct-v0.1": { - "rating": 3, "context_window": 32768, "description": "Mixtral 8x7B MoE model.", "strengths": ["chat", "moe"] - } - } - }, - - # === MEMORY / SUMMARIZATION === - # Best for: MEMORY.md updates, long-context summarization, KB compression - "memory": { - "primary": "meta/llama-4-maverick-17b-128e-instruct", - "fallback": ["moonshotai/kimi-k2.6"], - "models": { - "meta/llama-4-maverick-17b-128e-instruct": { - "rating": 5, "context_window": 1000000, "description": "Llama 4 Maverick. 1M token context, perfect for large codebase digests.", "strengths": ["summarization", "long_context", "memory"] - }, - "moonshotai/kimi-k2.6": { - "rating": 4, "context_window": 131072, "description": "Kimi K2.6. 128K context window, highly accurate long-document analysis.", "strengths": ["summarization", "long_context"] - } - } - }, - - # === SAFETY / CONTENT MODERATION === - # Best for: guardrails, PII detection, content safety checks, topic control - "safety": { - "primary": "nvidia/nemotron-3.5-content-safety", - "fallback": ["meta/llama-guard-4-12b", "nvidia/gliner-pii", "nvidia/llama-3.1-nemotron-safety-guard-8b-v3"], - "models": { - "nvidia/nemotron-3.5-content-safety": { - "rating": 5, "context_window": 4096, "description": "NVIDIA specialized content safety model.", "strengths": ["safety", "moderation"] - }, - "meta/llama-guard-4-12b": { - "rating": 4, "context_window": 128000, "description": "Llama Guard 4 12B safety classifier.", "strengths": ["safety", "moderation"] - }, - "nvidia/gliner-pii": { - "rating": 4, "context_window": 8192, "description": "GLiNER PII recognition model for anonymization.", "strengths": ["safety", "pii_redaction"] - }, - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": { - "rating": 4, "context_window": 8192, "description": "NVIDIA safety guard 8B v3.", "strengths": ["safety", "moderation"] - } - } - }, - - # === TRANSLATION === - "translation": { - "primary": "nvidia/riva-translate-4b-instruct-v1.1", - "models": { - "nvidia/riva-translate-4b-instruct-v1.1": { - "rating": 4, "context_window": 4096, "description": "Riva machine translation model.", "strengths": ["translation"] - } - } - }, - - # === EMBEDDING === - # Best for: vector embeddings for semantic search, RAG, document retrieval - "embedding": { - "primary": "nvidia/nv-embedcode-7b-v1", - "fallback": ["nvidia/nemotron-3-embed-1b", "nvidia/rerank-qa-mistral-4b"], - "models": { - "nvidia/nv-embedcode-7b-v1": { - "rating": 4, "context_window": 8192, "description": "NVIDIA specialized code embedding model.", "strengths": ["embedding", "code_search"] - }, - "nvidia/nemotron-3-embed-1b": { - "rating": 3, "context_window": 512, "description": "Nemotron-3 1B embedding model.", "strengths": ["embedding"] - }, - "nvidia/rerank-qa-mistral-4b": { - "rating": 4, "context_window": 4096, "description": "Mistral-based QA reranker model.", "strengths": ["rerank", "rag"] - } - } - }, - - # === VISION / MULTIMODAL === - # Best for: image understanding, chart analysis, VLM-based tasks - "vision": { - "primary": "meta/llama-3.2-90b-vision-instruct", - "fallback": ["meta/llama-3.2-11b-vision-instruct", "google/diffusiongemma-26b-a4b-it", "google/paligemma", "nvidia/nemotron-nano-12b-v2-vl", "nvidia/llama-3.1-nemotron-nano-vl-8b-v1"], - "models": { - "meta/llama-3.2-90b-vision-instruct": { - "rating": 5, "context_window": 128000, "description": "Meta Llama 3.2 90B Vision. High-end image and chart understanding.", "strengths": ["vision", "chart_analysis"] - }, - "meta/llama-3.2-11b-vision-instruct": { - "rating": 4, "context_window": 128000, "description": "Meta Llama 3.2 11B Vision. Efficient image captioning and Q&A.", "strengths": ["vision", "fast"] - }, - "google/diffusiongemma-26b-a4b-it": { - "rating": 4, "context_window": 8192, "description": "DiffusionGemma 26B multimodal visual instruction model.", "strengths": ["vision", "diffusion"] - }, - "google/paligemma": { - "rating": 3, "context_window": 4096, "description": "Google PaliGemma multimodal model.", "strengths": ["vision", "vqa"] - }, - "nvidia/nemotron-nano-12b-v2-vl": { - "rating": 4, "context_window": 32768, "description": "NVIDIA Nemotron Nano 12B VL model.", "strengths": ["vision", "vl"] - }, - "nvidia/llama-3.1-nemotron-nano-vl-8b-v1": { - "rating": 3, "context_window": 128000, "description": "NVIDIA tuned Llama 3.1 Nano VL 8B.", "strengths": ["vision", "fast"] - } - } - }, - - # === FAST / LIGHTWEIGHT === - # Best for: quick classification, token-budget-conscious tasks, high-throughput pipelines - "fast": { - "primary": "meta/llama-3.2-3b-instruct", - "fallback": ["meta/llama-3.2-1b-instruct", "nvidia/nemotron-mini-4b-instruct", "google/gemma-2-2b-it"], - "models": { - "meta/llama-3.2-3b-instruct": { - "rating": 4, "context_window": 128000, "description": "Llama 3.2 3B. Fast, lightweight edge-tier chatbot.", "strengths": ["fast", "lightweight"] - }, - "meta/llama-3.2-1b-instruct": { - "rating": 3, "context_window": 128000, "description": "Llama 3.2 1B. Compact model for low token footprint.", "strengths": ["fast", "lightweight"] - }, - "nvidia/nemotron-mini-4b-instruct": { - "rating": 3, "context_window": 4096, "description": "Nemotron Mini 4B instruction model.", "strengths": ["fast", "compact"] - }, - "google/gemma-2-2b-it": { - "rating": 3, "context_window": 8192, "description": "Google Gemma-2 2B instruct.", "strengths": ["fast", "compact"] - } - } - }, - - # === SCIENCE & SPECIALIZED === - "science": { - "primary": "nvidia/esmfold", - "fallback": ["nvidia/esm2-650m"], - "models": { - "nvidia/esmfold": { - "rating": 5, "context_window": 1024, "description": "ESMFold model for direct protein structure prediction from sequence.", "strengths": ["science", "protein_folding"] - }, - "nvidia/esm2-650m": { - "rating": 4, "context_window": 1024, "description": "ESM-2 650M parameter protein language model.", "strengths": ["science", "protein_language"] - } - } - }, - - # === MULTIMODAL VISION / VEHICLES === - "multimodal_vision": { - "primary": "nvidia/cosmos3-nano", - "fallback": ["nvidia/cosmos3-nano-reasoner", "nvidia/ai-synthetic-video-detector", "nvidia/active-speaker-detection", "nvidia/ising-calibration-1-35b-a3b", "nvidia/cosmos-transfer2.5-2b", "nvidia/streampetr", "nvidia/cosmos-transfer1-7b", "nvidia/sparsedrive", "nvidia/bevformer"], - "models": { - "nvidia/cosmos3-nano": { - "rating": 5, "context_window": 8192, "description": "Cosmos-3 Nano spatial video model.", "strengths": ["video", "spatial"] - }, - "nvidia/cosmos3-nano-reasoner": { - "rating": 5, "context_window": 8192, "description": "Cosmos-3 Nano reasoning visual agent.", "strengths": ["video", "reasoning"] - }, - "nvidia/ai-synthetic-video-detector": { - "rating": 4, "context_window": 4096, "description": "Synthetic video generation detector.", "strengths": ["video", "detection"] - }, - "nvidia/active-speaker-detection": { - "rating": 4, "context_window": 4096, "description": "Maxine active speaker detection.", "strengths": ["video", "maxine"] - }, - "nvidia/ising-calibration-1-35b-a3b": { - "rating": 3, "context_window": 4096, "description": "Ising calibration vision model.", "strengths": ["video"] - }, - "nvidia/cosmos-transfer2.5-2b": { - "rating": 4, "context_window": 4096, "description": "Cosmos Transfer 2.5 2B video diffusion helper.", "strengths": ["video"] - }, - "nvidia/streampetr": { - "rating": 4, "context_window": 4096, "description": "StreamPETR 3D object detection for autonomous driving.", "strengths": ["driving", "3d_detection"] - }, - "nvidia/cosmos-transfer1-7b": { - "rating": 4, "context_window": 4096, "description": "Cosmos Transfer 1 7B video model.", "strengths": ["video"] - }, - "nvidia/sparsedrive": { - "rating": 4, "context_window": 4096, "description": "SparseDrive planning for autonomous vehicles.", "strengths": ["driving"] - }, - "nvidia/bevformer": { - "rating": 4, "context_window": 4096, "description": "BEVFormer bird's-eye-view camera perception.", "strengths": ["driving", "perception"] - } - } - }, - - # === AUDIO / SPEECH === - "audio": { - "primary": "nvidia/nemotron-voicechat", - "fallback": ["nvidia/magpie-tts-zeroshot", "nvidia/background-noise-removal", "nvidia/studio-voice-enhance"], - "models": { - "nvidia/nemotron-voicechat": { - "rating": 5, "context_window": 4096, "description": "Nemotron model optimized for voice-based conversational turn-taking.", "strengths": ["voice", "chat"] - }, - "nvidia/magpie-tts-zeroshot": { - "rating": 4, "context_window": 4096, "description": "Magpie zero-shot Text-to-Speech synthesis.", "strengths": ["voice", "tts"] - }, - "nvidia/background-noise-removal": { - "rating": 4, "context_window": 4096, "description": "Maxine background noise removal audio NIM.", "strengths": ["audio", "maxine"] - }, - "nvidia/studio-voice-enhance": { - "rating": 4, "context_window": 4096, "description": "Maxine Studio voice enhance audio NIM.", "strengths": ["audio", "maxine"] - } - } - } -} - -# Keyword → category routing table for automatic dispatch -ROUTING_KEYWORDS = { - "coding": ["code", "python", "function", "class", "debug", "refactor", "implement", "syntax", "algorithm", "programming", "script", "module", "bug", "fix", "rewrite", "agent_workflow", "agent"], - "reasoning": ["reason", "analyze", "plan", "strategy", "optimize", "complex", "math", "calculate", "evaluate", "multi-step", "explain", "compare", "decide", "judge", "score", "audit", "rate", "hallucin", "assess", "review trajectory", "llm-as-a-judge", "finance", "stock", "trading", "market", "portfolio", "financial", "investment", "equity", "backtest", "capital"], - "memory": ["summarize", "memory", "kb", "knowledge", "log", "history", "update memory", "compress", "recall", "persist"], - "chat": ["chat", "write", "creative", "document", "report", "essay", "story", "generate text"], - "safety": ["safety", "harmful", "pii", "content moderation", "guardrail", "toxic", "filter", "censor"], - "fast": ["quick", "fast", "brief", "short", "classify", "yes/no", "simple", "ping"], - "science": ["science", "protein", "esmfold", "esm2", "biology", "molecule", "structure", "medical", "health", "clinical", "patient", "diagnosis", "symptom", "drug"], - "vision": ["image", "chart", "graph", "visual", "picture", "plot", "screenshot", "vision"], - "multimodal_vision": ["cosmos", "video", "driving", "bevformer", "sparsedrive", "calibration"], - "audio": ["voice", "tts", "speech", "noise", "audio", "sound", "enhance"], - "translation": ["translate", "riva", "language", "bilingual", "multilingual"], -} - - -def route_request(prompt: str, hint: Optional[str] = None) -> tuple[str, str]: - """ - Determine the best category and model for a given prompt. - Returns (category, model_id). - - hint: explicit category override (e.g. 'coding', 'judge', 'memory') - """ - if hint and hint in NVIDIA_MODEL_REGISTRY: - cat = hint - return cat, NVIDIA_MODEL_REGISTRY[cat]["primary"] - - prompt_lower = prompt.lower() - scores = {} - for category, keywords in ROUTING_KEYWORDS.items(): - score = sum(1 for kw in keywords if kw in prompt_lower) - if score > 0: - scores[category] = score - - if scores: - best_cat = max(scores, key=scores.get) - return best_cat, NVIDIA_MODEL_REGISTRY[best_cat]["primary"] - - # Default: fast general chat - return "chat", NVIDIA_MODEL_REGISTRY["chat"]["primary"] - - -def get_model_registry() -> dict: - """Return the full model registry dict.""" - return NVIDIA_MODEL_REGISTRY - - -def get_best_model(category: str) -> str: - """Return the primary model ID for a given category.""" - # Handle remapped/deprecated categories gracefully - category_map = { - "judge": "reasoning", - "finance": "reasoning", - "creative": "chat", - "medical": "science" - } - mapped_cat = category_map.get(category, category) - entry = NVIDIA_MODEL_REGISTRY.get(mapped_cat) - if entry: - return entry["primary"] - return "meta/llama-3.1-8b-instruct" - - -def list_categories() -> list: - """Return all routing categories.""" - return list(NVIDIA_MODEL_REGISTRY.keys()) - - -def save_model_registry(): - """Persist registry to JSON for external inspection / KB.""" - os.makedirs(os.path.dirname(MODEL_REGISTRY_FILE), exist_ok=True) - try: - with open(MODEL_REGISTRY_FILE, "w") as f: - json.dump({ - "updated_at": datetime.datetime.now().isoformat(), - "total_nvidia_nim_models": 74, - "categories": list(NVIDIA_MODEL_REGISTRY.keys()), - "registry": NVIDIA_MODEL_REGISTRY, - "routing_keywords": ROUTING_KEYWORDS, - }, f, indent=2) - print(f"[NVIDIA LLM] Model registry saved to {MODEL_REGISTRY_FILE}") - except Exception as e: - print(f"[NVIDIA LLM] Failed to save registry: {e}") - - -# --------------------------------------------------------------------------- -# Keyring resolution and API status testing -# --------------------------------------------------------------------------- -_NVIDIA_API_STATUS = { - "tested": False, - "success": False, - "message": "Not initialized", - "source": "None" -} - -def resolve_nvidia_api_key() -> tuple[str | None, str]: - """ - Fetch NVIDIA API Key from environment variable, .env files, or system keyring. - Returns (api_key, source_description). - """ - # 1. Environment variable - key = os.getenv("NVIDIA_API_KEY") or os.getenv("NVIDIA_NIM_API_KEY") - if key: - clean_key = str(key).replace("\n", "").replace("\r", "").strip() - if clean_key: - return clean_key, "Environment Variable" - - # 2. Check dotenv explicitly - try: - from dotenv import find_dotenv, dotenv_values - found = find_dotenv(usecwd=True) - if found: - vals = dotenv_values(found) - key = vals.get("NVIDIA_API_KEY") or vals.get("NVIDIA_NIM_API_KEY") - if key: - clean_key = str(key).replace("\n", "").replace("\r", "").strip() - if clean_key: - os.environ["NVIDIA_API_KEY"] = clean_key - return clean_key, f"Dotenv File ({os.path.basename(found)})" - except Exception: - pass - - # 3. System Keyring - if keyring: - candidate_services = ["system", "nvidia", "nsedash", "NVIDIA_API_KEY", "NVIDIA", "nvidia_nim"] - candidate_usernames = ["NVIDIA_API_KEY", "api_key", "key", "default"] - for svc in candidate_services: - for usr in candidate_usernames: - try: - secret = keyring.get_password(svc, usr) - if secret: - clean_secret = str(secret).replace("\n", "").replace("\r", "").strip() - if clean_secret: - os.environ["NVIDIA_API_KEY"] = clean_secret - return clean_secret, f"System Keyring ({svc}/{usr})" - except Exception: - pass - - return None, "None" - - -def test_nvidia_api_connection(api_key: str = None) -> tuple[bool, str]: - """ - Test NVIDIA NIM API connection with the resolved API key. - Returns (success: bool, message: str). - """ - key = api_key or os.getenv("NVIDIA_API_KEY") - if not key: - return False, "NVIDIA_API_KEY is not set in environment or system keyring" - - headers = { - "Authorization": f"Bearer {key.strip()}", - "Accept": "application/json" - } - try: - resp = requests.get("https://integrate.api.nvidia.com/v1/models", headers=headers, timeout=5) - if resp.status_code == 200: - return True, "API connection tested & successful" - elif resp.status_code == 401: - return False, "NVIDIA API Key Unauthorized (HTTP 401)" - else: - return False, f"NVIDIA API test returned HTTP status {resp.status_code}" - except Exception as err: - return False, f"NVIDIA API connection test failed: {err}" - - -def init_nvidia_api_status(force_retest: bool = False) -> dict: - """Initialize key resolution and API connection test once at startup.""" - global _NVIDIA_API_STATUS - if _NVIDIA_API_STATUS["tested"] and not force_retest: - return _NVIDIA_API_STATUS - - key, source = resolve_nvidia_api_key() - if not key: - _NVIDIA_API_STATUS = { - "tested": True, - "success": False, - "message": "NVIDIA_API_KEY missing in environment and system keyring", - "source": source - } - return _NVIDIA_API_STATUS - - success, msg = test_nvidia_api_connection(key) - _NVIDIA_API_STATUS = { - "tested": True, - "success": success, - "message": msg, - "source": source - } - if success: - print(f"[NVIDIA LLM] API Key loaded from {source}. Connection tested and successful.") - else: - print(f"[NVIDIA LLM] API Key loaded from {source}. Connection test warning: {msg}") - return _NVIDIA_API_STATUS - - -def get_nvidia_api_status() -> dict: - """Return the cached or freshly initialized NVIDIA API status.""" - return init_nvidia_api_status() - - -# --------------------------------------------------------------------------- -# NvidiaLLM — CrewAI-compatible wrapper with intelligent routing -# --------------------------------------------------------------------------- -class NvidiaLLM(BaseLLM): - """ - CrewAI-compatible LLM wrapper for NVIDIA NIM API. - Supports intelligent routing to specialist models by task category. - """ - - def __init__(self, **kwargs): - # Resolve API key from environment variable first, then fallback to keyring - api_key = kwargs.pop("api_key", None) - if not api_key: - api_key, _ = resolve_nvidia_api_key() - - if api_key: - api_key = str(api_key).replace("\n", "").replace("\r", "").strip() - - if not api_key: - global _WARNED_MISSING_KEY - if not _WARNED_MISSING_KEY: - print("WARNING: NVIDIA_API_KEY is not set. Free NVIDIA NIM API features require NVIDIA_API_KEY in environment or keyring.") - _WARNED_MISSING_KEY = True - - # Default to fast chat model - kwargs["api_key"] = api_key - kwargs.setdefault("model", "meta/llama-3.1-8b-instruct") - super().__init__(**kwargs) - - self.max_tokens = kwargs.get("max_tokens", 4096) - self.temperature = kwargs.get("temperature", 1.0) - self.top_p = kwargs.get("top_p", 0.95) - self.timeout = kwargs.get("timeout", 120) - - def _raw_call(self, model_id: str, formatted_messages: list, max_tokens: int, timeout: int = None) -> str: - """Make a single raw API call with explicit model_id.""" - effective_timeout = timeout or self.timeout - invoke_url = "https://integrate.api.nvidia.com/v1/chat/completions" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Accept": "application/json", - "Content-Type": "application/json", - } - payload = { - "model": model_id, - "messages": formatted_messages, - "max_tokens": max_tokens, - "temperature": self.temperature, - "top_p": self.top_p, - "stream": False, - } - - resp = requests.post(invoke_url, headers=headers, json=payload, timeout=effective_timeout) - resp.raise_for_status() - resp_data = resp.json() - - # Track tokens per model - usage = resp_data.get("usage", {}) - pt = usage.get("prompt_tokens", 0) - ct = usage.get("completion_tokens", 0) - if pt > 0 or ct > 0: - add_tokens(pt, ct, model_id=model_id) - - if "choices" in resp_data and len(resp_data["choices"]) > 0: - message = resp_data["choices"][0].get("message", {}) - if message.get("content"): - return message["content"].strip() - return "" - - def call(self, messages: list[dict], callbacks: list[Any] | None = None, **kwargs: Any) -> str: - """ - Main entry point. Supports: - - hint="coding"/"judge"/"memory" etc. for explicit routing - - Automatic keyword-based routing if hint not provided - - Fallback chain through category fallback models on failure - """ - # Budget guardrail - current_usage = get_token_usage() - if current_usage.get("total_tokens", 0) > 10000000: - print("[AGENTS DIAG ERROR] Token budget exceeded (10M limit)! Throttling LLM call.") - return "[LLM ERROR] Token budget exceeded. API call blocked by guardrail." - - # Normalize messages - if isinstance(messages, str): - formatted_messages = [{"role": "user", "content": messages}] - prompt_text = messages - elif isinstance(messages, list): - formatted_messages = messages - prompt_text = " ".join(m.get("content", "") for m in messages if isinstance(m, dict)) - else: - formatted_messages = [{"role": "user", "content": str(messages)}] - prompt_text = str(messages) - - max_tokens = kwargs.get("max_tokens") or self.max_tokens - - # --- Intelligent Routing --- - hint = kwargs.get("hint") # caller can pass hint="coding" etc. - - # Per-category dynamic timeouts (large models need more time on free tier) - CATEGORY_TIMEOUTS = { - "coding": 180, - "reasoning": 300, # 300s (5 minutes) - "memory": 180, - "judge": 120, - "finance": 300, # 300s (5 minutes) - "vision": 90, - "safety": 30, - "fast": 20, - "chat": 60, - "creative": 120, - "medical": 60, - "embedding": 30, - } - - # Per-category max_tokens caps - CATEGORY_MAX_TOKENS = { - "coding": 8192, - "reasoning": 32768, # 32K context / output - "memory": 8192, - "judge": 2048, - "finance": 32768, # 32K context / output - "chat": 2048, - "fast": 512, - "creative": 4096, - "medical": 2048, - "safety": 256, - "embedding": 512, - } - - # --- Intelligent Routing & Explicit Overrides --- - hint = kwargs.get("hint") # caller can pass hint="coding" etc. - explicit_model = kwargs.get("model") or kwargs.get("selected_model") or (self.model if self.model != "meta/llama-3.1-8b-instruct" else None) - explicit_timeout = kwargs.get("timeout") - - # Determine category, primary model, and fallbacks - if explicit_model: - category = hint or "reasoning" - model_id = explicit_model - registry_entry = NVIDIA_MODEL_REGISTRY.get(category, {}) - all_fallbacks = registry_entry.get("fallback", []) - fallbacks = [m for m in all_fallbacks if m != model_id] - else: - category, model_id = route_request(prompt_text, hint=hint) - registry_entry = NVIDIA_MODEL_REGISTRY.get(category, {}) - fallbacks = registry_entry.get("fallback", []) - - # Build candidate chain starting with primary/selected model - candidates = [] - for cand in [model_id] + fallbacks: - if cand and cand not in candidates: - candidates.append(cand) - - # Apply timeout override or category-tuned timeout - if explicit_timeout: - try: - effective_timeout = int(explicit_timeout) - except (ValueError, TypeError): - effective_timeout = CATEGORY_TIMEOUTS.get(category, self.timeout) - else: - effective_timeout = CATEGORY_TIMEOUTS.get(category, self.timeout) - - effective_max_tokens = min(max_tokens, CATEGORY_MAX_TOKENS.get(category, max_tokens)) - - print(f"[NVIDIA LLM ROUTER] Primary/Selected={model_id} | Category={category} | Total Candidates={len(candidates)} | Overall Budget={effective_timeout}s") - - last_error = None - start_t = time.time() - detailed_req = kwargs.get("detailed", False) - - for idx, candidate in enumerate(candidates): - elapsed_so_far = time.time() - start_t - remaining_budget = effective_timeout - elapsed_so_far - if remaining_budget <= 3: - print(f"[NVIDIA LLM ROUTER] Overall timeout budget ({effective_timeout}s) nearly exhausted ({elapsed_so_far:.1f}s elapsed). Stopping candidate loop.") - break - - # Cap each candidate call to 35s max to move quickly to responsive models on free tier - cand_timeout = min(35, int(remaining_budget)) - print(f"[NVIDIA LLM ROUTER] Candidate [{idx+1}/{len(candidates)}]: {candidate} (call timeout={cand_timeout}s, remaining budget={int(remaining_budget)}s)") - - try: - result = self._raw_call_detailed(candidate, formatted_messages, effective_max_tokens, timeout=cand_timeout) - if result and result.get("text"): - if candidate != model_id: - print(f"[NVIDIA LLM ROUTER] Fallback model successfully generated report: {candidate}") - result["elapsed_seconds"] = round(time.time() - start_t, 2) - return result if detailed_req else result["text"] - except requests.exceptions.HTTPError as e: - last_error = f"[LLM HTTP ERROR] {e}" - if hasattr(e, "response") and e.response is not None and e.response.status_code == 401: - err_msg = "[LLM ERROR] Unauthorized: Invalid NVIDIA_API_KEY." - return {"text": err_msg, "model_id": candidate} if detailed_req else err_msg - print(f"[NVIDIA LLM ROUTER] Model {candidate} failed ({e}), trying next candidate...") - except requests.exceptions.ConnectionError: - last_error = "[LLM OFFLINE] Cannot connect to the NVIDIA API." - print(f"[NVIDIA LLM ROUTER] Connection error for {candidate}, trying next candidate...") - except requests.exceptions.Timeout: - last_error = f"[LLM TIMEOUT] Model {candidate} timed out after {cand_timeout}s." - print(f"[NVIDIA LLM ROUTER] Model {candidate} timed out after {cand_timeout}s, trying next candidate...") - except Exception as e: - last_error = f"[LLM ERROR] {str(e)}" - print(f"[NVIDIA LLM ROUTER] Model {candidate} error: {e}, trying next candidate...") - - err_out = last_error or "[LLM ERROR] All models in fallback chain failed or budget exhausted." - return {"text": err_out, "model_id": model_id} if detailed_req else err_out - - def _raw_call_detailed(self, model_id: str, formatted_messages: list, max_tokens: int, timeout: int = None) -> dict: - """Make a raw API call and return text + usage metadata.""" - effective_timeout = timeout or self.timeout - invoke_url = "https://integrate.api.nvidia.com/v1/chat/completions" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Accept": "application/json", - "Content-Type": "application/json", - } - payload = { - "model": model_id, - "messages": formatted_messages, - "max_tokens": max_tokens, - "temperature": self.temperature, - "top_p": self.top_p, - "stream": False, - } - - resp = requests.post(invoke_url, headers=headers, json=payload, timeout=effective_timeout) - resp.raise_for_status() - resp_data = resp.json() - - usage = resp_data.get("usage", {}) - pt = usage.get("prompt_tokens", 0) - ct = usage.get("completion_tokens", 0) - if pt > 0 or ct > 0: - add_tokens(pt, ct, model_id=model_id) - - content = "" - if "choices" in resp_data and len(resp_data["choices"]) > 0: - message = resp_data["choices"][0].get("message", {}) - if message.get("content"): - content = message["content"].strip() - - return { - "text": content, - "model_id": model_id, - "prompt_tokens": pt, - "completion_tokens": ct, - "total_tokens": pt + ct, - } - - def supports_function_calling(self) -> bool: - return False - - def supports_stop_words(self) -> bool: - return False - - -# --------------------------------------------------------------------------- -# Convenience factory functions for specific categories -# --------------------------------------------------------------------------- -def get_coding_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for coding tasks.""" - return NvidiaLLM(model=get_best_model("coding"), **kwargs) - -def get_reasoning_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for complex reasoning.""" - return NvidiaLLM(model=get_best_model("reasoning"), **kwargs) - -def get_memory_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for memory/summarization.""" - return NvidiaLLM(model=get_best_model("memory"), **kwargs) - -def get_judge_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for LLM-as-a-judge evaluation.""" - return NvidiaLLM(model=get_best_model("judge"), **kwargs) - -def get_finance_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for financial analysis.""" - return NvidiaLLM(model=get_best_model("finance"), **kwargs) - -def get_fast_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for fast/lightweight tasks.""" - return NvidiaLLM(model=get_best_model("fast"), **kwargs) - -def get_creative_llm(**kwargs) -> NvidiaLLM: - """Return an NvidiaLLM pre-configured for creative writing and documentation.""" - return NvidiaLLM(model=get_best_model("creative"), **kwargs) - - -# --------------------------------------------------------------------------- -# Default singleton (chat model, auto-routing enabled) -# --------------------------------------------------------------------------- -_llm_instance = None - -def get_nvidia_llm() -> NvidiaLLM: - global _llm_instance - if _llm_instance is None: - _llm_instance = NvidiaLLM() - return _llm_instance - - -# --------------------------------------------------------------------------- -# Eager initialization on main thread + save registry to JSON -# --------------------------------------------------------------------------- -try: - init_nvidia_api_status() - get_nvidia_llm() - save_model_registry() -except Exception as e: - print(f"[NVIDIA LLM] Initialization warning: {e}") diff --git a/nsedash/core/nvidia_model_registry.json b/nsedash/core/nvidia_model_registry.json deleted file mode 100644 index 006b5aa8ae6b6dfaa5530ac115c026d0e02924c4..0000000000000000000000000000000000000000 --- a/nsedash/core/nvidia_model_registry.json +++ /dev/null @@ -1,981 +0,0 @@ -{ - "updated_at": "2026-07-30T01:20:19.883433", - "total_nvidia_nim_models": 74, - "categories": [ - "coding", - "reasoning", - "chat", - "memory", - "safety", - "translation", - "embedding", - "vision", - "fast", - "science", - "multimodal_vision", - "audio" - ], - "registry": { - "coding": { - "primary": "deepseek-ai/deepseek-v4-pro", - "fallback": [ - "deepseek-ai/deepseek-v4-flash", - "poolside/laguna-xs-2.1", - "abacusai/dracarys-llama-3.1-70b-instruct", - "nvidia/nv-embedcode-7b-v1" - ], - "models": { - "deepseek-ai/deepseek-v4-pro": { - "rating": 5, - "context_window": 163840, - "description": "State-of-the-art coder. Outstanding performance on code generation and multi-step agent debugging.", - "strengths": [ - "code_gen", - "agentic", - "debugging" - ] - }, - "deepseek-ai/deepseek-v4-flash": { - "rating": 4, - "context_window": 163840, - "description": "High-throughput coding model. Ideal for low-latency coding loops.", - "strengths": [ - "code_gen", - "fast", - "agentic" - ] - }, - "poolside/laguna-xs-2.1": { - "rating": 5, - "context_window": 32768, - "description": "Poolside Laguna XS. Specially trained for agentic coding and long-horizon developer workflows.", - "strengths": [ - "code_gen", - "agentic", - "planning" - ] - }, - "abacusai/dracarys-llama-3.1-70b-instruct": { - "rating": 4, - "context_window": 131072, - "description": "Abacus.AI fine-tune of Llama 3.1 70B, optimized for complex code instruction following.", - "strengths": [ - "code_gen", - "instruction" - ] - }, - "nvidia/nv-embedcode-7b-v1": { - "rating": 3, - "context_window": 8192, - "description": "NVIDIA specialized coding embedding model for semantic code search.", - "strengths": [ - "embedding", - "code_search" - ] - } - } - }, - "reasoning": { - "primary": "google/diffusiongemma-26b-a4b-it", - "fallback": [ - "meta/llama-3.1-70b-instruct", - "minimaxai/minimax-m3", - "z-ai/glm-5.2", - "nvidia/nemotron-3-ultra-550b-a55b", - "thinkingmachines/inkling", - "meta/llama-3.3-70b-instruct", - "nvidia/llama-3.3-nemotron-super-49b-v1.5", - "meta/llama-3.1-8b-instruct" - ], - "models": { - "minimaxai/minimax-m3": { - "rating": 5, - "context_window": 128000, - "description": "MiniMax-M3 SOTA reasoning model. Exceptional financial analysis and long context planning.", - "strengths": [ - "reasoning", - "finance", - "instruction" - ] - }, - "google/diffusiongemma-26b-a4b-it": { - "rating": 5, - "context_window": 128000, - "description": "Google DiffusionGemma 26B A4B IT. Next-gen reasoning and fast inference.", - "strengths": [ - "reasoning", - "finance", - "agentic" - ] - }, - "z-ai/glm-5.2": { - "rating": 5, - "context_window": 128000, - "description": "GLM-5.2 SOTA reasoning model. Outstanding multi-step planning and financial instruction following.", - "strengths": [ - "reasoning", - "finance", - "instruction" - ] - }, - "nvidia/nemotron-3-ultra-550b-a55b": { - "rating": 5, - "context_window": 128000, - "description": "NVIDIA 550B reasoning model. Exceptional logical analysis.", - "strengths": [ - "reasoning", - "agentic", - "math" - ] - }, - "thinkingmachines/inkling": { - "rating": 4, - "context_window": 32768, - "description": "Inkling 975B MoE reasoning model. Excels at complex analysis.", - "strengths": [ - "reasoning", - "moe" - ] - }, - "meta/llama-3.1-70b-instruct": { - "rating": 5, - "context_window": 128000, - "description": "Meta Llama 3.1 70B Instruct. High throughput, low latency.", - "strengths": [ - "reasoning", - "finance", - "fast" - ] - }, - "meta/llama-3.3-70b-instruct": { - "rating": 5, - "context_window": 128000, - "description": "Meta Llama 3.3 70B Instruct.", - "strengths": [ - "reasoning", - "finance" - ] - }, - "nvidia/llama-3.3-nemotron-super-49b-v1.5": { - "rating": 4, - "context_window": 128000, - "description": "NVIDIA Nemotron Super 49B v1.5.", - "strengths": [ - "reasoning", - "agentic" - ] - }, - "meta/llama-3.1-8b-instruct": { - "rating": 4, - "context_window": 128000, - "description": "Llama 3.1 8B Instruct. Fast fallback.", - "strengths": [ - "fast", - "general" - ] - } - } - }, - "chat": { - "primary": "meta/llama-3.3-70b-instruct", - "fallback": [ - "meta/llama-3.1-70b-instruct", - "meta/llama-3.1-8b-instruct", - "minimaxai/minimax-m3", - "mistralai/mistral-medium-3.5-128b", - "google/gemma-4-31b-it", - "mistralai/mistral-small-4-119b-2603", - "qwen/qwen3.5-122b-a10b", - "nvidia/nemotron-3-nano-30b-a3b", - "mistralai/ministral-14b-instruct-2512", - "bytedance/seed-oss-36b-instruct", - "nvidia/nvidia-nemotron-nano-9b-v2", - "openai/gpt-oss-20b", - "openai/gpt-oss-120b", - "sarvamai/sarvam-m", - "google/gemma-3n-e4b-it", - "google/gemma-3n-e2b-it", - "mistralai/mistral-nemotron", - "nvidia/llama-3.1-nemotron-nano-8b-v1", - "upstage/solar-10.7b-instruct", - "mistralai/mixtral-8x7b-instruct-v0.1", - "minimaxai/minimax-m2.7", - "stepfun-ai/step-3.7-flash", - "stepfun-ai/step-3.5-flash" - ], - "models": { - "meta/llama-3.3-70b-instruct": { - "rating": 5, - "context_window": 128000, - "description": "Meta Llama 3.3 70B. High-quality instruction following, default conversational model.", - "strengths": [ - "chat", - "general", - "instruction" - ] - }, - "meta/llama-3.1-70b-instruct": { - "rating": 4, - "context_window": 128000, - "description": "Meta Llama 3.1 70B. Excellent instruction execution and formatting.", - "strengths": [ - "chat", - "general" - ] - }, - "meta/llama-3.1-8b-instruct": { - "rating": 4, - "context_window": 128000, - "description": "Llama 3.1 8B. Fast, reliable conversational chat model.", - "strengths": [ - "chat", - "fast", - "general" - ] - }, - "minimaxai/minimax-m3": { - "rating": 4, - "context_window": 40960, - "description": "MiniMax M3. Strong general assistant, optimized for long-form dialogue.", - "strengths": [ - "chat", - "dialogue" - ] - }, - "minimaxai/minimax-m2.7": { - "rating": 4, - "context_window": 40960, - "description": "MiniMax M2.7. Fast conversational LLM.", - "strengths": [ - "chat", - "fast" - ] - }, - "stepfun-ai/step-3.7-flash": { - "rating": 4, - "context_window": 128000, - "description": "StepFun 3.7 Flash. Ultra-fast high-concurrency conversational assistant.", - "strengths": [ - "chat", - "fast" - ] - }, - "stepfun-ai/step-3.5-flash": { - "rating": 3, - "context_window": 128000, - "description": "StepFun 3.5 Flash model.", - "strengths": [ - "chat", - "fast" - ] - }, - "mistralai/mistral-medium-3.5-128b": { - "rating": 4, - "context_window": 128000, - "description": "Mistral Medium 3.5. Balanced instruct model with great reasoning capacity.", - "strengths": [ - "chat", - "instruction" - ] - }, - "google/gemma-4-31b-it": { - "rating": 4, - "context_window": 128000, - "description": "Google Gemma 4 31B instruction-tuned model. Top scores in general chat.", - "strengths": [ - "chat", - "instruction" - ] - }, - "mistralai/mistral-small-4-119b-2603": { - "rating": 4, - "context_window": 32768, - "description": "Mistral Small 4 119B. High quality intermediate model for general instruct tasks.", - "strengths": [ - "chat", - "instruction" - ] - }, - "qwen/qwen3.5-122b-a10b": { - "rating": 4, - "context_window": 32768, - "description": "Qwen3.5 122B MoE model, highly conversational.", - "strengths": [ - "chat", - "moe" - ] - }, - "nvidia/nemotron-3-nano-30b-a3b": { - "rating": 3, - "context_window": 32768, - "description": "NVIDIA Nemotron 3 Nano.", - "strengths": [ - "chat", - "fast" - ] - }, - "mistralai/ministral-14b-instruct-2512": { - "rating": 3, - "context_window": 32768, - "description": "Ministral 14B instruction model.", - "strengths": [ - "chat", - "fast" - ] - }, - "bytedance/seed-oss-36b-instruct": { - "rating": 4, - "context_window": 32768, - "description": "ByteDance Seed OSS 36B.", - "strengths": [ - "chat", - "general" - ] - }, - "nvidia/nvidia-nemotron-nano-9b-v2": { - "rating": 3, - "context_window": 32768, - "description": "NVIDIA Nemotron Nano 9B v2.", - "strengths": [ - "chat", - "fast" - ] - }, - "openai/gpt-oss-20b": { - "rating": 4, - "context_window": 32768, - "description": "OpenAI GPT-OSS 20B model.", - "strengths": [ - "chat", - "general" - ] - }, - "openai/gpt-oss-120b": { - "rating": 4, - "context_window": 32768, - "description": "OpenAI GPT-OSS 120B model.", - "strengths": [ - "chat", - "general" - ] - }, - "sarvamai/sarvam-m": { - "rating": 3, - "context_window": 8192, - "description": "Sarvam-M multilingual chat model.", - "strengths": [ - "chat", - "multilingual" - ] - }, - "google/gemma-3n-e4b-it": { - "rating": 4, - "context_window": 128000, - "description": "Google Gemma 3N E4B model.", - "strengths": [ - "chat", - "general" - ] - }, - "google/gemma-3n-e2b-it": { - "rating": 4, - "context_window": 128000, - "description": "Google Gemma 3N E2B model.", - "strengths": [ - "chat", - "general" - ] - }, - "mistralai/mistral-nemotron": { - "rating": 4, - "context_window": 128000, - "description": "Mistral-Nemotron collaboration model.", - "strengths": [ - "chat", - "general" - ] - }, - "nvidia/llama-3.1-nemotron-nano-8b-v1": { - "rating": 3, - "context_window": 128000, - "description": "NVIDIA tuned Llama 3.1 Nano 8B.", - "strengths": [ - "chat", - "fast" - ] - }, - "upstage/solar-10.7b-instruct": { - "rating": 3, - "context_window": 4096, - "description": "Upstage Solar 10.7B instruction-tuned model.", - "strengths": [ - "chat", - "instruction" - ] - }, - "mistralai/mixtral-8x7b-instruct-v0.1": { - "rating": 3, - "context_window": 32768, - "description": "Mixtral 8x7B MoE model.", - "strengths": [ - "chat", - "moe" - ] - } - } - }, - "memory": { - "primary": "meta/llama-4-maverick-17b-128e-instruct", - "fallback": [ - "moonshotai/kimi-k2.6" - ], - "models": { - "meta/llama-4-maverick-17b-128e-instruct": { - "rating": 5, - "context_window": 1000000, - "description": "Llama 4 Maverick. 1M token context, perfect for large codebase digests.", - "strengths": [ - "summarization", - "long_context", - "memory" - ] - }, - "moonshotai/kimi-k2.6": { - "rating": 4, - "context_window": 131072, - "description": "Kimi K2.6. 128K context window, highly accurate long-document analysis.", - "strengths": [ - "summarization", - "long_context" - ] - } - } - }, - "safety": { - "primary": "nvidia/nemotron-3.5-content-safety", - "fallback": [ - "meta/llama-guard-4-12b", - "nvidia/gliner-pii", - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3" - ], - "models": { - "nvidia/nemotron-3.5-content-safety": { - "rating": 5, - "context_window": 4096, - "description": "NVIDIA specialized content safety model.", - "strengths": [ - "safety", - "moderation" - ] - }, - "meta/llama-guard-4-12b": { - "rating": 4, - "context_window": 128000, - "description": "Llama Guard 4 12B safety classifier.", - "strengths": [ - "safety", - "moderation" - ] - }, - "nvidia/gliner-pii": { - "rating": 4, - "context_window": 8192, - "description": "GLiNER PII recognition model for anonymization.", - "strengths": [ - "safety", - "pii_redaction" - ] - }, - "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": { - "rating": 4, - "context_window": 8192, - "description": "NVIDIA safety guard 8B v3.", - "strengths": [ - "safety", - "moderation" - ] - } - } - }, - "translation": { - "primary": "nvidia/riva-translate-4b-instruct-v1.1", - "models": { - "nvidia/riva-translate-4b-instruct-v1.1": { - "rating": 4, - "context_window": 4096, - "description": "Riva machine translation model.", - "strengths": [ - "translation" - ] - } - } - }, - "embedding": { - "primary": "nvidia/nv-embedcode-7b-v1", - "fallback": [ - "nvidia/nemotron-3-embed-1b", - "nvidia/rerank-qa-mistral-4b" - ], - "models": { - "nvidia/nv-embedcode-7b-v1": { - "rating": 4, - "context_window": 8192, - "description": "NVIDIA specialized code embedding model.", - "strengths": [ - "embedding", - "code_search" - ] - }, - "nvidia/nemotron-3-embed-1b": { - "rating": 3, - "context_window": 512, - "description": "Nemotron-3 1B embedding model.", - "strengths": [ - "embedding" - ] - }, - "nvidia/rerank-qa-mistral-4b": { - "rating": 4, - "context_window": 4096, - "description": "Mistral-based QA reranker model.", - "strengths": [ - "rerank", - "rag" - ] - } - } - }, - "vision": { - "primary": "meta/llama-3.2-90b-vision-instruct", - "fallback": [ - "meta/llama-3.2-11b-vision-instruct", - "google/diffusiongemma-26b-a4b-it", - "google/paligemma", - "nvidia/nemotron-nano-12b-v2-vl", - "nvidia/llama-3.1-nemotron-nano-vl-8b-v1" - ], - "models": { - "meta/llama-3.2-90b-vision-instruct": { - "rating": 5, - "context_window": 128000, - "description": "Meta Llama 3.2 90B Vision. High-end image and chart understanding.", - "strengths": [ - "vision", - "chart_analysis" - ] - }, - "meta/llama-3.2-11b-vision-instruct": { - "rating": 4, - "context_window": 128000, - "description": "Meta Llama 3.2 11B Vision. Efficient image captioning and Q&A.", - "strengths": [ - "vision", - "fast" - ] - }, - "google/diffusiongemma-26b-a4b-it": { - "rating": 4, - "context_window": 8192, - "description": "DiffusionGemma 26B multimodal visual instruction model.", - "strengths": [ - "vision", - "diffusion" - ] - }, - "google/paligemma": { - "rating": 3, - "context_window": 4096, - "description": "Google PaliGemma multimodal model.", - "strengths": [ - "vision", - "vqa" - ] - }, - "nvidia/nemotron-nano-12b-v2-vl": { - "rating": 4, - "context_window": 32768, - "description": "NVIDIA Nemotron Nano 12B VL model.", - "strengths": [ - "vision", - "vl" - ] - }, - "nvidia/llama-3.1-nemotron-nano-vl-8b-v1": { - "rating": 3, - "context_window": 128000, - "description": "NVIDIA tuned Llama 3.1 Nano VL 8B.", - "strengths": [ - "vision", - "fast" - ] - } - } - }, - "fast": { - "primary": "meta/llama-3.2-3b-instruct", - "fallback": [ - "meta/llama-3.2-1b-instruct", - "nvidia/nemotron-mini-4b-instruct", - "google/gemma-2-2b-it" - ], - "models": { - "meta/llama-3.2-3b-instruct": { - "rating": 4, - "context_window": 128000, - "description": "Llama 3.2 3B. Fast, lightweight edge-tier chatbot.", - "strengths": [ - "fast", - "lightweight" - ] - }, - "meta/llama-3.2-1b-instruct": { - "rating": 3, - "context_window": 128000, - "description": "Llama 3.2 1B. Compact model for low token footprint.", - "strengths": [ - "fast", - "lightweight" - ] - }, - "nvidia/nemotron-mini-4b-instruct": { - "rating": 3, - "context_window": 4096, - "description": "Nemotron Mini 4B instruction model.", - "strengths": [ - "fast", - "compact" - ] - }, - "google/gemma-2-2b-it": { - "rating": 3, - "context_window": 8192, - "description": "Google Gemma-2 2B instruct.", - "strengths": [ - "fast", - "compact" - ] - } - } - }, - "science": { - "primary": "nvidia/esmfold", - "fallback": [ - "nvidia/esm2-650m" - ], - "models": { - "nvidia/esmfold": { - "rating": 5, - "context_window": 1024, - "description": "ESMFold model for direct protein structure prediction from sequence.", - "strengths": [ - "science", - "protein_folding" - ] - }, - "nvidia/esm2-650m": { - "rating": 4, - "context_window": 1024, - "description": "ESM-2 650M parameter protein language model.", - "strengths": [ - "science", - "protein_language" - ] - } - } - }, - "multimodal_vision": { - "primary": "nvidia/cosmos3-nano", - "fallback": [ - "nvidia/cosmos3-nano-reasoner", - "nvidia/ai-synthetic-video-detector", - "nvidia/active-speaker-detection", - "nvidia/ising-calibration-1-35b-a3b", - "nvidia/cosmos-transfer2.5-2b", - "nvidia/streampetr", - "nvidia/cosmos-transfer1-7b", - "nvidia/sparsedrive", - "nvidia/bevformer" - ], - "models": { - "nvidia/cosmos3-nano": { - "rating": 5, - "context_window": 8192, - "description": "Cosmos-3 Nano spatial video model.", - "strengths": [ - "video", - "spatial" - ] - }, - "nvidia/cosmos3-nano-reasoner": { - "rating": 5, - "context_window": 8192, - "description": "Cosmos-3 Nano reasoning visual agent.", - "strengths": [ - "video", - "reasoning" - ] - }, - "nvidia/ai-synthetic-video-detector": { - "rating": 4, - "context_window": 4096, - "description": "Synthetic video generation detector.", - "strengths": [ - "video", - "detection" - ] - }, - "nvidia/active-speaker-detection": { - "rating": 4, - "context_window": 4096, - "description": "Maxine active speaker detection.", - "strengths": [ - "video", - "maxine" - ] - }, - "nvidia/ising-calibration-1-35b-a3b": { - "rating": 3, - "context_window": 4096, - "description": "Ising calibration vision model.", - "strengths": [ - "video" - ] - }, - "nvidia/cosmos-transfer2.5-2b": { - "rating": 4, - "context_window": 4096, - "description": "Cosmos Transfer 2.5 2B video diffusion helper.", - "strengths": [ - "video" - ] - }, - "nvidia/streampetr": { - "rating": 4, - "context_window": 4096, - "description": "StreamPETR 3D object detection for autonomous driving.", - "strengths": [ - "driving", - "3d_detection" - ] - }, - "nvidia/cosmos-transfer1-7b": { - "rating": 4, - "context_window": 4096, - "description": "Cosmos Transfer 1 7B video model.", - "strengths": [ - "video" - ] - }, - "nvidia/sparsedrive": { - "rating": 4, - "context_window": 4096, - "description": "SparseDrive planning for autonomous vehicles.", - "strengths": [ - "driving" - ] - }, - "nvidia/bevformer": { - "rating": 4, - "context_window": 4096, - "description": "BEVFormer bird's-eye-view camera perception.", - "strengths": [ - "driving", - "perception" - ] - } - } - }, - "audio": { - "primary": "nvidia/nemotron-voicechat", - "fallback": [ - "nvidia/magpie-tts-zeroshot", - "nvidia/background-noise-removal", - "nvidia/studio-voice-enhance" - ], - "models": { - "nvidia/nemotron-voicechat": { - "rating": 5, - "context_window": 4096, - "description": "Nemotron model optimized for voice-based conversational turn-taking.", - "strengths": [ - "voice", - "chat" - ] - }, - "nvidia/magpie-tts-zeroshot": { - "rating": 4, - "context_window": 4096, - "description": "Magpie zero-shot Text-to-Speech synthesis.", - "strengths": [ - "voice", - "tts" - ] - }, - "nvidia/background-noise-removal": { - "rating": 4, - "context_window": 4096, - "description": "Maxine background noise removal audio NIM.", - "strengths": [ - "audio", - "maxine" - ] - }, - "nvidia/studio-voice-enhance": { - "rating": 4, - "context_window": 4096, - "description": "Maxine Studio voice enhance audio NIM.", - "strengths": [ - "audio", - "maxine" - ] - } - } - } - }, - "routing_keywords": { - "coding": [ - "code", - "python", - "function", - "class", - "debug", - "refactor", - "implement", - "syntax", - "algorithm", - "programming", - "script", - "module", - "bug", - "fix", - "rewrite", - "agent_workflow", - "agent" - ], - "reasoning": [ - "reason", - "analyze", - "plan", - "strategy", - "optimize", - "complex", - "math", - "calculate", - "evaluate", - "multi-step", - "explain", - "compare", - "decide", - "judge", - "score", - "audit", - "rate", - "hallucin", - "assess", - "review trajectory", - "llm-as-a-judge", - "finance", - "stock", - "trading", - "market", - "portfolio", - "financial", - "investment", - "equity", - "backtest", - "capital" - ], - "memory": [ - "summarize", - "memory", - "kb", - "knowledge", - "log", - "history", - "update memory", - "compress", - "recall", - "persist" - ], - "chat": [ - "chat", - "write", - "creative", - "document", - "report", - "essay", - "story", - "generate text" - ], - "safety": [ - "safety", - "harmful", - "pii", - "content moderation", - "guardrail", - "toxic", - "filter", - "censor" - ], - "fast": [ - "quick", - "fast", - "brief", - "short", - "classify", - "yes/no", - "simple", - "ping" - ], - "science": [ - "science", - "protein", - "esmfold", - "esm2", - "biology", - "molecule", - "structure", - "medical", - "health", - "clinical", - "patient", - "diagnosis", - "symptom", - "drug" - ], - "vision": [ - "image", - "chart", - "graph", - "visual", - "picture", - "plot", - "screenshot", - "vision" - ], - "multimodal_vision": [ - "cosmos", - "video", - "driving", - "bevformer", - "sparsedrive", - "calibration" - ], - "audio": [ - "voice", - "tts", - "speech", - "noise", - "audio", - "sound", - "enhance" - ], - "translation": [ - "translate", - "riva", - "language", - "bilingual", - "multilingual" - ] - } -} \ No newline at end of file diff --git a/nsedash/core/portfolio.py b/nsedash/core/portfolio.py deleted file mode 100644 index ae74f7ec8c7139686af8459d1e915fab5eded4e3..0000000000000000000000000000000000000000 --- a/nsedash/core/portfolio.py +++ /dev/null @@ -1,311 +0,0 @@ -""" -nsedash.core.portfolio -====================== -ICICI portfolio import (PortFolioEqtSummary.csv mapped through -NSEScripMaster.txt inside SecurityMaster.zip) and portfolio summary -calculation. Parity with v4.5 ``load_portfolio_data`` and -``update_portfolio_summary`` minus the UI coupling. -""" - -from __future__ import annotations - -import logging -import zipfile -from pathlib import Path - -import pandas as pd - -from ..config import Paths - - -class PortfolioImportError(Exception): - """Raised with a user-presentable message when the import cannot proceed.""" - - -def clean_symbol(sym: str) -> str: - """Clean standard ticker symbols from common suffixes like .NS and -EQ.""" - sym = str(sym).strip().upper() - if sym.endswith(".NS"): - sym = sym[:-3] - if sym.endswith("-EQ"): - sym = sym[:-3] - return sym - - -def identify_broker(file_name: str, columns: list[str]) -> str: - """Identify the broker based on filename and column header heuristics.""" - filename_lower = file_name.lower() - normalized_cols = [str(c).strip().lower() for c in columns] - - # Heuristics based on filename - if 'icici' in filename_lower or 'portfolioeqtsummary' in filename_lower: - return "ICICIDirect" - elif 'hdfc' in filename_lower: - return "HDFCSecurities" - elif 'groww' in filename_lower: - return "Groww" - elif 'angel' in filename_lower: - return "AngelOne" - elif 'upstox' in filename_lower: - return "Upstox" - elif 'zerodha' in filename_lower or 'holdings' in filename_lower: - return "Zerodha" - - # Heuristics based on headers - if 'average cost price' in normalized_cols or 'stock symbol' in normalized_cols: - return "ICICIDirect" - elif 'avg. cost' in normalized_cols or 'qty.' in normalized_cols: - return "Zerodha" - elif 'trading_symbol' in normalized_cols or 'groww_locked_quantity' in normalized_cols: - return "Groww" - elif 'average acquisition price' in normalized_cols or 'scrip name' in normalized_cols: - return "AngelOne" - elif 'holding qty' in normalized_cols: - return "Upstox" - elif 'sip_indicator' in normalized_cols or 'mtf_indicator' in normalized_cols or 'company_name' in normalized_cols: - return "HDFCSecurities" - - return "Generic" - - -def identify_columns(df_columns: list[str]) -> tuple[str | None, str | None, str | None]: - """Intelligently map columns to Symbol, Quantity, and Price using candidates list.""" - normalized_cols = [str(c).strip().lower() for c in df_columns] - col_map = {str(c).strip().lower(): c for c in df_columns} - - symbol_candidates = ['stock symbol', 'instrument', 'trading_symbol', 'symbol', 'trading symbol', 'scrip name', 'stock name', 'company name', 'isin name'] - qty_candidates = ['qty', 'qty.', 'quantity', 'available qty.', 'holding qty', 'current balance', 'available qty', 'qty_available', 'holding quantity', 'quantity'] - price_candidates = ['average cost price', 'avg. cost', 'average_price', 'average price', 'avg price', 'buy average', 'average cost', 'average acquisition price', 'avg cost', 'buy price', 'avg_price', 'avg_cost'] - - symbol_col = None - qty_col = None - price_col = None - - for cand in symbol_candidates: - if cand in normalized_cols: - symbol_col = col_map[cand] - break - if not symbol_col: - for c in normalized_cols: - if any(cand in c for cand in ['symbol', 'instrument', 'scrip', 'company_name']): - symbol_col = col_map[c] - break - - for cand in qty_candidates: - if cand in normalized_cols: - qty_col = col_map[cand] - break - if not qty_col: - for c in normalized_cols: - if any(cand in c for cand in ['qty', 'quantity', 'balance']): - qty_col = col_map[c] - break - - for cand in price_candidates: - if cand in normalized_cols: - price_col = col_map[cand] - break - if not price_col: - for c in normalized_cols: - if any(cand in c for cand in ['avg', 'price', 'cost', 'buy']): - price_col = col_map[c] - break - - return symbol_col, qty_col, price_col - - -def import_portfolio(file_path: Path | str | None, paths: Paths) -> tuple[list[dict], list[str], str]: - """ - Import holdings from a file path (or fallback to paths.portfolio_file). - Identifies the broker, parses columns, cleans symbols, and returns - (holdings, unmapped_symbols, broker_name). - """ - if file_path is None: - portfolio_file = paths.portfolio_file - # Check if CSV exists - csv_portfolio_file = Path(str(portfolio_file).replace('.parquet', '.csv')) - if csv_portfolio_file.exists(): - try: - df_temp = pd.read_csv(csv_portfolio_file) - df_temp.to_parquet(portfolio_file, index=False) - csv_portfolio_file.unlink() - logging.info(f"Auto-converted portfolio CSV '{csv_portfolio_file}' to Parquet.") - except Exception as e: - logging.warning(f"Failed to auto-convert portfolio CSV: {e}") - file_path = portfolio_file - elif portfolio_file.exists(): - file_path = portfolio_file - else: - raise PortfolioImportError(f"Portfolio file not found: {portfolio_file}") - - file_path = Path(file_path) - is_parquet = file_path.suffix.lower() == '.parquet' - - try: - if is_parquet: - df = pd.read_parquet(file_path) - else: - df = pd.read_csv(file_path) - except Exception as e: - raise PortfolioImportError(f"Failed to read portfolio file: {e}") - - if df.empty: - raise PortfolioImportError("One of the CSV files is empty.") - - # Strip columns - df.columns = df.columns.str.strip().str.replace('"', '') - - # Check if this is a previously standardized file by us - if 'Broker' in df.columns and 'SYMBOL' in df.columns: - broker_name = df['Broker'].iloc[0] if not df['Broker'].empty else "Generic" - holdings = [] - for _, row in df.iterrows(): - holdings.append({ - "SYMBOL": str(row['SYMBOL']), - "Quantity": float(row['Quantity']) if pd.notna(row['Quantity']) else 0.0, - "Price": float(row['Price']) if pd.notna(row['Price']) else 0.0, - }) - return holdings, [], broker_name - - broker_name = identify_broker(file_path.name, df.columns) - - symbol_col, qty_col, price_col = identify_columns(df.columns) - if not symbol_col or not qty_col or not price_col: - if len(df.columns) >= 3: - symbol_col = df.columns[0] - qty_col = df.columns[1] - price_col = df.columns[2] - else: - raise PortfolioImportError( - f"Could not map columns for broker '{broker_name}'. Expected columns for Symbol, Quantity, and Price." - ) - - holdings = [] - unmapped = [] - - if broker_name == "ICICIDirect": - # Load ICICI Direct specific mapping - mapping_zip_file = paths.security_master_zip - if not mapping_zip_file.exists(): - raise PortfolioImportError(f"Mapping zip file not found: {mapping_zip_file}") - - mapping_parquet = paths.data_dir / 'NSEScripMaster.parquet' - need_convert = True - if mapping_parquet.exists(): - if mapping_parquet.stat().st_mtime > mapping_zip_file.stat().st_mtime: - need_convert = False - - if need_convert: - with zipfile.ZipFile(mapping_zip_file, 'r') as z: - if 'NSEScripMaster.txt' not in z.namelist(): - raise PortfolioImportError("NSEScripMaster.txt not found in SecurityMaster.zip") - z.extract('NSEScripMaster.txt', path=paths.data_dir) - mapping_txt = paths.data_dir / 'NSEScripMaster.txt' - if not mapping_txt.exists(): - raise PortfolioImportError(f"Mapping file not found: {mapping_txt}") - try: - mapping_df = pd.read_csv(mapping_txt) - mapping_df.to_parquet(mapping_parquet, index=False) - mapping_txt.unlink() - logging.info("Auto-converted mapping txt to NSEScripMaster.parquet.") - except Exception as e: - raise PortfolioImportError(f"Failed to convert mapping CSV to Parquet: {e}") - - try: - mapping_df = pd.read_parquet(mapping_parquet) - except Exception as e: - raise PortfolioImportError(f"Failed to read mapping Parquet: {e}") - - mapping_df.columns = mapping_df.columns.str.strip().str.replace('"', '') - - try: - icici_to_nse = dict(zip(mapping_df['ShortName'].astype(str).str.strip(), mapping_df['ExchangeCode'].astype(str).str.strip())) - except KeyError as ke: - raise PortfolioImportError(f"Missing expected column in mapping file: {ke}") - - for _, row in df.iterrows(): - raw_sym = str(row[symbol_col]).strip() - nse_sym = icici_to_nse.get(raw_sym, None) - if nse_sym: - try: - qty = float(row[qty_col]) if pd.notna(row[qty_col]) else 0.0 - price = float(row[price_col]) if pd.notna(row[price_col]) else 0.0 - holdings.append({ - "SYMBOL": nse_sym, - "Quantity": qty, - "Price": price - }) - except (ValueError, KeyError) as e: - logging.warning(f"Failed to parse row for {raw_sym}: {e}") - else: - unmapped.append(raw_sym) - else: - # Standard cleaning for Zerodha, Groww, AngelOne, Upstox, HDFCSecurities, Generic - for _, row in df.iterrows(): - raw_sym = str(row[symbol_col]).strip() - nse_sym = clean_symbol(raw_sym) - if nse_sym: - try: - qty = float(row[qty_col]) if pd.notna(row[qty_col]) else 0.0 - price = float(row[price_col]) if pd.notna(row[price_col]) else 0.0 - holdings.append({ - "SYMBOL": nse_sym, - "Quantity": qty, - "Price": price - }) - except (ValueError, KeyError) as e: - logging.warning(f"Failed to parse row for {raw_sym}: {e}") - else: - unmapped.append(raw_sym) - - return holdings, unmapped, broker_name - - -def import_icici_portfolio(paths: Paths) -> tuple[list[dict], list[str]]: - """Thin backwards-compatible wrapper calling import_portfolio.""" - holdings, unmapped, _ = import_portfolio(None, paths) - return holdings, unmapped - - - -def compute_portfolio_summary(cells: list[dict]) -> dict: - """ - Aggregate the portfolio metrics displayed in the top bar. - - Each cell dict must provide: Quantity, Price (cost), current_cmp, - prev_close (0.0 when unknown — the caller applies the v4.5 fallback of - prev_close := current_cmp when no historic row exists). - """ - total_invested = 0.0 - total_current_value = 0.0 - total_day_gain = 0.0 - total_previous_close_value = 0.0 - for c in cells: - qty = float(c.get('Quantity', 0.0) or 0.0) - cost_price = float(c.get('Price', 0.0) or 0.0) - current_cmp = float(c.get('current_cmp', 0.0) or 0.0) - prev_close = float(c.get('prev_close', 0.0) or 0.0) - if prev_close == 0.0 and current_cmp != 0.0: - prev_close = current_cmp - total_invested += qty * cost_price - total_current_value += qty * current_cmp - if prev_close > 0 and current_cmp > 0: - total_day_gain += qty * (current_cmp - prev_close) - total_previous_close_value += qty * prev_close - absolute_returns = total_current_value - total_invested - day_gain_pct = ( - total_day_gain / total_previous_close_value * 100 - if total_previous_close_value != 0 else 0.0 - ) - abs_returns_pct = ( - absolute_returns / total_invested * 100 - if total_invested != 0 else 0.0 - ) - return { - "invested": total_invested, - "current_value": total_current_value, - "day_gain_abs": total_day_gain, - "day_gain_pct": day_gain_pct, - "abs_returns_abs": absolute_returns, - "abs_returns_pct": abs_returns_pct, - } diff --git a/nsedash/core/recommender_bridge.py b/nsedash/core/recommender_bridge.py deleted file mode 100644 index 5faec27e4949578b5d11a3ef5725623d144e422e..0000000000000000000000000000000000000000 --- a/nsedash/core/recommender_bridge.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -nsedash.core.recommender_bridge -=============================== -Bridge to the external ``recommender.SymbolRecommender`` module (LSTM -buy/sell). v4.5 hard-imported ``recommender`` (and TensorFlow transitively) -at module import time, so the entire dashboard failed to start when the -optional deep-learning stack was absent (AUDIT FIX H-9). This bridge imports -lazily and degrades to ``N/A`` recommendations when unavailable, while -preserving full behaviour when the module is present. -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -from ..config import Recommendation - -_MIN_ROWS_FOR_PREDICTION = 61 # parity with v4.5 run_buy_sell_recommendation - - -def _load_recommender(recommender_dir: Path): - try: - import sys - nsereco_path = str(Path(__file__).resolve().parents[2] / 'nsereco') - if nsereco_path not in sys.path: - sys.path.append(nsereco_path) - from recommender import SymbolRecommender # noqa: PLC0415 - return SymbolRecommender(recommender_dir) - except Exception as e: - logging.warning(f"SymbolRecommender unavailable: {e}") - return None - - -def recommender_available(recommender_dir: Path) -> bool: - return _load_recommender(recommender_dir) is not None - - -def run_recommendations(recommender_dir: Path, symbols: list[str], - history_lookup) -> list[dict]: - """ - Generate buy/sell predictions for the given symbols. - - ``history_lookup(symbol)`` returns the symbol's historical DataFrame - sorted ascending by TRAN_DATE. - - Returns a list of {'symbol','action','confidence'} sorted by confidence - descending — the same records v4.5 showed in the results dialog, plus - one entry per skipped symbol carrying Recommendation.NA. - """ - recommender = _load_recommender(recommender_dir) - results: list[dict] = [] - for symbol in symbols: - model_file = Path(recommender_dir) / f"{symbol}.h5" - if recommender is None or not model_file.exists(): - if recommender is not None: - logging.warning(f"[{symbol}] No trained model found at {model_file}.") - results.append({ - "symbol": symbol, - "action": Recommendation.NA.value, - "confidence": 0.0, - "predicted": False, - }) - continue - symbol_df = history_lookup(symbol) - if symbol_df is not None and not symbol_df.empty \ - and len(symbol_df) >= _MIN_ROWS_FOR_PREDICTION: - tracker = getattr(recommender, 'tracker_data', {}) or {} - entry = tracker.get(symbol, {}) - confidence = entry.get('confidence_score', 0.0) - action = entry.get('recommendation', Recommendation.NA) - action = action.value if isinstance(action, Recommendation) else str(action) - results.append({ - "symbol": symbol, - "action": action, - "confidence": float(confidence), - "predicted": True, - }) - else: - logging.warning(f"[{symbol}] Skipping prediction: insufficient data.") - results.append({ - "symbol": symbol, - "action": Recommendation.NA.value, - "confidence": 0.0, - "predicted": False, - }) - results.sort(key=lambda r: r["confidence"], reverse=True) - return results - - -def train_all_models(recommender_dir: Path, symbols: list[str], - history_lookup) -> dict: - """ - Train LSTM models for all symbols. v4.5 delegated to - ``SymbolRecommender.train_all_async(symbols, lookup, tk_root)``; here the - web job system supplies the asynchrony, so the synchronous variant is - preferred when exposed by the module, falling back to the async API with - a ``None`` UI handle. - """ - recommender = _load_recommender(recommender_dir) - if recommender is None: - raise RuntimeError( - "SymbolRecommender module (and its TensorFlow dependency) is not " - "installed; deep-learning training is unavailable." - ) - if hasattr(recommender, 'train_all'): - recommender.train_all(symbols, history_lookup) - else: - recommender.train_all_async(symbols, history_lookup, None) - return {"trained_symbols": len(symbols)} diff --git a/nsedash/core/stock_data.py b/nsedash/core/stock_data.py deleted file mode 100644 index 0e494abd1a8b8a88819543524ad3169e24060f8a..0000000000000000000000000000000000000000 --- a/nsedash/core/stock_data.py +++ /dev/null @@ -1,504 +0,0 @@ -""" -nsedash.core.stock_data -======================= -Headless data layer (no UI coupling). Owns the historical DataFrame and all -derived analytics. Functional parity with v4.5 ``StockData`` minus the -Tkinter dialogs, which are replaced by exceptions/return values surfaced to -the web layer. - -Audit fixes applied here: - P-4 ``get_data_for_date`` no longer performs an O(rows) boolean scan of - the date slice per column assignment per symbol (O(S²) in v4.5); - results are accumulated per symbol and joined once. - P-5 The 5-day volume-change scan (30 per-day full-column comparisons per - symbol in v4.5) is replaced by a single windowed slice per symbol. - A-2 ``df['DAILY_RETURN'].fillna(0, inplace=True)`` (chained-assignment, - removed in pandas 3.x) replaced with safe column assignment. - P-6 The parquet cache is only rewritten when the source CSV has changed - (mtime check) instead of on every load. - D-1 Data-layer methods no longer call ``messagebox``; errors are logged - and raised/returned for the caller to present. - T-1 A re-entrant lock guards DataFrame replacement so background jobs and - request handlers cannot observe a half-swapped frame. -""" - -from __future__ import annotations - -import gc -import json -import logging -import threading -from datetime import datetime, timedelta -from pathlib import Path - -import numpy as np -import pandas as pd - -from ..config import (ANALYSIS_LIMIT, CSV_COLUMNS, CSV_DTYPES, - DATA_RETENTION_DAYS, Paths) -from ..logging_utils import elapsed_time_logger -from .live_fetch import LiveFetcher - - -class StockData: - """ - Manages all data I/O, historical analysis, and real-time scraping. - - Responsibilities (unchanged from v4.5): - - Loading / caching historical CSV data - - Computing returns, volatility, volume changes - - Real-time fetching (delegated to LiveFetcher) - - Watchlist JSON read/write helpers (delegated to watchlists module) - - ETF exclusion list management - - Moneycontrol score data access - """ - - def __init__(self, paths: Paths): - self.paths = paths - self.paths.ensure_dirs() - self._lock = threading.RLock() - self.df: pd.DataFrame = pd.DataFrame() - self.last_date = None - self.two_years_ago = None - self.etf_symbols = self.load_etf_symbols() - self.fetcher = LiveFetcher(prev_volume_lookup=self.previous_day_volume) - self.load_data() - - # ------------------------------------------------------------------ - # ETF Management - # ------------------------------------------------------------------ - - def load_etf_symbols(self) -> set: - """Load ETF symbols from the JSON exclusion list file.""" - try: - if self.paths.etf_list_file.exists(): - with open(self.paths.etf_list_file, 'r') as f: - return set(json.load(f)) - logging.warning( - f"ETF list file not found: {self.paths.etf_list_file}. " - "No ETFs will be excluded." - ) - return set() - except Exception as e: - logging.error( - f"Error loading ETF symbols from {self.paths.etf_list_file}: {e}" - ) - return set() - - # ------------------------------------------------------------------ - # Symbol Utilities - # ------------------------------------------------------------------ - - def get_all_symbols_from_df(self) -> list[str]: - """Return list of all unique stock symbols in the loaded DataFrame.""" - with self._lock: - if not self.df.empty: - return self.df['SYMBOL'].unique().tolist() - return [] - - def symbol_history(self, symbol: str) -> pd.DataFrame: - """All rows for a symbol sorted ascending by TRAN_DATE.""" - with self._lock: - return self.df[self.df['SYMBOL'] == symbol].sort_values('TRAN_DATE').copy() - - def previous_day_volume(self, symbol: str) -> float: - """Latest historic NET_TRDQTY strictly before today (for live fetch).""" - with self._lock: - df = self.df - if df.empty: - return 0.0 - rows = df[df['SYMBOL'] == symbol] - today = datetime.now().date() - prev = rows[rows['TRAN_DATE'].dt.date < today] - if prev.empty: - logging.warning(f"No previous day's historic volume found for {symbol}.") - return 0.0 - return float(prev.sort_values('TRAN_DATE').iloc[-1]['NET_TRDQTY']) - - # ------------------------------------------------------------------ - # Real-time delegation (API parity with v4.5) - # ------------------------------------------------------------------ - - def fetch_real_time_data(self, symbols, preferred_source="Google"): - return self.fetcher.fetch_real_time_data(symbols, preferred_source) - - # ------------------------------------------------------------------ - # Data Loading - # ------------------------------------------------------------------ - - @elapsed_time_logger("Loading stock data...") - def load_data(self) -> None: - """ - Load nse_processed_data.parquet, trim to the last 2 years, - compute DAILY_RETURN and VOLATILITY, and cache as parquet. - """ - try: - pq_path = self.paths.data_dir / 'nse_processed_data.parquet' - - if pq_path.exists(): - logging.info(f"Loading data from Parquet: {pq_path}") - df = pd.read_parquet(pq_path, columns=CSV_COLUMNS) - # Ensure TRAN_DATE is datetime type - if not pd.api.types.is_datetime64_any_dtype(df['TRAN_DATE']): - df['TRAN_DATE'] = pd.to_datetime(df['TRAN_DATE']) - else: - raise FileNotFoundError(f"Parquet file ({pq_path}) not found.") - - last_date = df['TRAN_DATE'].max() - two_years_ago = last_date - timedelta(days=DATA_RETENTION_DAYS) - df = df[df['TRAN_DATE'] >= two_years_ago].copy() - df = df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True) - - # Daily return (% change in closing price per symbol) - df['DAILY_RETURN'] = ( - df.groupby('SYMBOL', observed=True)['CLOSE_PRICE'].pct_change() * 100 - ) - df['DAILY_RETURN'] = df['DAILY_RETURN'].fillna(0) # AUDIT FIX A-2 - - # 30-day rolling volatility (std of daily returns) - df['VOLATILITY'] = df.groupby('SYMBOL', observed=True)['DAILY_RETURN'].transform( - lambda x: x.rolling(window=30, min_periods=1).std() - ) - df['VOLATILITY'] = df['VOLATILITY'].fillna(0) - - with self._lock: - self.df = df - self.last_date = last_date - self.two_years_ago = two_years_ago - - # Keep extract file updated for compatibility - try: - pq = self.paths.extract_file - if not pq.exists() or pq.stat().st_mtime < pq_path.stat().st_mtime: - df.to_parquet(pq, index=False) - except Exception as e: - logging.warning(f"Parquet cache write skipped: {e}") - - logging.info( - f"Data loaded from {two_years_ago} with " - "DAILY_RETURN and VOLATILITY calculated." - ) - gc.collect() - except Exception as e: - logging.error(f"Failed to load data: {e}") - raise - - # ------------------------------------------------------------------ - # Per-date enrichment - # ------------------------------------------------------------------ - - @elapsed_time_logger("Loading date data...") - def get_data_for_date(self, date) -> pd.DataFrame: - """ - Return enriched DataFrame for a specific trading date, including: - WEEK_CHANGE, MONTH_CHANGE, THREE_MONTH_CHANGE, TWO_YEAR_CHANGE, - PRICE_HISTORY, VOLUME_HISTORY, AVG_VOLUME, VOLUME_CHANGES. - Raises ValueError if no rows exist for the date. - """ - logging.info(f"Getting data for {date}") - if isinstance(date, datetime): - date = date.date() - if isinstance(date, pd.Timestamp): - date = date.date() - with self._lock: - df = self.df - date_data = df[df['TRAN_DATE'].dt.date == date].copy() - if date_data.empty: - raise ValueError(f"No data for {date}") - - grouped = df.groupby('SYMBOL', observed=True) - results: dict[str, dict] = {} - symbol_errors: list[tuple[str, str]] = [] - - for symbol in date_data['SYMBOL'].unique(): - try: - try: - symbol_data = grouped.get_group(symbol) - except KeyError: - raise ValueError(f"No historical data for symbol {symbol}") - if symbol_data.empty: - raise ValueError(f"No historical data for symbol {symbol}") - - curr_date = date_data.loc[ - date_data['SYMBOL'] == symbol, 'TRAN_DATE' - ].iloc[0] - dates = symbol_data['TRAN_DATE'] - closes = symbol_data['CLOSE_PRICE'] - res: dict = {} - - def window_change(days_back: int) -> float: - start = curr_date - timedelta(days=days_back) - prices = closes[(dates <= curr_date) & (dates >= start)] - if prices.empty: - return 0.0 - return float((prices.iloc[-1] / prices.iloc[0] - 1) * 100) - - res['WEEK_CHANGE'] = window_change(7) - res['MONTH_CHANGE'] = window_change(30) - res['THREE_MONTH_CHANGE'] = window_change(90) - - # --- 2-year window: change, histories, average volume --- - two_years_ago = curr_date - timedelta(days=365 * 2) - mask_2y = (dates <= curr_date) & (dates >= two_years_ago) - two_year_data = symbol_data[mask_2y] - if not two_year_data.empty: - res['TWO_YEAR_CHANGE'] = float( - (two_year_data['CLOSE_PRICE'].iloc[-1] / - two_year_data['CLOSE_PRICE'].iloc[0] - 1) * 100 - ) - res['PRICE_HISTORY'] = ( - two_year_data[['TRAN_DATE', 'CLOSE_PRICE']].values.tolist() - ) - res['VOLUME_HISTORY'] = ( - two_year_data[['TRAN_DATE', 'NET_TRDQTY']].values.tolist() - ) - res['AVG_VOLUME'] = float(two_year_data['NET_TRDQTY'].mean()) - else: - res['TWO_YEAR_CHANGE'] = 0.0 - res['PRICE_HISTORY'] = None - res['VOLUME_HISTORY'] = None - res['AVG_VOLUME'] = 0.0 - - # --- 5-day volume changes (last 6 trading days within a - # 30-calendar-day lookback, parity with v4.5) --- - lookback_start = curr_date - timedelta(days=29) - recent = symbol_data[ - (dates <= curr_date) & (dates > lookback_start - timedelta(days=1)) - ] - volumes_asc = recent['NET_TRDQTY'].tail(6).astype(float).tolist() - while len(volumes_asc) < 6: - volumes_asc.insert(0, 0.0) - volume_changes = [] - for i in range(1, len(volumes_asc)): - prev_v = volumes_asc[i - 1] - change = ( - ((volumes_asc[i] - prev_v) / prev_v) * 100 - if prev_v != 0 else 0 - ) - volume_changes.append(round(change, 2)) - while len(volume_changes) < 5: - volume_changes.insert(0, 0.0) - res['VOLUME_CHANGES'] = volume_changes[-5:] - - results[symbol] = res - except Exception as e: - logging.error(f"Failed to process calculations for {symbol}: {e}") - symbol_errors.append((symbol, str(e))) - - # AUDIT FIX P-4: single vectorised join instead of per-symbol .loc - # scans. (Note: SYMBOL is categorical, so plain-string iteration is - # used — Series.map on a categorical would try to build categories - # from the mapped values, which fails for list-valued columns.) - row_symbols = date_data['SYMBOL'].astype(str).tolist() - for col, default in [ - ('WEEK_CHANGE', 0.0), ('MONTH_CHANGE', 0.0), - ('THREE_MONTH_CHANGE', 0.0), ('TWO_YEAR_CHANGE', 0.0), - ('AVG_VOLUME', 0.0), - ]: - date_data[col] = pd.Series( - [results.get(s, {}).get(col, default) for s in row_symbols], - index=date_data.index, dtype=float, - ) - for col in ('PRICE_HISTORY', 'VOLUME_HISTORY', 'VOLUME_CHANGES'): - date_data[col] = pd.Series( - [results.get(s, {}).get(col) for s in row_symbols], - index=date_data.index, dtype=object, - ) - - if symbol_errors: - logging.warning( - f"Completed date data load for {date} with " - f"{len(symbol_errors)} symbol failures. " - f"Failures: {symbol_errors[:10]}..." - ) - logging.info(f"Successful to get data for {date}") - return date_data - - # ------------------------------------------------------------------ - # Volume analytics - # ------------------------------------------------------------------ - - @staticmethod - def _calculate_average_volume_change(symbol_data: pd.DataFrame, - days_back: int) -> float: - """ - Average daily volume % change for a symbol over ``days_back`` days. - ``symbol_data`` must be sorted ascending by TRAN_DATE. - """ - if symbol_data.empty or len(symbol_data) < 2: - return 0.0 - end_date = symbol_data['TRAN_DATE'].iloc[-1] - start_date = end_date - timedelta(days=days_back) - period = symbol_data[ - (symbol_data['TRAN_DATE'] <= end_date) & - (symbol_data['TRAN_DATE'] > start_date - timedelta(days=1)) - ] - if len(period) < 2: - return 0.0 - vols = period['NET_TRDQTY'].to_numpy(dtype=float) - prev, curr = vols[:-1], vols[1:] - with np.errstate(divide='ignore', invalid='ignore'): - changes = np.where(prev != 0, (curr - prev) / prev * 100, 0.0) - return float(changes.mean()) if changes.size else 0.0 - - @elapsed_time_logger("Fetching top volume movers...") - def get_top_volume_movers(self, duration: str, limit: int = ANALYSIS_LIMIT, - volume_cutoff: float = 0) -> pd.DataFrame: - """ - Rank stocks by average volume % change for the given duration - ('1D', '1W', '1M', '3M'). ETFs excluded. Returns DataFrame with - SYMBOL, VOLUME_CHANGE_PCT, SECURITY, AVG_VOLUME. - """ - days_map = {"1D": 1, "1W": 7, "1M": 30, "3M": 90} - days_back = days_map.get(duration) - if days_back is None: - logging.error(f"Invalid duration for volume analysis: {duration}") - return pd.DataFrame() - - with self._lock: - df = self.df - latest_date = df['TRAN_DATE'].max() - try: - all_latest = self.get_data_for_date(latest_date) - except Exception as e: - logging.error( - f"Error getting data for latest date in get_top_volume_movers: {e}" - ) - raise RuntimeError( - "Could not retrieve necessary data for volume analysis." - ) from e - if all_latest.empty: - logging.warning( - f"No data available for {latest_date} to perform volume analysis." - ) - return pd.DataFrame() - - grouped = df.groupby('SYMBOL', observed=True) - top_movers = [] - for _, row in all_latest.iterrows(): - symbol = row['SYMBOL'] - if symbol in self.etf_symbols: - continue - if "ETF" in str(row['SECURITY']).upper(): - continue - avg_volume = row.get('AVG_VOLUME', 0) or 0 - if avg_volume < volume_cutoff: - continue - if duration == "1D": - vc = row.get('VOLUME_CHANGES') or [] - volume_pct_change = vc[4] if len(vc) >= 5 else 0.0 - else: - try: - hist = grouped.get_group(symbol) - except KeyError: - hist = pd.DataFrame() - volume_pct_change = self._calculate_average_volume_change( - hist, days_back - ) - top_movers.append({ - "SYMBOL": symbol, - "VOLUME_CHANGE_PCT": volume_pct_change, - "SECURITY": row['SECURITY'], - "AVG_VOLUME": avg_volume, - }) - movers_df = pd.DataFrame(top_movers) - if movers_df.empty: - return pd.DataFrame() - return movers_df.sort_values( - by="VOLUME_CHANGE_PCT", ascending=False - ).head(limit) - - # ------------------------------------------------------------------ - # Moneycontrol scores - # ------------------------------------------------------------------ - - @elapsed_time_logger("Loading and sorting Moneycontrol data by score...") - def get_top_mc_score_movers(self, limit: int = ANALYSIS_LIMIT) -> pd.DataFrame: - """ - Read nse_dashboard_mclink.json and return the top ``limit`` symbols - ranked by MC_SCORE descending. - """ - try: - if not self.paths.mc_link_file.exists(): - logging.warning( - f"Moneycontrol link file not found: {self.paths.mc_link_file}." - ) - return pd.DataFrame() - with open(self.paths.mc_link_file, 'r') as f: - mc_data = json.load(f) - processed = [ - {"SYMBOL": symbol, "MC_SCORE": d['MC_SCORE'], - "SECURITY": d.get('MC_COMPANY_NAME', '')} - for symbol, d in mc_data.items() if 'MC_SCORE' in d - ] - if not processed: - logging.warning("No valid MC_SCORE data found.") - return pd.DataFrame() - mc_df = pd.DataFrame(processed) - return mc_df.sort_values(by="MC_SCORE", ascending=False).head(limit) - except (json.JSONDecodeError, FileNotFoundError) as e: - logging.error(f"Error reading Moneycontrol data: {e}") - raise RuntimeError(f"Could not read Moneycontrol data: {e}") from e - except Exception as e: - logging.error(f"Unexpected error getting top MC score movers: {e}") - raise RuntimeError(f"Failed to analyze top MC scores: {e}") from e - - # ------------------------------------------------------------------ - # Moneycontrol link store (shared by all stock cards; AUDIT FIX P-7: - # v4.5 re-read the JSON from disk once per card per refresh) - # ------------------------------------------------------------------ - - def load_mc_link_data(self) -> dict: - """Load nse_dashboard_mclink.json; create an empty file if missing.""" - try: - p = Path(self.paths.mc_link_file) - if p.exists(): - with open(p, 'r') as f: - return json.load(f) - with open(p, 'w') as f: - json.dump({}, f) - return {} - except Exception as e: - logging.error( - f"Error loading Moneycontrol links from {self.paths.mc_link_file}: {e}" - ) - return {} - - def save_mc_link_data(self, data: dict) -> None: - """Write the updated Moneycontrol links dict back to JSON.""" - try: - with open(self.paths.mc_link_file, 'w') as f: - json.dump(data, f, indent=4) - print(f"[ACTIVITY] Moneycontrol links config file updated: {self.paths.mc_link_file.name}", flush=True) - except Exception as e: - logging.error( - f"Error saving Moneycontrol links to {self.paths.mc_link_file}: {e}" - ) - - def set_mc_link(self, symbol: str, security: str, link: str) -> dict: - """ - Register/replace the Moneycontrol URL for a symbol (parity with the - v4.5 URL-prompt flow, including MC_COMPANY_CODE extraction). - """ - input_link = (link or "").strip() - if not input_link: - raise ValueError("Please enter the Moneycontrol URL.") - url_parts = input_link.strip('/').split('/') - input_mc_code = next((p for p in reversed(url_parts) if p), None) - if not input_mc_code: - raise ValueError("Could not extract MC Company Code from the URL.") - mc_links = self.load_mc_link_data() - if symbol in mc_links: - mc_links[symbol]["MC_LINK"] = input_link - mc_links[symbol]["MC_COMPANY_CODE"] = input_mc_code - mc_links[symbol]["MC_COMPANY_NAME"] = security - else: - mc_links[symbol] = { - "MC_ATH": 0.0, "MC_ATL": 0.0, - "MC_COMPANY_CODE": input_mc_code, - "MC_COMPANY_NAME": security, - "MC_FV": 0.0, "MC_LINK": input_link, - "MC_MCAP": 0.0, "MC_SCORE": 0.0, "MC_STATUS": False, - } - self.save_mc_link_data(mc_links) - return mc_links[symbol] diff --git a/nsedash/core/token_state.json b/nsedash/core/token_state.json deleted file mode 100644 index 5f68e8ad98789ab019edc20e683a74a051155d3e..0000000000000000000000000000000000000000 --- a/nsedash/core/token_state.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "prompt_tokens": 186398, - "completion_tokens": 72204, - "total_tokens": 258602, - "per_model": { - "nvidia/llama-3.3-nemotron-super-49b-v1": { - "prompt_tokens": 5935, - "completion_tokens": 1999, - "total_tokens": 7934 - }, - "nvidia/nemotron-3-ultra-550b-a55b": { - "prompt_tokens": 8822, - "completion_tokens": 8192, - "total_tokens": 17014 - }, - "nvidia/llama-3.3-nemotron-super-49b-v1.5": { - "prompt_tokens": 15581, - "completion_tokens": 11471, - "total_tokens": 27052 - }, - "meta/llama-3.1-70b-instruct": { - "prompt_tokens": 17996, - "completion_tokens": 5482, - "total_tokens": 23478 - }, - "thinkingmachines/inkling": { - "prompt_tokens": 3426, - "completion_tokens": 4096, - "total_tokens": 7522 - }, - "google/diffusiongemma-26b-a4b-it": { - "prompt_tokens": 134638, - "completion_tokens": 40964, - "total_tokens": 175602 - } - } -} \ No newline at end of file diff --git a/nsedash/core/user_auth.py b/nsedash/core/user_auth.py deleted file mode 100644 index a1610408586814a12a8fef2b1ec1133028e89ec6..0000000000000000000000000000000000000000 --- a/nsedash/core/user_auth.py +++ /dev/null @@ -1,835 +0,0 @@ -""" -nsedash.core.user_auth -======================= -User authentication and session management for nsedash — ported from -the mlbench ``security/app_security.py`` module and adapted for the -nsedash Flask API. - -Authentication flow --------------------- -1. ``POST /api/auth/login`` — userid + password → bcrypt verify → TOTP required -2. ``POST /api/auth/totp`` — userid + TOTP code → issues a session token -3. All subsequent API calls send ``X-Session-Token`` header. -4. ``SessionManager.validate_session()`` resolves the token → userid. -5. ``_svc(userid)`` creates / retrieves the per-user ``DashboardService``. - -Seed users (from mlbench users.db) ------------------------------------- -The file ``security/users.db`` is created on first run using the same -schema as mlbench. The initial user set is: - admin, sam, mainak, tunir, rama - -Schema of users.db --------------------- -{ - "users": { - "": { - "userid": str, - "password_hash": str | null, - "email": str | null, - "totp_secret": str | null, - "totp_secret_encrypted": str | null, - "is_admin": bool, - "created_at": ISO8601, - "last_login": ISO8601 | null, - "login_count": int, - "active": bool, - "login_status": "online" | "offline" - } - } -} -""" - -from __future__ import annotations - -import base64 -import json -import logging -import os -import secrets -from datetime import datetime, timedelta -from pathlib import Path -from typing import Any, Dict, Optional, Tuple - -log = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# File paths (relative to the app base_dir — set via UserAuth.init()) -# --------------------------------------------------------------------------- - -_SECURITY_DIR: Path | None = None -_USERS_DB_FILE: Path | None = None -_ENCRYPTION_KEY_FILE: Path | None = None -_SESSION_TOKENS_FILE: Path | None = None -_HF_SYNC = None - -import re - -SESSION_EXPIRATION_HOURS = 24 -TOTP_WINDOW = 1 # ±1 30-second window - - -def validate_password_policy(password: str) -> Tuple[bool, str]: - """ - Validates password against policy: - - Minimum length: 8 characters - - At least 1 uppercase letter (A-Z) - - At least 1 lowercase letter (a-z) - - At least 1 digit (0-9) - - At least 1 special character (!@#$%^&*()_+-=[]{}|;:,.<>?) - """ - if not password or len(password) < 8: - return False, "Password must be at least 8 characters long." - if not re.search(r"[A-Z]", password): - return False, "Password must contain at least 1 uppercase letter (A-Z)." - if not re.search(r"[a-z]", password): - return False, "Password must contain at least 1 lowercase letter (a-z)." - if not re.search(r"[0-9]", password): - return False, "Password must contain at least 1 number (0-9)." - if not re.search(r"[^A-Za-z0-9]", password): - return False, "Password must contain at least 1 special character (e.g. !@#$%^&*)." - return True, "OK" - - -def init(base_dir: Path | str, hf_sync=None) -> None: - """ - Initialise paths. Must be called once from ``create_app()`` before - any auth function is used. - """ - global _SECURITY_DIR, _USERS_DB_FILE, _ENCRYPTION_KEY_FILE, _SESSION_TOKENS_FILE, _HF_SYNC - base_dir = Path(base_dir) - _SECURITY_DIR = base_dir / "processed" / "security" - _SECURITY_DIR.mkdir(parents=True, exist_ok=True) - _USERS_DB_FILE = _SECURITY_DIR / "users.db" - _ENCRYPTION_KEY_FILE = _SECURITY_DIR / "encryption_key.key" - if not _ENCRYPTION_KEY_FILE.exists() and (_SECURITY_DIR / ".encryption_key").exists(): - _ENCRYPTION_KEY_FILE = _SECURITY_DIR / ".encryption_key" - _SESSION_TOKENS_FILE = _SECURITY_DIR / ".session_tokens.json" - _HF_SYNC = hf_sync - - # Initialise encrypted SQLite database (userdata.db) - from .userdata_db import init as init_userdata_db - init_userdata_db(base_dir, hf_sync=hf_sync) - - log.info("[Auth] Initialised security dir: %s (hf_sync linked: %s)", _SECURITY_DIR, hf_sync is not None) - - -def _check_init(): - if _USERS_DB_FILE is None: - raise RuntimeError("user_auth.init(base_dir) must be called first") - - -# --------------------------------------------------------------------------- -# Encryption (Fernet — same key format as mlbench) -# --------------------------------------------------------------------------- - -class EncryptionManager: - """Handles encryption/decryption of TOTP secrets.""" - - @staticmethod - def _get_or_create_key() -> bytes: - _check_init() - from cryptography.fernet import Fernet - if _ENCRYPTION_KEY_FILE.exists(): - return _ENCRYPTION_KEY_FILE.read_bytes() - key = Fernet.generate_key() - _ENCRYPTION_KEY_FILE.write_bytes(key) - log.info("[Auth] Encryption key created.") - if _HF_SYNC: - _HF_SYNC.push_file_async(_ENCRYPTION_KEY_FILE) - return key - - @staticmethod - def encrypt(plaintext: str) -> str: - try: - from cryptography.fernet import Fernet - key = EncryptionManager._get_or_create_key() - ciphertext = Fernet(key).encrypt(plaintext.encode()) - return base64.b64encode(ciphertext).decode() - except Exception as exc: - log.error("[Auth] Encryption error: %s", exc) - return plaintext - - @staticmethod - def decrypt(ciphertext: str) -> str | None: - try: - from cryptography.fernet import Fernet - key = EncryptionManager._get_or_create_key() - raw = base64.b64decode(ciphertext.encode()) - return Fernet(key).decrypt(raw).decode() - except Exception as exc: - log.error("[Auth] Decryption error: %s", exc) - return None - - -# --------------------------------------------------------------------------- -# User Database -# --------------------------------------------------------------------------- - -# Seed users ported from mlbench users.db. -# Passwords are null (set on first login via TOTP setup flow). -# Only the admin entry has the real bcrypt hash from mlbench. -_SEED_USERS: Dict[str, Any] = { - "admin": { - "userid": "admin", - "password_hash": "pbkdf2:sha256:1000000$3QTFDU2NQmIiNJCs$8e2fa3fd5daae1c5270426e2c2080c600a11fbeabee071f115ec5f584098571b", - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": True, - "created_at": "2026-02-28T16:05:18.324809", - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - }, - "sam": { - "userid": "sam", - "password_hash": "pbkdf2:sha256:1000000$zAgOwq9jWdoLseRy$bec16031d940efcdb7b83b5d34735b3a11bcf4ce613ff7c5e0252c01d98cbdd5", - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": False, - "created_at": "2026-03-26T00:31:49.856999", - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - }, - "mainak": { - "userid": "mainak", - "password_hash": None, - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": False, - "created_at": "2026-02-28T20:58:29.175636", - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - }, - "tunir": { - "userid": "tunir", - "password_hash": None, - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": False, - "created_at": "2026-02-28T20:58:42.323454", - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - }, - "rama": { - "userid": "rama", - "password_hash": None, - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": False, - "created_at": "2026-02-28T23:18:58.881545", - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - }, -} - - -class UserDatabase: - """JSON-backed user database — same schema as mlbench users.db.""" - - @staticmethod - def load() -> Dict[str, Any]: - _check_init() - if not _USERS_DB_FILE.exists(): - return UserDatabase._create_defaults() - try: - return json.loads(_USERS_DB_FILE.read_text()) - except Exception as exc: - log.error("[Auth] Failed to load users.db: %s", exc) - return UserDatabase._create_defaults() - - @staticmethod - def _create_defaults() -> Dict[str, Any]: - data = {"users": _SEED_USERS.copy()} - UserDatabase.save(data) - log.warning("[Auth] Created default users.db from seed (admin, sam, mainak, tunir, rama)") - return data - - @staticmethod - def save(data: Dict[str, Any]) -> bool: - _check_init() - try: - _USERS_DB_FILE.write_text(json.dumps(data, indent=2)) - if _HF_SYNC: - _HF_SYNC.push_file_async(_USERS_DB_FILE) - return True - except Exception as exc: - log.error("[Auth] Failed to save users.db: %s", exc) - return False - - @staticmethod - def get_user(userid: str) -> Optional[Dict[str, Any]]: - u = UserDatabase.load().get("users", {}).get(userid) - if u and userid == "admin": - u["is_admin"] = True - return u - - @staticmethod - def update_user(userid: str, updates: Dict[str, Any]) -> bool: - data = UserDatabase.load() - if userid not in data.get("users", {}): - return False - data["users"][userid].update(updates) - if userid == "admin": - data["users"][userid]["is_admin"] = True - return UserDatabase.save(data) - - @staticmethod - def create_user(userid: str, is_admin: bool = False) -> bool: - if UserDatabase.get_user(userid): - return False - data = UserDatabase.load() - data["users"][userid] = { - "userid": userid, - "password_hash": None, - "email": None, - "totp_secret": None, - "totp_secret_encrypted": None, - "is_admin": True if userid == "admin" else is_admin, - "created_at": datetime.now().isoformat(), - "last_login": None, - "login_count": 0, - "active": True, - "login_status": "offline", - } - return UserDatabase.save(data) - - @staticmethod - def delete_user(userid: str) -> bool: - if userid == "admin": - return False - data = UserDatabase.load() - if userid in data.get("users", {}): - del data["users"][userid] - return UserDatabase.save(data) - return False - - @staticmethod - def reset_totp(userid: str) -> bool: - data = UserDatabase.load() - if userid in data.get("users", {}): - data["users"][userid]["password_hash"] = None - data["users"][userid]["totp_secret"] = None - data["users"][userid]["totp_secret_encrypted"] = None - if userid == "admin": - data["users"][userid]["is_admin"] = True - return UserDatabase.save(data) - return False - - @staticmethod - def get_all_users() -> Dict[str, Any]: - users = UserDatabase.load().get("users", {}) - return { - uid: { - "userid": uid, - "is_admin": True if uid == "admin" else u.get("is_admin", False), - "active": u.get("active"), - "last_login": u.get("last_login"), - "login_count": u.get("login_count", 0), - "login_status": u.get("login_status", "offline"), - } - for uid, u in users.items() - } - - -# --------------------------------------------------------------------------- -# Authentication -# --------------------------------------------------------------------------- - -class AuthenticationService: - """Userid + bcrypt password + TOTP — same flow as mlbench.""" - - @staticmethod - def login(userid: str, password: str) -> Tuple[bool, str, bool]: - """ - Verify password. - - Returns - ------- - (success, message, totp_required) - """ - user = UserDatabase.get_user(userid) - if not user: - ActivityLogger.log_activity(userid, "login", "failed", "User not found") - return False, "User not found.", False - if not user.get("active", True): - ActivityLogger.log_activity(userid, "login", "failed", "Account inactive") - return False, "Account inactive.", False - - from werkzeug.security import generate_password_hash, check_password_hash - - if user.get("password_hash") is None: - # First login — set password and generate TOTP secret - pw_hash = generate_password_hash(password, method="pbkdf2:sha256") - import pyotp - totp_secret = pyotp.random_base32() - enc = EncryptionManager.encrypt(totp_secret) - UserDatabase.update_user(userid, { - "password_hash": pw_hash, - "totp_secret": totp_secret, - "totp_secret_encrypted": enc, - }) - log.info("[Auth] First login for '%s' — TOTP provisioned.", userid) - ActivityLogger.log_activity(userid, "login", "first_totp_setup", "First TOTP setup triggered") - return True, "first_login", True # caller should show QR - - if not check_password_hash(user["password_hash"], password): - ActivityLogger.log_activity(userid, "login", "failed", "Invalid password") - return False, "Invalid password.", False - - # Validate password policy compliance - valid_pw, _ = validate_password_policy(password) - if not valid_pw or user.get("force_password_change"): - UserDatabase.update_user(userid, {"force_password_change": True}) - - # If TOTP not yet set up (e.g. admin copied over without secret) - if not user.get("totp_secret"): - try: - import pyotp - except ImportError: - ActivityLogger.log_activity(userid, "login", "error", "pyotp missing") - return False, "Server error: pyotp library is missing", False - - totp_secret = pyotp.random_base32() - enc = EncryptionManager.encrypt(totp_secret) - if not UserDatabase.update_user(userid, { - "totp_secret": totp_secret, - "totp_secret_encrypted": enc, - }): - ActivityLogger.log_activity(userid, "login", "error", "failed_to_save_totp") - return False, "Server error: Failed to save TOTP data", False - - ActivityLogger.log_activity(userid, "login", "totp_setup", "TOTP setup triggered") - return True, "totp_setup", True - - ActivityLogger.log_activity(userid, "login", "password_verified", "Password verified successfully") - return True, "ok", True - - -def change_user_password(userid: str, current_password: str | None, new_password: str, confirm_password: str, is_admin_override: bool = False) -> Tuple[bool, str]: - """ - Change user password enforcing: - 1. Reconfirmation entry (new_password == confirm_password) - 2. Password policy (min 8 chars, 1 upper, 1 lower, 1 digit, 1 special char) - 3. Current password verification (unless admin override) - """ - if new_password != confirm_password: - return False, "New password and confirm password do not match." - - valid, msg = validate_password_policy(new_password) - if not valid: - return False, msg - - user = UserDatabase.get_user(userid) - if not user: - return False, "User not found." - - from werkzeug.security import generate_password_hash, check_password_hash - - if not is_admin_override: - if not current_password or not user.get("password_hash"): - return False, "Current password is required." - if not check_password_hash(user["password_hash"], current_password): - return False, "Current password is incorrect." - - new_hash = generate_password_hash(new_password, method="pbkdf2:sha256") - UserDatabase.update_user(userid, { - "password_hash": new_hash, - "force_password_change": False - }) - ActivityLogger.log_activity(userid, "change_password", "success", "Password updated successfully") - return True, "Password changed successfully." - - @staticmethod - def get_totp_qr(userid: str) -> str | None: - """Return base64 PNG data-URI for TOTP QR code.""" - user = UserDatabase.get_user(userid) - if not user: - return None - secret = user.get("totp_secret") - if not secret and user.get("totp_secret_encrypted"): - secret = EncryptionManager.decrypt(user["totp_secret_encrypted"]) - if not secret: - return None - try: - import pyotp - import qrcode - except ImportError as exc: - log.error("[Auth] Missing dependencies for QR generation: %s", exc) - return None - - try: - from io import BytesIO - uri = pyotp.TOTP(secret).provisioning_uri( - name=userid, issuer_name="NSE Dashboard" - ) - qr = qrcode.make(uri) - buf = BytesIO() - qr.save(buf, format="PNG") - return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() - except Exception as exc: - log.error("[Auth] QR generation failed for '%s': %s", userid, exc) - return None - - @staticmethod - def verify_totp(userid: str, code: str) -> Tuple[bool, str]: - """Verify TOTP code and — on success — issue a session token.""" - user = UserDatabase.get_user(userid) - if not user: - ActivityLogger.log_activity(userid, "totp_verification", "failed", "User not found") - return False, "User not found." - secret = user.get("totp_secret") - if not secret and user.get("totp_secret_encrypted"): - secret = EncryptionManager.decrypt(user["totp_secret_encrypted"]) - if not secret: - ActivityLogger.log_activity(userid, "totp_verification", "failed", "TOTP not configured") - return False, "TOTP not configured." - try: - import pyotp - totp = pyotp.TOTP(secret) - if not totp.verify(code, valid_window=TOTP_WINDOW): - ActivityLogger.log_activity(userid, "totp_verification", "failed", "Invalid TOTP code") - return False, "Invalid TOTP code." - except Exception as exc: - ActivityLogger.log_activity(userid, "totp_verification", "error", f"TOTP error: {exc}") - return False, f"TOTP error: {exc}" - - # Success — update login metadata - cnt = user.get("login_count", 0) - UserDatabase.update_user(userid, { - "last_login": datetime.now().isoformat(), - "login_count": cnt + 1, - "login_status": "online", - }) - token = SessionManager.create_session(userid) - ActivityLogger.log_activity(userid, "login", "success", "TOTP verified, logged in") - return True, token - - - -# --------------------------------------------------------------------------- -# Session Manager -# --------------------------------------------------------------------------- - -class SessionManager: - """File-backed session tokens — same approach as mlbench SessionManager.""" - - @staticmethod - def _load() -> Dict[str, Any]: - _check_init() - if not _SESSION_TOKENS_FILE or not _SESSION_TOKENS_FILE.exists(): - return {} - try: - return json.loads(_SESSION_TOKENS_FILE.read_text()) - except Exception: - return {} - - @staticmethod - def _save(sessions: Dict[str, Any]) -> None: - _check_init() - try: - _SESSION_TOKENS_FILE.write_text(json.dumps(sessions, indent=2)) - if _HF_SYNC: - _HF_SYNC.push_file_async(_SESSION_TOKENS_FILE) - except Exception as exc: - log.error("[Auth] Failed to save sessions: %s", exc) - - @staticmethod - def create_session(userid: str) -> str: - token = secrets.token_urlsafe(32) - sessions = SessionManager._load() - sessions[token] = { - "userid": userid, - "created_at": datetime.now().isoformat(), - "last_activity": datetime.now().isoformat(), - } - SessionManager._save(sessions) - log.info("[Auth] Session created for '%s'", userid) - return token - - @staticmethod - def validate_session(token: str) -> Tuple[bool, Optional[str]]: - """Returns (valid, userid).""" - if not token: - return False, None - sessions = SessionManager._load() - entry = sessions.get(token) - if not entry: - return False, None - created = datetime.fromisoformat(entry["created_at"]) - if (datetime.now() - created).total_seconds() > SESSION_EXPIRATION_HOURS * 3600: - del sessions[token] - SessionManager._save(sessions) - return False, None - # Touch last_activity - entry["last_activity"] = datetime.now().isoformat() - SessionManager._save(sessions) - return True, entry["userid"] - - @staticmethod - def destroy_session(token: str) -> bool: - sessions = SessionManager._load() - if token in sessions: - userid = sessions[token].get("userid", "?") - del sessions[token] - SessionManager._save(sessions) - # Mark user offline - UserDatabase.update_user(userid, {"login_status": "offline"}) - log.info("[Auth] Session destroyed for '%s'", userid) - ActivityLogger.log_activity(userid, "logout", "success", "Logged out successfully") - return True - return False - - @staticmethod - def get_userid_sessions(userid: str) -> list[str]: - """Return all active token strings for a given userid.""" - sessions = SessionManager._load() - return [tok for tok, d in sessions.items() if d.get("userid") == userid] - - @staticmethod - def clear_sessions_for_user(userid: str) -> int: - """Deletes all active sessions for a specific user, forcing them to log in again.""" - sessions = SessionManager._load() - tokens_to_delete = [tok for tok, d in sessions.items() if d.get("userid") == userid] - for tok in tokens_to_delete: - del sessions[tok] - if tokens_to_delete: - SessionManager._save(sessions) - log.info("[Auth] Cleared %d sessions for '%s'", len(tokens_to_delete), userid) - - # Always force offline status to resolve "ghost" logins - UserDatabase.update_user(userid, {"login_status": "offline"}) - return len(tokens_to_delete) - - @staticmethod - def cleanup_expired() -> int: - sessions = SessionManager._load() - expired = [ - tok for tok, d in sessions.items() - if (datetime.now() - datetime.fromisoformat(d["created_at"])).total_seconds() - > SESSION_EXPIRATION_HOURS * 3600 - ] - for tok in expired: - del sessions[tok] - if expired: - SessionManager._save(sessions) - return len(expired) - - @staticmethod - def invalidate_all_sessions_on_startup() -> int: - """ - Invalidates all active session tokens and updates all users as offline - on application startup to enforce fresh per-session logins. - """ - sessions = SessionManager._load() - count = len(sessions) - SessionManager._save({}) - - # Update all users to offline status and ensure admin preserves is_admin - data = UserDatabase.load() - users = data.get("users", {}) - updated = False - for userid, user_info in users.items(): - if user_info.get("login_status") != "offline": - user_info["login_status"] = "offline" - updated = True - if userid == "admin" and not user_info.get("is_admin"): - user_info["is_admin"] = True - updated = True - if updated: - UserDatabase.save(data) - - log.info("[Auth Startup] Invalidated %d active sessions. Updated users to offline status.", count) - return count - - -def invalidate_all_sessions_on_startup() -> int: - return SessionManager.invalidate_all_sessions_on_startup() - - - -# --------------------------------------------------------------------------- -# Activity Logger -# --------------------------------------------------------------------------- - -class ActivityLogger: - """Logs user activities for audit trail (parity with mlbench).""" - - @staticmethod - def log_activity(userid: str, action: str, status: str, details: str = "", duration_seconds: Optional[int] = None): - _check_init() - try: - log_file = _SECURITY_DIR / "activity_log.jsonl" - log_entry = { - 'timestamp': datetime.now().isoformat(), - 'userid': userid, - 'action': action, - 'status': status, - 'details': details - } - if duration_seconds is not None: - log_entry['duration_seconds'] = duration_seconds - - with open(log_file, 'a') as f: - f.write(json.dumps(log_entry) + '\n') - log.info("[Activity] Logged: %s - %s - %s", userid, action, status) - if _HF_SYNC: - _HF_SYNC.push_file_async(log_file) - except Exception as e: - log.error("[Activity] Error logging activity: %s", e) - - @staticmethod - def get_activity_logs(userid: Optional[str] = None, limit: int = 1000) -> list: - _check_init() - try: - log_file = _SECURITY_DIR / "activity_log.jsonl" - logs = [] - if log_file.exists(): - with open(log_file, 'r') as f: - for line in f: - if line.strip(): - log_entry = json.loads(line) - if userid is None or log_entry.get('userid') == userid: - logs.append(log_entry) - return logs[-limit:] - except Exception as e: - log.error("[Activity] Error reading logs: %s", e) - return [] - - -# --------------------------------------------------------------------------- -# Encrypted User Data Sync in users.db -# --------------------------------------------------------------------------- - -def backup_session_to_db(userid: str, base_dir: Path | str) -> bool: - """Read the session files (config, state, watchlists, portfolio) and save encrypted to users.db.""" - _check_init() - if userid == "admin": - return True # Admin uses global shared files, not isolated/encrypted - - from ..config import Paths - paths = Paths(base_dir=base_dir, session_token=userid) - - # Read files - try: - state_data = None - if paths.state_file.exists(): - try: - state_data = json.loads(paths.state_file.read_text()) - except Exception: - pass - - config_data = None - if paths.config_file.exists(): - try: - config_data = json.loads(paths.config_file.read_text()) - except Exception: - pass - - portfolio_data = None - if paths.portfolio_file.exists(): - try: - # Read binary Parquet file and encode to base64 string - binary_data = paths.portfolio_file.read_bytes() - portfolio_data = base64.b64encode(binary_data).decode('utf-8') - except Exception as exc: - log.error("[Auth] Failed to backup portfolio Parquet: %s", exc) - - watchlists_data = {} - if paths.watchlist_dir.exists(): - for item in paths.watchlist_dir.iterdir(): - if item.is_file() and item.suffix == ".json": - try: - watchlists_data[item.name] = json.loads(item.read_text()) - except Exception: - pass - - user_data = { - "state": state_data, - "config": config_data, - "portfolio": portfolio_data, - "watchlists": watchlists_data, - } - - # Encrypt & save - plain = json.dumps(user_data) - enc = EncryptionManager.encrypt(plain) - UserDatabase.update_user(userid, {"user_data_encrypted": enc}) - return True - except Exception as exc: - log.error("[Auth] Failed to backup session files for user %s: %s", userid, exc) - return False - - -def restore_session_from_db(userid: str, base_dir: Path | str) -> bool: - """Load and decrypt the user's data from users.db and write to the isolated session files.""" - _check_init() - if userid == "admin": - return False # Admin uses global shared files, no restore needed - - user = UserDatabase.get_user(userid) - if not user: - return False - - enc = user.get("user_data_encrypted") - if not enc: - return False # No data saved yet - - try: - plain = EncryptionManager.decrypt(enc) - if not plain: - return False - - user_data = json.loads(plain) - from ..config import Paths - paths = Paths(base_dir=base_dir, session_token=userid) - paths.ensure_dirs() - - # Restore state - if user_data.get("state"): - paths.state_file.write_text(json.dumps(user_data["state"], indent=2)) - - # Restore config - if user_data.get("config"): - paths.config_file.write_text(json.dumps(user_data["config"], indent=2)) - - # Restore portfolio - if user_data.get("portfolio"): - try: - # Decode base64 to binary bytes and save - port_raw = user_data["portfolio"] - try: - binary_data = base64.b64decode(port_raw.encode('utf-8')) - paths.portfolio_file.write_bytes(binary_data) - except Exception: - # Fallback to legacy plain text write if it was not base64 - paths.portfolio_file.write_text(port_raw) - except Exception as exc: - log.error("[Auth] Failed to restore portfolio Parquet: %s", exc) - - # Restore watchlists - if user_data.get("watchlists"): - for filename, wl_content in user_data["watchlists"].items(): - file_path = paths.watchlist_dir / filename - file_path.write_text(json.dumps(wl_content, indent=2)) - - log.info("[Auth] Restored session files from users.db for user %s", userid) - return True - except Exception as exc: - log.error("[Auth] Failed to restore session files for user %s: %s", userid, exc) - return False - diff --git a/nsedash/core/userdata_db.py b/nsedash/core/userdata_db.py deleted file mode 100644 index 8a9509194dc5121240f3d34e572f2e9d46d5f6ef..0000000000000000000000000000000000000000 --- a/nsedash/core/userdata_db.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -nsedash.core.userdata_db -======================== -Encrypted SQLite database module (userdata.db) for per-user private Watchlists 1–5. -Uses Fernet AES-256 encryption via EncryptionManager to encrypt watchlist contents. -""" - -from __future__ import annotations - -import json -import logging -import sqlite3 -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, List, Optional - -log = logging.getLogger(__name__) - -_USERDATA_DB_FILE: Path | None = None -_SECURITY_DIR: Path | None = None -_HF_SYNC = None - - -def init(base_dir: Path | str, hf_sync=None) -> None: - """Initialise SQLite database paths for userdata.db.""" - global _USERDATA_DB_FILE, _SECURITY_DIR, _HF_SYNC - base_dir = Path(base_dir) - _SECURITY_DIR = base_dir / "processed" / "security" - _SECURITY_DIR.mkdir(parents=True, exist_ok=True) - _USERDATA_DB_FILE = _SECURITY_DIR / "userdata.db" - _HF_SYNC = hf_sync - - UserDataDB.ensure_tables() - UserDataDB.perform_initial_reset_and_migration(base_dir) - log.info("[UserDataDB] Initialised encrypted SQLite database: %s", _USERDATA_DB_FILE) - - -class UserDataDB: - """Encrypted SQLite database manager for per-user private watchlists (slots 1..5).""" - - @staticmethod - def _get_connection() -> sqlite3.Connection: - if _USERDATA_DB_FILE is None: - raise RuntimeError("userdata_db.init(base_dir) must be called first") - conn = sqlite3.connect(_USERDATA_DB_FILE) - conn.row_factory = sqlite3.Row - return conn - - @staticmethod - def ensure_tables() -> None: - """Create user_watchlists table if it does not exist.""" - conn = UserDataDB._get_connection() - try: - with conn: - conn.execute(""" - CREATE TABLE IF NOT EXISTS user_watchlists ( - userid TEXT NOT NULL, - slot INTEGER NOT NULL, - name TEXT NOT NULL, - encrypted_data TEXT NOT NULL, - updated_at TEXT NOT NULL, - PRIMARY KEY (userid, slot) - ); - """) - log.info("[UserDataDB] Schema verified for user_watchlists table in userdata.db.") - finally: - conn.close() - - @staticmethod - def save_watchlist(userid: str, slot: int, name: str, symbols: List[str]) -> bool: - """Encrypt and save a user's private watchlist (slots 1..5).""" - if slot < 1 or slot > 5: - log.warning("[UserDataDB] Slot %d is outside private range 1..5 for user %s", slot, userid) - return False - - from .user_auth import EncryptionManager - payload = { - "name": name or f"Watchlist {slot}", - "symbols": list(symbols or []) - } - plain_json = json.dumps(payload) - encrypted_data = EncryptionManager.encrypt(plain_json) - now_iso = datetime.now().isoformat() - - conn = UserDataDB._get_connection() - try: - with conn: - conn.execute(""" - INSERT INTO user_watchlists (userid, slot, name, encrypted_data, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(userid, slot) DO UPDATE SET - name = excluded.name, - encrypted_data = excluded.encrypted_data, - updated_at = excluded.updated_at; - """, (userid, slot, name or f"Watchlist {slot}", encrypted_data, now_iso)) - log.info("[UserDataDB] Saved encrypted watchlist slot %d for user %s (%d symbols)", slot, userid, len(symbols)) - if _HF_SYNC and _USERDATA_DB_FILE: - _HF_SYNC.push_file_async(_USERDATA_DB_FILE) - return True - except Exception as exc: - log.error("[UserDataDB] Error saving watchlist for user %s slot %d: %s", userid, slot, exc, exc_info=True) - return False - finally: - conn.close() - - @staticmethod - def get_watchlist(userid: str, slot: int) -> Optional[Dict[str, Any]]: - """Fetch and decrypt a user's private watchlist (slots 1..5).""" - if slot < 1 or slot > 5: - return None - - from .user_auth import EncryptionManager - conn = UserDataDB._get_connection() - try: - cur = conn.execute( - "SELECT name, encrypted_data FROM user_watchlists WHERE userid = ? AND slot = ?", - (userid, slot) - ) - row = cur.fetchone() - if not row: - return None - - name = row["name"] - enc_data = row["encrypted_data"] - plain_json = EncryptionManager.decrypt(enc_data) - if not plain_json: - log.warning("[UserDataDB] Decryption returned empty string for user %s slot %d", userid, slot) - return {"name": name, "symbols": []} - - data = json.loads(plain_json) - return { - "name": data.get("name", name), - "symbols": data.get("symbols", []) - } - except Exception as exc: - log.error("[UserDataDB] Error fetching watchlist for user %s slot %d: %s", userid, slot, exc, exc_info=True) - return None - finally: - conn.close() - - @staticmethod - def get_all_user_watchlists(userid: str) -> Dict[int, Dict[str, Any]]: - """Return all watchlists 1..5 for userid.""" - result = {} - for slot in range(1, 6): - wl = UserDataDB.get_watchlist(userid, slot) - if wl: - result[slot] = wl - else: - result[slot] = {"name": f"Watchlist {slot}", "symbols": []} - return result - - @staticmethod - def reset_watchlists_except(allowed_users: List[str] = None) -> int: - """ - Resets Watchlists 1–5 to blank (empty symbols list) for all users - EXCEPT those listed in allowed_users (default: ['admin', 'sam']). - """ - allowed = set(allowed_users or ["admin", "sam"]) - from .user_auth import UserDatabase - all_users = UserDatabase.get_all_users() - - reset_count = 0 - for uid in all_users.keys(): - if uid not in allowed: - for slot in range(1, 6): - UserDataDB.save_watchlist(uid, slot, f"Watchlist {slot}", []) - reset_count += 1 - return reset_count - - @staticmethod - def perform_initial_reset_and_migration(base_dir: Path | str) -> None: - """ - Migrates existing Watchlists 1..5 for admin and sam into userdata.db (if not present), - and resets Watchlists 1..5 to blank for all other users. - """ - from .user_auth import UserDatabase - all_users = UserDatabase.get_all_users() - base_dir = Path(base_dir) - wl_dir = base_dir / "processed" / "watchlists" - - for uid in all_users.keys(): - if uid in ("admin", "sam"): - # Migrate existing JSON files for admin / sam if slot not yet populated in userdata.db - user_wl_dir = wl_dir / f"session_{uid}" if uid != "admin" else wl_dir - for slot in range(1, 6): - existing_wl = UserDataDB.get_watchlist(uid, slot) - if existing_wl is None: - json_file = user_wl_dir / f"nse_dashboard_watchlist{slot}.json" - name = f"Watchlist {slot}" - symbols = [] - if json_file.exists(): - try: - data = json.loads(json_file.read_text()) - name = data.get("name", name) - symbols_raw = data.get("symbols", []) - symbols = [s.get("SYMBOL") if isinstance(s, dict) else str(s) for s in symbols_raw] - except Exception: - pass - UserDataDB.save_watchlist(uid, slot, name, symbols) - log.info("[UserDataDB] Migrated slot %d for '%s' to userdata.db (%d symbols)", slot, uid, len(symbols)) - else: - # Reset slots 1..5 to blank for non-exempt users - for slot in range(1, 6): - UserDataDB.save_watchlist(uid, slot, f"Watchlist {slot}", []) - log.info("[UserDataDB] Reset Watchlists 1–5 to blank for user '%s'", uid) diff --git a/nsedash/core/watchlists.py b/nsedash/core/watchlists.py deleted file mode 100644 index 9047a8d0a33c1b1ea7d47540d7e7674e73c6c9c2..0000000000000000000000000000000000000000 --- a/nsedash/core/watchlists.py +++ /dev/null @@ -1,262 +0,0 @@ -""" -nsedash.core.watchlists -======================= -Watchlist persistence: 20 named slots, the default dashboard state file, -analysis-cache watchlists, and application config. Functional parity with -the v4.5 methods ``_save_watchlist_json``, ``_load_symbols_from_json``, -``_load_analysis_json_if_recent``, ``load_watchlist_names``, -``save_dashboard_state``, ``load_config`` and ``save_config``. -""" - -from __future__ import annotations - -import json -import logging -from datetime import datetime -from pathlib import Path -from typing import TYPE_CHECKING - -import pandas as pd - -from ..config import WATCHLIST_SLOTS, Paths - -if TYPE_CHECKING: - from .hf_sync import HFDataSync, _NoOpSync - - -class WatchlistStore: - - def __init__(self, paths: Paths, hf_sync=None): - self.paths = paths - # hf_sync is a HFDataSync or _NoOpSync — push_file_async is always safe to call - self._hf = hf_sync - - def _backup(self) -> None: - if self.paths.session_token: - from . import user_auth - user_auth.backup_session_to_db(self.paths.session_token, self.paths.base_dir) - - # ------------------------------------------------------------------ - # Low-level JSON helpers (parity with StockData._* in v4.5) - # ------------------------------------------------------------------ - - def save_watchlist_json(self, df_to_save: pd.DataFrame, - file_path: Path, name: str) -> None: - """Save a DataFrame as a watchlist JSON with name and timestamp.""" - watchlist_data = { - "name": name, - "symbols": [ - {"SYMBOL": row['SYMBOL'], "Quantity": 0.0, "Price": 0.0} - for _, row in df_to_save.iterrows() - ], - "last_updated": datetime.now().isoformat(), - } - try: - file_path.parent.mkdir(parents=True, exist_ok=True) - with open(file_path, 'w') as f: - json.dump(watchlist_data, f, indent=4) - print(f"[ACTIVITY] Session watchlist JSON updated: {file_path.name}", flush=True) - logging.info(f"Saved '{name}' watchlist to {file_path}") - if self._hf: - self._hf.push_file_async(file_path) - self._backup() - except Exception as e: - logging.error(f"Error saving '{name}' watchlist to {file_path}: {e}") - - def load_symbols_from_json(self, file_path: Path, - current_data_tran_date=None - ) -> tuple[str, list[dict]]: - """ - Load symbol entries from a watchlist JSON file. - Returns (watchlist_name, [{'SYMBOL','Quantity','Price'}, ...]). - If ``current_data_tran_date`` is provided and the file's - ``last_updated`` predates it, returns a cache miss (name, []). - Returns ("", []) on error. - """ - file_path = Path(file_path) - import re - m = re.search(r"nse_dashboard_watchlist([1-5])\.json$", file_path.name) - if m: - slot = int(m.group(1)) - userid = self.paths.session_token or "admin" - from .userdata_db import UserDataDB - wl = UserDataDB.get_watchlist(userid, slot) - if wl is not None: - symbols = [{"SYMBOL": s, "Quantity": 0.0, "Price": 0.0} for s in wl.get("symbols", [])] - return wl.get("name", f"Watchlist {slot}"), symbols - - if not file_path.exists(): - logging.info(f"JSON file not found: {file_path}") - return "", [] - try: - with open(file_path, 'r') as f: - data = json.load(f) - watchlist_name = data.get('name', file_path.stem) - if current_data_tran_date: - last_updated_str = data.get('last_updated') - if not last_updated_str: - logging.warning( - f"'{file_path}' has no 'last_updated' field. Recalculating." - ) - return watchlist_name, [] - last_updated_date = datetime.fromisoformat(last_updated_str).date() - ref = current_data_tran_date - ref_date = ref.date() if hasattr(ref, 'date') else ref - if last_updated_date < ref_date: - logging.info(f"'{file_path}' is outdated. Recalculating.") - return watchlist_name, [] - symbols_data = [] - for entry in data.get('symbols', []): - if isinstance(entry, dict) and 'SYMBOL' in entry: - symbols_data.append({ - "SYMBOL": entry['SYMBOL'], - "Quantity": entry.get('Quantity', 0.0), - "Price": entry.get('Price', 0.0), - }) - elif isinstance(entry, str): - symbols_data.append( - {"SYMBOL": entry, "Quantity": 0.0, "Price": 0.0} - ) - else: - logging.warning(f"Skipping invalid entry in JSON file: {entry}") - return watchlist_name, symbols_data - except (json.JSONDecodeError, ValueError) as e: - logging.warning(f"Error parsing '{file_path}': {e}.") - return "", [] - except Exception as e: - logging.error(f"Unexpected error loading JSON from {file_path}: {e}.") - return "", [] - - def load_analysis_json_if_recent(self, file_path: Path, - current_data_tran_date) -> list[str] | None: - """Symbol list from an analysis-cache watchlist if fresh, else None.""" - _name, loaded = self.load_symbols_from_json(file_path, current_data_tran_date) - if loaded: - return [item['SYMBOL'] for item in loaded] - return None - - # ------------------------------------------------------------------ - # Named slots (1..20) - # ------------------------------------------------------------------ - - def load_watchlist_names(self) -> list[str]: - """Read watchlist names from slot files and UserDataDB for slots 1..5.""" - names = [f"Watchlist {i}" for i in range(1, WATCHLIST_SLOTS + 1)] - userid = self.paths.session_token or "admin" - from .userdata_db import UserDataDB - - for i in range(WATCHLIST_SLOTS): - slot = i + 1 - if 1 <= slot <= 5: - wl = UserDataDB.get_watchlist(userid, slot) - if wl and wl.get("name"): - names[i] = wl["name"] - continue - file_path = self.paths.watchlist_slot(slot) - try: - with open(file_path, "r") as f: - data = json.load(f) - names[i] = data.get("name", names[i]) - except (FileNotFoundError, json.JSONDecodeError): - continue - return names - - def save_slot(self, slot: int, name: str, symbols: list[dict]) -> Path: - """Write a full watchlist to a numbered slot and UserDataDB if slot 1..5.""" - if not (1 <= slot <= WATCHLIST_SLOTS): - raise ValueError(f"Watchlist slot must be 1..{WATCHLIST_SLOTS}") - file_path = self.paths.watchlist_slot(slot) - with open(file_path, "w") as f: - json.dump({ - "name": name, - "symbols": symbols, - "last_updated": datetime.now().isoformat(), - }, f, indent=2) - - if 1 <= slot <= 5: - userid = self.paths.session_token or "admin" - from .userdata_db import UserDataDB - sym_list = [s.get("SYMBOL") if isinstance(s, dict) else str(s) for s in symbols] - UserDataDB.save_watchlist(userid, slot, name, sym_list) - - print(f"[ACTIVITY] Session watchlist slot {slot} updated: {file_path.name}", flush=True) - if self._hf: - self._hf.push_file_async(file_path) - self._backup() - return file_path - - def add_symbols_to_slot(self, slot: int, symbols: list[str]) -> tuple[str, int]: - """ - Merge symbols into a numbered slot (dedup). Returns - (watchlist_name, number_of_new_symbols_added). - """ - file_path = self.paths.watchlist_slot(slot) - existing_name, existing = self.load_symbols_from_json(file_path) - watchlist_name = existing_name or f"Watchlist {slot}" - existing_set = {e['SYMBOL'] for e in existing if 'SYMBOL' in e} - added = 0 - for s in symbols: - if s not in existing_set: - existing.append({"SYMBOL": s, "Quantity": 0.0, "Price": 0.0}) - existing_set.add(s) - added += 1 - if added > 0: - self.save_slot(slot, watchlist_name, existing) - return watchlist_name, added - - # ------------------------------------------------------------------ - # Dashboard state + config - # ------------------------------------------------------------------ - - def save_dashboard_state(self, cells: list[dict]) -> None: - """Auto-save current stocks to the main state file between sessions.""" - try: - state = { - 'symbols': cells, - 'last_updated': datetime.now().isoformat(), - } - with open(self.paths.state_file, 'w') as f: - json.dump(state, f, indent=2) - print(f"[ACTIVITY] Session dashboard state file updated: {self.paths.state_file.name}", flush=True) - if self._hf: - self._hf.push_file_async(self.paths.state_file) - self._backup() - except Exception as e: - logging.error(f"Failed to save main dashboard state file: {e}") - - def load_config(self) -> dict: - """Load persisted UI config (window geometry kept for compatibility).""" - defaults = { - 'window_size': '1280x800', - 'window_position': '+100+100', - 'current_watchlist_name': "Default List", - } - try: - with open(self.paths.config_file, 'r') as f: - config = json.load(f) - return {**defaults, **config} - except FileNotFoundError: - return defaults - except Exception as e: - logging.error(f"Failed to load config: {e}") - return defaults - - def save_config(self, current_watchlist_name: str, - window_size: str = '1280x800', - window_position: str = '+100+100') -> None: - """Persist config. Geometry fields retained for file compatibility.""" - try: - config = { - 'window_size': window_size, - 'window_position': window_position, - 'last_updated': datetime.now().isoformat(), - 'current_watchlist_name': current_watchlist_name, - } - with open(self.paths.config_file, 'w') as f: - json.dump(config, f, indent=2) - print(f"[ACTIVITY] Session configuration file updated: {self.paths.config_file.name}", flush=True) - if self._hf: - self._hf.push_file_async(self.paths.config_file) - self._backup() - except Exception as e: - logging.error(f"Failed to save config: {e}") diff --git a/nsedash/crew/__init__.py b/nsedash/crew/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/nsedash/crew/agents.py b/nsedash/crew/agents.py deleted file mode 100644 index 2352795f82a8b437101e456c40ad08af34af286c..0000000000000000000000000000000000000000 --- a/nsedash/crew/agents.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -nsedash.crew.agents -=================== -Domain agents. Each agent's tools are deterministic Python callables over -the core services — no LLM. The analysis tools reproduce, byte-for-byte in -behaviour, the compute paths of v4.5's ``filter_stocks_by_change``, -``filter_stocks_by_volume_change``, ``filter_stocks_by_change_return`` and -``load_top_score_stocks``, including their JSON cache-freshness logic. -""" - -from __future__ import annotations - -import logging -from datetime import datetime - -from ..config import ANALYSIS_LIMIT, Paths -from ..core.portfolio import import_portfolio -from ..core.recommender_bridge import run_recommendations, train_all_models -from ..core.stock_data import StockData -from ..core.watchlists import WatchlistStore -from .framework import Agent, Tool - -PRICE_CHANGE_COLUMNS = { - "1D": "DAILY_CHANGE", "1W": "WEEK_CHANGE", - "1M": "MONTH_CHANGE", "3M": "THREE_MONTH_CHANGE", -} -RETURN_COLUMNS = { - "1W": "WEEK_CHANGE", "1M": "MONTH_CHANGE", "3M": "THREE_MONTH_CHANGE", -} - - -def build_agents(stock_data: StockData, watchlists: WatchlistStore, - paths: Paths) -> dict[str, Agent]: - """Construct the full agent roster wired to live service instances.""" - - # ------------------------------------------------------------------ - # Data steward tools - # ------------------------------------------------------------------ - - def reload_csv() -> dict: - stock_data.load_data() - return {"last_date": str(stock_data.last_date)} - - def enrich_date(date) -> "object": - return stock_data.get_data_for_date(date) - - # ------------------------------------------------------------------ - # Market watcher tools - # ------------------------------------------------------------------ - - def live_quotes(symbols: list[str], preferred_source: str = "Google") -> dict: - data, source = stock_data.fetch_real_time_data(symbols, preferred_source) - return {"quotes": data, "source": source} - - # ------------------------------------------------------------------ - # Analyst tools (cache-first, parity with v4.5) - # ------------------------------------------------------------------ - - def price_change_analysis(duration: str, change_type: str, - threshold: float, selected_date) -> dict: - entry = paths.hist_price_files.get((duration, change_type)) - if not entry: - raise ValueError("Invalid duration or change type.") - json_name = f"{duration} {change_type} Price Portfolio" - current_max = stock_data.df['TRAN_DATE'].max() - cached = watchlists.load_analysis_json_if_recent(entry, current_max) - if cached: - return {"symbols": cached, "json_name": json_name, "cached": True} - change_column = PRICE_CHANGE_COLUMNS.get(duration) - if not change_column: - raise ValueError("Invalid duration specified.") - data = stock_data.get_data_for_date(selected_date) - if data.empty: - return {"symbols": [], "json_name": json_name, "cached": False} - if change_column == "DAILY_CHANGE": - # 1D change is derived, matching the card's daily-change formula. - data = data.copy() - data["DAILY_CHANGE"] = ( - (data['CLOSE_PRICE'] / data['PREV_CL_PR'] - 1) * 100 - ).where(data['PREV_CL_PR'] != 0, 0.0) - if change_column not in data.columns: - raise ValueError(f"'{change_column}' data not available.") - filtered = data.dropna(subset=[change_column]).copy() - if change_type == "Max": - filtered = filtered[filtered[change_column] >= threshold] \ - .sort_values(by=change_column, ascending=False) - else: - filtered = filtered[filtered[change_column] <= threshold] \ - .sort_values(by=change_column, ascending=True) - top = filtered.head(ANALYSIS_LIMIT).copy() - watchlists.save_watchlist_json(top, entry, json_name) - return {"symbols": top['SYMBOL'].tolist(), - "json_name": json_name, "cached": False} - - def volume_change_analysis(duration: str, volume_cutoff: float) -> dict: - entry = paths.hist_vol_files.get(duration) - if not entry: - raise ValueError("Invalid duration specified.") - json_name = f"{duration} Max Volume Portfolio" - current_max = stock_data.df['TRAN_DATE'].max() - cached = watchlists.load_analysis_json_if_recent(entry, current_max) - if cached: - return {"symbols": cached, "json_name": json_name, "cached": True} - movers = stock_data.get_top_volume_movers( - duration, limit=ANALYSIS_LIMIT, volume_cutoff=volume_cutoff - ) - if movers.empty: - return {"symbols": [], "json_name": json_name, "cached": False} - watchlists.save_watchlist_json(movers, entry, json_name) - return {"symbols": movers['SYMBOL'].tolist(), - "json_name": json_name, "cached": False} - - def return_analysis(duration: str, selected_date) -> dict: - entry = paths.max_return_files.get(duration) - if not entry: - raise ValueError("Invalid duration specified.") - json_name = f"{duration} Max Return Portfolio" - current_max = stock_data.df['TRAN_DATE'].max() - cached = watchlists.load_analysis_json_if_recent(entry, current_max) - if cached: - return {"symbols": cached, "json_name": json_name, "cached": True} - change_column = RETURN_COLUMNS.get(duration) - if not change_column: - raise ValueError("Invalid duration specified.") - data = stock_data.get_data_for_date(selected_date) - if data.empty: - return {"symbols": [], "json_name": json_name, "cached": False} - if change_column not in data.columns: - raise ValueError(f"'{change_column}' data not available.") - filtered = data.dropna(subset=[change_column]).copy() - filtered = filtered[filtered[change_column] > 0] \ - .sort_values(by=change_column, ascending=False) - top = filtered.head(ANALYSIS_LIMIT).copy() - watchlists.save_watchlist_json(top, entry, json_name) - return {"symbols": top['SYMBOL'].tolist(), - "json_name": json_name, "cached": False} - - def top_score_analysis() -> dict: - entry = paths.max_score_watchlist_file - json_name = "Top 50 MC Score" - cached = watchlists.load_analysis_json_if_recent(entry, datetime.now()) - if cached: - return {"symbols": cached, "json_name": json_name, "cached": True} - top = stock_data.get_top_mc_score_movers(limit=ANALYSIS_LIMIT) - if top.empty: - return {"symbols": [], "json_name": json_name, "cached": False} - watchlists.save_watchlist_json(top, entry, json_name) - return {"symbols": top['SYMBOL'].tolist(), - "json_name": json_name, "cached": False} - - # ------------------------------------------------------------------ - # Portfolio manager tools - # ------------------------------------------------------------------ - - def portfolio_import(filepath: str | None = None) -> dict: - holdings, unmapped, broker_name = import_portfolio(filepath, paths) - return {"holdings": holdings, "unmapped": unmapped, "broker_name": broker_name} - - # ------------------------------------------------------------------ - # Recommender tools - # ------------------------------------------------------------------ - - def recommend(symbols: list[str]) -> list[dict]: - return run_recommendations( - paths.recommender_dir, symbols, stock_data.symbol_history - ) - - def train_models() -> dict: - symbols = stock_data.get_all_symbols_from_df() - logging.info(f"Training LSTM models for {len(symbols)} symbols.") - return train_all_models( - paths.recommender_dir, symbols, stock_data.symbol_history - ) - - # ------------------------------------------------------------------ - # Agent roster - # ------------------------------------------------------------------ - - return { - "data_steward": Agent( - role="Data Steward", - goal="Own the historical dataset: load, trim, enrich, and cache.", - backstory="Deterministic custodian of nse_processed_data.csv.", - tools=[ - Tool("reload_csv", reload_csv, "Reload the source CSV"), - Tool("enrich_date", enrich_date, "Enriched frame for a date"), - ], - ), - "market_watcher": Agent( - role="Market Watcher", - goal="Fetch live quotes with Google↔Yahoo failover.", - backstory="Scrapes real-time prices concurrently with timeouts.", - tools=[Tool("live_quotes", live_quotes, "Fetch live quotes")], - ), - "analyst": Agent( - role="Quant Analyst", - goal="Rank the universe by price, volume, return and MC score.", - backstory="Cache-first analytics identical to dashboard v4.5.", - tools=[ - Tool("price_change_analysis", price_change_analysis), - Tool("volume_change_analysis", volume_change_analysis), - Tool("return_analysis", return_analysis), - Tool("top_score_analysis", top_score_analysis), - ], - ), - "portfolio_manager": Agent( - role="Portfolio Manager", - goal="Import broker holdings and value the book.", - backstory="Maps ICICI short names to NSE symbols.", - tools=[Tool("portfolio_import", portfolio_import)], - ), - "recommender": Agent( - role="Recommender", - goal="Surface LSTM buy/sell calls when models exist.", - backstory="Bridges to the optional recommender module.", - tools=[ - Tool("recommend", recommend), - Tool("train_models", train_models), - ], - ), - } diff --git a/nsedash/crew/crews.py b/nsedash/crew/crews.py deleted file mode 100644 index e42ead9e399e2849ced327546b93eba419490ec7..0000000000000000000000000000000000000000 --- a/nsedash/crew/crews.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -nsedash.crew.crews -================== -Pre-wired crews the dashboard service kicks off for its operation -categories. Each crew is a deterministic pipeline over the agents in -``nsedash.crew.agents`` — no LLM involved. -""" - -from __future__ import annotations - -from .framework import Agent, Crew, Process, Task - - -class CrewFactory: - """Builds crews bound to the shared agent roster.""" - - def __init__(self, agents: dict[str, Agent]): - self.agents = agents - - def live_refresh_crew(self, symbols: list[str], preferred_source: str) -> Crew: - """One live polling tick: fetch quotes for the displayed symbols.""" - watcher = self.agents["market_watcher"] - return Crew( - agents=[watcher], - tasks=[Task( - description="Fetch live quotes for displayed symbols", - agent=watcher, tool="live_quotes", output_key="live", - inputs={"symbols": symbols, "preferred_source": preferred_source}, - )], - name="live_refresh", - ) - - def price_analysis_crew(self, duration: str, change_type: str, - threshold: float, selected_date) -> Crew: - analyst = self.agents["analyst"] - return Crew( - agents=[analyst], - tasks=[Task( - description=f"{duration} {change_type} price-change screen", - agent=analyst, tool="price_change_analysis", - output_key="analysis", - inputs={"duration": duration, "change_type": change_type, - "threshold": threshold, "selected_date": selected_date}, - )], - name="price_analysis", - ) - - def volume_analysis_crew(self, duration: str, volume_cutoff: float) -> Crew: - analyst = self.agents["analyst"] - return Crew( - agents=[analyst], - tasks=[Task( - description=f"{duration} max volume screen", - agent=analyst, tool="volume_change_analysis", - output_key="analysis", - inputs={"duration": duration, "volume_cutoff": volume_cutoff}, - )], - name="volume_analysis", - ) - - def return_analysis_crew(self, duration: str, selected_date) -> Crew: - analyst = self.agents["analyst"] - return Crew( - agents=[analyst], - tasks=[Task( - description=f"{duration} max return simulation", - agent=analyst, tool="return_analysis", output_key="analysis", - inputs={"duration": duration, "selected_date": selected_date}, - )], - name="return_analysis", - ) - - def top_score_crew(self) -> Crew: - analyst = self.agents["analyst"] - return Crew( - agents=[analyst], - tasks=[Task( - description="Top-50 Moneycontrol score screen", - agent=analyst, tool="top_score_analysis", output_key="analysis", - )], - name="top_score", - ) - - def refresh_crew(self, selected_date) -> Crew: - """Full dashboard refresh: reload CSV then re-enrich the date.""" - steward = self.agents["data_steward"] - return Crew( - agents=[steward], - tasks=[ - Task(description="Reload historical CSV", - agent=steward, tool="reload_csv", output_key="reload"), - Task(description="Re-enrich the selected date", - agent=steward, tool="enrich_date", output_key="date_data", - inputs={"date": selected_date}), - ], - process=Process.sequential, - name="dashboard_refresh", - ) - - def portfolio_crew(self, filepath: str | None = None) -> Crew: - pm = self.agents["portfolio_manager"] - return Crew( - agents=[pm], - tasks=[Task(description="Import portfolio", - agent=pm, tool="portfolio_import", - output_key="portfolio", - inputs={"filepath": filepath})], - name="portfolio_import", - ) - - def recommendation_crew(self, symbols: list[str]) -> Crew: - rec = self.agents["recommender"] - return Crew( - agents=[rec], - tasks=[Task(description="Buy/sell recommendations", - agent=rec, tool="recommend", output_key="recommendations", - inputs={"symbols": symbols})], - name="recommendation", - ) - - def training_crew(self) -> Crew: - rec = self.agents["recommender"] - return Crew( - agents=[rec], - tasks=[Task(description="Train all LSTM models", - agent=rec, tool="train_models", output_key="training")], - name="training", - ) diff --git a/nsedash/crew/framework.py b/nsedash/crew/framework.py deleted file mode 100644 index 6b24e8dc021edae2bda9418fcfac236ddf16e665..0000000000000000000000000000000000000000 --- a/nsedash/crew/framework.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -nsedash.crew.framework -====================== -A minimal crewAI-style agentic framework implemented in pure Python. - -Deliberately NO LLM anywhere: agents are deterministic executors whose -"reasoning" is plain Python code. The surface mirrors crewAI's mental model -(Agent / Task / Crew / Process, ``kickoff()``) so orchestration reads the -same way, while every task resolves to a concrete Python callable (a Tool). - -Concurrency: crews execute on an asyncio event loop. Blocking tool -functions are dispatched to a thread-pool executor, so Process.parallel -crews run their tasks genuinely concurrently and never block the caller's -event loop (used by the web layer's background job system). -""" - -from __future__ import annotations - -import asyncio -import logging -import time -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from enum import Enum -from typing import Any, Callable - - -class Process(str, Enum): - sequential = "sequential" - parallel = "parallel" - - -@dataclass -class Tool: - """A named deterministic capability: a plain Python callable.""" - name: str - func: Callable[..., Any] - description: str = "" - - def run(self, *args, **kwargs) -> Any: - return self.func(*args, **kwargs) - - -@dataclass -class Agent: - """ - A deterministic worker with a role, a goal, and a toolbox. - ``execute(tool_name, ...)`` dispatches to the named tool. There is no - model call and no free-form planning — dispatch is exact-match. - """ - role: str - goal: str - backstory: str = "" - tools: list[Tool] = field(default_factory=list) - - def tool(self, name: str) -> Tool: - for t in self.tools: - if t.name == name: - return t - raise KeyError(f"Agent '{self.role}' has no tool named '{name}'") - - def execute(self, tool_name: str, /, *args, **kwargs) -> Any: - start = time.time() - try: - result = self.tool(tool_name).run(*args, **kwargs) - return result - finally: - logging.info( - f"AGENT[{self.role}] tool={tool_name} " - f"elapsed={time.time() - start:.2f}s" - ) - - -@dataclass -class Task: - """ - A unit of work bound to an agent. ``tool`` names the agent tool to run; - ``inputs`` may be a dict, or a callable(context) -> dict evaluated at - execution time so later tasks can consume earlier task outputs. - ``output_key`` names the result in the crew context. - """ - description: str - agent: Agent - tool: str - output_key: str - inputs: dict | Callable[[dict], dict] | None = None - - def resolve_inputs(self, context: dict) -> dict: - if self.inputs is None: - return {} - if callable(self.inputs): - return self.inputs(context) - return dict(self.inputs) - - -class Crew: - """ - Orchestrates tasks across agents. - - Process.sequential — tasks run in declared order; each task sees prior - outputs in the shared context. - Process.parallel — tasks run concurrently; each task sees only the - initial inputs. - """ - - _executor = ThreadPoolExecutor(max_workers=16, thread_name_prefix="crew") - - def __init__(self, agents: list[Agent], tasks: list[Task], - process: Process = Process.sequential, name: str = "crew"): - self.agents = agents - self.tasks = tasks - self.process = process - self.name = name - - async def _run_task(self, task: Task, context: dict) -> Any: - loop = asyncio.get_running_loop() - kwargs = task.resolve_inputs(context) - return await loop.run_in_executor( - self._executor, - lambda: task.agent.execute(task.tool, **kwargs), - ) - - async def kickoff_async(self, inputs: dict | None = None) -> dict: - context: dict = dict(inputs or {}) - logging.info(f"CREW[{self.name}] kickoff process={self.process.value} " - f"tasks={len(self.tasks)}") - if self.process is Process.sequential: - for task in self.tasks: - context[task.output_key] = await self._run_task(task, context) - else: - results = await asyncio.gather( - *(self._run_task(t, context) for t in self.tasks) - ) - for task, result in zip(self.tasks, results): - context[task.output_key] = result - return context - - def kickoff(self, inputs: dict | None = None) -> dict: - """ - Synchronous entry point. Runs on a private event loop when no loop - is active in the current thread; otherwise callers should await - ``kickoff_async`` directly. - """ - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(self.kickoff_async(inputs)) - raise RuntimeError( - "Crew.kickoff() called from a running event loop; " - "use `await crew.kickoff_async(...)` instead." - ) diff --git a/nsedash/logging_utils.py b/nsedash/logging_utils.py deleted file mode 100644 index 0913db227103f5c8ce43c4b57e5cc4815bfeef19..0000000000000000000000000000000000000000 --- a/nsedash/logging_utils.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -nsedash.logging_utils -===================== -Application + metrics logging, and the elapsed-time decorator. - -The Tkinter ``wait_dialog_decorator`` from v4.5 has no equivalent in a -server process; its UX role (a modal "please wait" indicator) is fulfilled -by the web job system (``nsedash.web.background``) which the frontend polls -to show a spinner. ``elapsed_time_logger`` is retained with identical log -format: ``timestamp | function | message | N.NN sec``. -""" - -from __future__ import annotations - -import logging -import threading -import time -from datetime import datetime -from functools import wraps -from pathlib import Path - -_metrics_lock = threading.Lock() -_metrics_file: Path | None = None - - -def setup_logging(log_file: Path, metrics_file: Path) -> None: - """Configure the root logger to write to the dashboard log file.""" - global _metrics_file - _metrics_file = metrics_file - log_file.parent.mkdir(parents=True, exist_ok=True) - root = logging.getLogger() - root.setLevel(logging.INFO) - # Avoid duplicate handlers on repeated app-factory calls (tests). - if not any( - isinstance(h, logging.FileHandler) - and getattr(h, 'baseFilename', None) == str(log_file) - for h in root.handlers - ): - handler = logging.FileHandler(str(log_file)) - handler.setFormatter( - logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - ) - root.addHandler(handler) - - -def log_operation(operation_name: str, elapsed_time: float) -> None: - """Append an operation timing entry to the metrics log (thread-safe).""" - if _metrics_file is None: - return - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - line = f"{timestamp} | {operation_name} | {elapsed_time:.2f} sec\n" - # AUDIT FIX M-10: original opened the file unsynchronised from multiple - # threads; interleaved writes could corrupt lines. - with _metrics_lock: - with open(_metrics_file, "a") as fh: - fh.write(line) - - -def log_dashboard_message(message: str) -> None: - """Write a progress message to the dashboard log.""" - logging.info(f"DASHBOARD_PROGRESS: {message}") - - -def elapsed_time_logger(message: str = "Processing..."): - """Decorator: logs wall-clock execution time of the wrapped callable.""" - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() - function_name = func.__name__ - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - try: - result = func(*args, **kwargs) - finally: - elapsed = time.time() - start_time - logging.info( - f"{timestamp} | {function_name} | {message} | {elapsed:.2f} sec" - ) - log_operation(function_name, elapsed) - return result - return wrapper - return decorator diff --git a/nsedash/web/__init__.py b/nsedash/web/__init__.py deleted file mode 100644 index 86403a0d5180dc72f7766573a00e9f29a75ec41b..0000000000000000000000000000000000000000 --- a/nsedash/web/__init__.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -nsedash.web -=========== -Flask application factory. Wires the service layer, agent roster, crews, -async runtime, job manager, live scheduler and script runner into -``app.extensions["nsedash"]``. - -HF Persistence --------------- -When ``SPACE_ID`` is set (HuggingFace Space), ``HFDataSync`` is -activated: -- On startup: pulls all processed files from ``Sam-max1/nsedash_data`` - *before* ``DashboardService`` is created so the service sees fresh data. -- On every file write: pushes the changed file asynchronously (3-second - debounce, fire-and-forget background thread). -- On session cleanup: removes per-user files from the dataset. - -Multi-User Auth ---------------- -``user_auth.init(base_dir)`` is called once to set file paths, then the -``UserDatabase`` is loaded (creating the seed users.db if absent). -""" - -from __future__ import annotations - -import os - -from flask import Flask - -from ..config import HF_DATASET_REPO, Paths -from ..core.dashboard_service import DashboardService -from ..core.hf_sync import make_sync -from ..core import user_auth -from ..crew.agents import build_agents -from ..crew.crews import CrewFactory -from ..logging_utils import setup_logging -from .api import api -from .background import (AsyncRuntime, JobManager, LiveRefreshScheduler, - ScriptRunner) -from .routes import pages - - -def create_app(base_dir=None) -> Flask: - paths = Paths(base_dir) - paths.ensure_dirs() - setup_logging(paths.log_file, paths.metrics_file) - - # ------------------------------------------------------------------ - # 1. HF Dataset sync — startup pull BEFORE DashboardService / auth init - # ------------------------------------------------------------------ - hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HF_PRIVATE_TOKEN") - hf_sync = make_sync(data_dir=paths.data_dir, repo_id=HF_DATASET_REPO, token=hf_token) - hf_sync.pull_on_startup() - - # ------------------------------------------------------------------ - # 2. Initialise user auth (file paths, seed users.db, link hf_sync) - # ------------------------------------------------------------------ - user_auth.init(paths.base_dir, hf_sync=hf_sync) - user_auth.invalidate_all_sessions_on_startup() - - - # ------------------------------------------------------------------ - # 3. Service / crew layer - # ------------------------------------------------------------------ - service = DashboardService(paths, hf_sync=hf_sync) - agents = build_agents(service.stock_data, service.watchlists, paths) - crews = CrewFactory(agents) - runtime = AsyncRuntime() - jobs = JobManager(runtime) - scheduler = LiveRefreshScheduler(runtime, service.load_live_data, base_dir=paths.base_dir) - scripts = ScriptRunner() - - app = Flask(__name__) - app.extensions["nsedash"] = { - "paths": paths, - "service": service, - "services": {"admin": service}, - "base_dir": paths.base_dir, - "agents": agents, - "crews": crews, - "runtime": runtime, - "jobs": jobs, - "scheduler": scheduler, - "scripts": scripts, - "hf_sync": hf_sync, - } - app.register_blueprint(pages) - app.register_blueprint(api) - - # Start inactivity cleanup daemon thread and stats reporting thread - from .api import start_cleanup_thread, start_stats_thread - start_cleanup_thread(app) - start_stats_thread(app) - - @app.after_request - def add_security_headers(response): - response.headers["Content-Security-Policy"] = ( - "default-src 'self'; " - "script-src 'self' https://cdnjs.cloudflare.com; " - "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " - "font-src 'self' https://fonts.gstatic.com; " - "img-src 'self' data:; " - "connect-src 'self';" - ) - response.headers["X-Frame-Options"] = "SAMEORIGIN" - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - response.headers["Permissions-Policy"] = "geolocation=(), camera=(), microphone=()" - return response - - return app - diff --git a/nsedash/web/api.py b/nsedash/web/api.py deleted file mode 100644 index 044c8531aaa56a37af900a4d33aec7fddb3412b9..0000000000000000000000000000000000000000 --- a/nsedash/web/api.py +++ /dev/null @@ -1,1196 +0,0 @@ -""" -nsedash.web.api -=============== -REST API. Every v4.5 dashboard action maps to an endpoint. Long-running -operations (analysis crews, refresh, portfolio import, recommendations, -training) are submitted to the JobManager and polled by the frontend — -the web equivalent of the v4.5 wait-dialog + worker-thread pattern. - -Authentication --------------- -All endpoints (except /api/auth/* and /api/status) require a valid -``X-Session-Token`` header issued by the login + TOTP flow. -The token resolves to a ``userid`` via ``user_auth.SessionManager``, -which is then used as the per-user isolation key in ``_svc()``. -""" - -from __future__ import annotations - -import logging -import os -import signal -import subprocess -import threading -import time -from pathlib import Path - -import pandas as pd -from flask import Blueprint, current_app, jsonify, request, send_file -from werkzeug.utils import secure_filename - -from ..config import UTILITY_SCRIPTS, WATCHLIST_SLOTS -from ..core import user_auth - -api = Blueprint("api", __name__, url_prefix="/api") - -# In-memory activity tracker (userid -> last_active timestamp) -_active_sessions: dict[str, float] = {} -SESSION_TIMEOUT_SECONDS = 15 * 60 # 15 minutes - -# In-memory tracker for recently inactivated sessions to notify UI -_inactive_tokens: dict[str, float] = {} - -# Hourly session stats -total_connections_this_hour = 0 -total_disconnections_this_hour = 0 - -# Routes exempt from auth check -_AUTH_EXEMPT = { - "/api/auth/login", - "/api/auth/totp", - "/api/auth/logout", - "/api/auth/status", - "/api/status", -} - - -@api.before_request -def check_auth(): - if current_app.testing: - return - if request.path in _AUTH_EXEMPT: - return - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token") - or request.args.get("session_token")) - if not token: - return jsonify({"error": "Authentication required", "auth_required": True}), 401 - valid, _userid = user_auth.SessionManager.validate_session(token) - if not valid: - if token: - now = time.time() - # Prune old entries to prevent leaks - for k in list(_inactive_tokens.keys()): - if now - _inactive_tokens[k] > 3600: - _inactive_tokens.pop(k, None) - if token in _inactive_tokens: - _inactive_tokens.pop(token, None) - return jsonify({"error": "Session terminated due to inactivity", "auth_required": True, "reason": "inactivity"}), 401 - return jsonify({"error": "Session expired or invalid", "auth_required": True}), 401 - - -# --------------------------------------------------------------------------- -# Auth endpoints -# --------------------------------------------------------------------------- - -@api.post("/auth/login") -def auth_login(): - """Step 1: userid + password -> returns totp_required and optional QR.""" - body = request.get_json(force=True) - userid = (body.get("userid") or "").strip() - password = body.get("password") or "" - if not userid or not password: - return jsonify({"ok": False, "message": "userid and password are required."}), 400 - - success, message, totp_required = user_auth.AuthenticationService.login(userid, password) - if not success: - return jsonify({"ok": False, "message": message}), 401 - - resp = {"ok": True, "userid": userid, "totp_required": totp_required, "message": message} - if message in ("first_login", "totp_setup"): - resp["qr"] = user_auth.AuthenticationService.get_totp_qr(userid) - user = user_auth.UserDatabase.get_user(userid) or {} - secret = user.get("totp_secret") - if not secret and user.get("totp_secret_encrypted"): - secret = user_auth.EncryptionManager.decrypt(user["totp_secret_encrypted"]) - resp["secret"] = secret - return jsonify(resp) - - -@api.post("/auth/totp") -def auth_totp(): - """Step 2: userid + TOTP code -> issues a session token.""" - body = request.get_json(force=True) - userid = (body.get("userid") or "").strip() - code = (body.get("code") or "").strip() - if not userid or not code: - return jsonify({"ok": False, "message": "userid and code are required."}), 400 - - success, result = user_auth.AuthenticationService.verify_totp(userid, code) - if not success: - return jsonify({"ok": False, "message": result}), 401 - - user = user_auth.UserDatabase.get_user(userid) or {} - force_pw = bool(user.get("force_password_change", False)) - - _active_sessions[userid] = time.time() - global total_connections_this_hour - total_connections_this_hour += 1 - print(f"[SESSION] Login: {userid} (IP: {request.remote_addr})", flush=True) - logging.info("Login: %s from %s", userid, request.remote_addr) - return jsonify({"ok": True, "session_token": result, "userid": userid, "force_password_change": force_pw}) - - -@api.post("/auth/logout") -def auth_logout(): - """Destroy the current session.""" - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token")) - if token: - valid, userid = user_auth.SessionManager.validate_session(token) - user_auth.SessionManager.destroy_session(token) - if userid: - _active_sessions.pop(userid, None) - global total_disconnections_this_hour - total_disconnections_this_hour += 1 - print(f"[SESSION] Logout: {userid}", flush=True) - return jsonify({"ok": True}) - - -@api.get("/auth/status") -def auth_status(): - """Check if the current session is valid.""" - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token") - or request.args.get("session_token")) - if not token: - return jsonify({"authenticated": False}) - valid, userid = user_auth.SessionManager.validate_session(token) - if valid: - u = user_auth.UserDatabase.get_user(userid) or {} - is_adm = True if userid == "admin" else bool(u.get("is_admin", False)) - force_pw = bool(u.get("force_password_change", False)) - from ..core.nvidia_llm import get_nvidia_api_status - return jsonify({"authenticated": True, "userid": userid, - "is_admin": is_adm, - "force_password_change": force_pw, - "nvidia_status": get_nvidia_api_status()}) - - # Check if this token was terminated due to inactivity - if token: - now = time.time() - for k in list(_inactive_tokens.keys()): - if now - _inactive_tokens[k] > 3600: - _inactive_tokens.pop(k, None) - if token in _inactive_tokens: - _inactive_tokens.pop(token, None) - return jsonify({"authenticated": False, "reason": "inactivity"}) - - return jsonify({"authenticated": False}) - - -@api.post("/auth/change-password") -def auth_change_password(): - """Change user password enforcing policy & 2-time entry confirmation.""" - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token") - or request.args.get("session_token")) - if not token: - return jsonify({"ok": False, "message": "Authentication required."}), 401 - valid, userid = user_auth.SessionManager.validate_session(token) - if not valid or not userid: - return jsonify({"ok": False, "message": "Session invalid or expired."}), 401 - - body = request.get_json(force=True) - current_pw = body.get("current_password") - new_pw = body.get("new_password") or "" - confirm_pw = body.get("confirm_password") or "" - - user = user_auth.UserDatabase.get_user(userid) or {} - is_forced = bool(user.get("force_password_change", False)) - - ok, msg = user_auth.change_user_password( - userid, current_pw, new_pw, confirm_pw, is_admin_override=is_forced - ) - if not ok: - return jsonify({"ok": False, "message": msg}), 400 - - return jsonify({"ok": True, "message": msg}) - - -def start_cleanup_thread(app): - import threading - from ..core import user_auth as _ua - - def _cleanup(): - while True: - time.sleep(60) - try: - with app.app_context(): - _ua.SessionManager.cleanup_expired() - - now = time.time() - extensions = app.extensions["nsedash"] - services = extensions["services"] - base_dir = extensions["base_dir"] - hf_sync = extensions.get("hf_sync") - from ..config import Paths - import shutil - - expired_users = [ - userid for userid, last_active in list(_active_sessions.items()) - if userid not in ("admin",) and (now - last_active) > SESSION_TIMEOUT_SECONDS - ] - - if expired_users: - logging.info("[Cleanup] Expiring %d inactive session(s).", len(expired_users)) - - for userid in expired_users: - logging.info("[Cleanup] Purging session: %s", userid) - print(f"[SESSION] Disconnected (Inactivity): {userid}", flush=True) - global total_disconnections_this_hour - total_disconnections_this_hour += 1 - - for tok in _ua.SessionManager.get_userid_sessions(userid): - _inactive_tokens[tok] = time.time() - _ua.SessionManager.destroy_session(tok) - - if userid in services: - del services[userid] - _active_sessions.pop(userid, None) - - paths = Paths(base_dir=base_dir, session_token=userid) - try: - for f in (paths.state_file, paths.config_file, paths.portfolio_file): - if f.exists(): - f.unlink() - if paths.watchlist_dir.exists(): - shutil.rmtree(paths.watchlist_dir) - except Exception as exc: - logging.error("[Cleanup] Error removing files for %s: %s", userid, exc) - - if hf_sync: - hf_sync.delete_session_files(userid) - - except Exception as ex: - logging.error("[Cleanup] Exception: %s", ex) - - threading.Thread(target=_cleanup, daemon=True).start() - - -def start_stats_thread(app): - import threading - import time - - def _stats(): - last_15m = time.time() - last_hour = time.time() - while True: - time.sleep(10) - now = time.time() - if now - last_15m >= 900: - last_15m = now - active_count = len(_active_sessions) - print(f"[STATS] {time.strftime('%Y-%m-%d %H:%M:%S')} - Active sessions: {active_count}", flush=True) - logging.info("[STATS] Active sessions: %d", active_count) - if now - last_hour >= 3600: - last_hour = now - global total_connections_this_hour, total_disconnections_this_hour - print(f"[STATS] Hourly: Logins={total_connections_this_hour}, Logouts={total_disconnections_this_hour}", flush=True) - logging.info("[STATS] Hourly: Logins=%d, Logouts=%d", total_connections_this_hour, total_disconnections_this_hour) - total_connections_this_hour = 0 - total_disconnections_this_hour = 0 - - threading.Thread(target=_stats, daemon=True).start() - - -def get_session_token() -> str: - """ - Resolve the current request's session token to a userid. - - The userid is the isolation key for DashboardService lookup. - Falls back to 'admin' for unauthenticated requests. - """ - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token") - or request.args.get("session_token")) - if token: - valid, userid = user_auth.SessionManager.validate_session(token) - if valid and userid: - _active_sessions[userid] = time.time() - return userid - _active_sessions["admin"] = time.time() - return "admin" - - -@api.after_request -def backup_user_data_on_change(response): - """Automatically backs up user files encrypted to users.db on POST/PUT/DELETE.""" - if request.method in ("POST", "PUT", "DELETE"): - # Ensure we don't back up auth or admin-action endpoints themselves - if not request.path.startswith(("/api/auth/", "/api/admin/")): - userid = get_session_token() - if userid and userid != "admin": - extensions = current_app.extensions["nsedash"] - base_dir = extensions["base_dir"] - user_auth.backup_session_to_db(userid, base_dir) - return response - - -def _svc(): - token = get_session_token() - _active_sessions[token] = time.time() # Record last active timestamp - extensions = current_app.extensions["nsedash"] - services = extensions["services"] - if token not in services: - import shutil - from ..config import Paths - from ..core.dashboard_service import DashboardService - - base_dir = extensions["base_dir"] - paths = Paths(base_dir=base_dir, session_token=token) - paths.ensure_dirs() - - # Check if we can restore from encrypted users.db database - restored = user_auth.restore_session_from_db(token, base_dir) - - if not restored: - # Seed default state file if missing - default_state_file = base_dir / 'processed' / 'nse_dashboard_contents.json' - if paths.state_file != default_state_file and not paths.state_file.exists(): - if default_state_file.exists(): - shutil.copy2(default_state_file, paths.state_file) - - # Seed default config file if missing - default_config_file = base_dir / 'processed' / 'nse_dashboard_config.json' - if paths.config_file != default_config_file and not paths.config_file.exists(): - if default_config_file.exists(): - shutil.copy2(default_config_file, paths.config_file) - - # Seed default watchlists if slot directory is empty or missing slots - default_wl_dir = base_dir / 'processed' / 'watchlists' - if paths.watchlist_dir != default_wl_dir and default_wl_dir.exists() and not any(paths.watchlist_dir.iterdir()): - import re - for item in default_wl_dir.iterdir(): - if item.is_file(): - match = re.match(r"nse_dashboard_watchlist(\d+)\.json", item.name) - if match: - num = int(match.group(1)) - if 1 <= num <= WATCHLIST_SLOTS: - if default_wl_dir.joinpath(item.name).exists(): - shutil.copy2(item, paths.watchlist_dir / item.name) - else: - # Create blank watchlist slot for other users - with open(paths.watchlist_dir / item.name, "w") as f: - json.dump({ - "name": f"Watchlist {num}", - "symbols": [], - "last_updated": datetime.now().isoformat() - }, f, indent=2) - else: - # Copy non-slot files (like analysis cache) - shutil.copy2(item, paths.watchlist_dir / item.name) - - # Ensure all 1..20 watchlist slots exist in paths.watchlist_dir - from datetime import datetime - import json - for num in range(1, WATCHLIST_SLOTS + 1): - slot_file = paths.watchlist_dir / f"nse_dashboard_watchlist{num}.json" - if not slot_file.exists(): - src_file = default_wl_dir / f"nse_dashboard_watchlist{num}.json" - if paths.watchlist_dir != default_wl_dir and src_file.exists(): - try: - shutil.copy2(src_file, slot_file) - except Exception: - pass - else: - try: - with open(slot_file, "w") as f: - json.dump({ - "name": f"Watchlist {num}", - "symbols": [], - "last_updated": datetime.now().isoformat() - }, f, indent=2) - except Exception: - pass - - services[token] = DashboardService(paths, hf_sync=extensions.get("hf_sync")) - - return services[token] - - - -def _crews(): - return current_app.extensions["nsedash"]["crews"] - - -def _jobs(): - return current_app.extensions["nsedash"]["jobs"] - - -def _scheduler(): - return current_app.extensions["nsedash"]["scheduler"] - - -def _scripts(): - return current_app.extensions["nsedash"]["scripts"] - - -# --------------------------------------------------------------------------- -# Board snapshot -# --------------------------------------------------------------------------- - -@api.get("/board") -def board(): - charts = request.args.get("charts", "1") != "0" - return jsonify(_svc().board_payload(include_charts=charts)) - - -# --------------------------------------------------------------------------- -# Cell management -# --------------------------------------------------------------------------- - -@api.post("/stocks") -def add_stock(): - body = request.get_json(force=True) - key = body.get("key") - svc = _svc() - cell = svc.add_stock(key) - if cell is None: - return jsonify({"ok": False, - "message": f"Stock '{key}' not found in data."}), 404 - return jsonify({"ok": True, "cell": svc._cell_payload(cell)}) - - -@api.delete("/stocks/") -def remove_stock(symbol): - ok = _svc().remove_stock(symbol) - return jsonify({"ok": ok}) - - -@api.delete("/stocks") -def delete_all(): - _svc().delete_all_stocks(mark_modified=True) - return jsonify({"ok": True}) - - -@api.post("/stocks//inputs") -def cell_inputs(symbol): - body = request.get_json(force=True) - result = _svc().update_cell_inputs( - symbol, body.get("quantity", 0), body.get("price", 0) - ) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/stocks//select") -def cell_select(symbol): - body = request.get_json(force=True) - _svc().toggle_selection(symbol, bool(body.get("selected"))) - return jsonify({"ok": True}) - - -@api.post("/mc-link") -def set_mc_link(): - body = request.get_json(force=True) - svc = _svc() - try: - entry = svc.stock_data.set_mc_link( - body.get("symbol", ""), body.get("security", ""), - body.get("link", "") - ) - except ValueError as e: - return jsonify({"ok": False, "message": str(e)}), 400 - svc._mc_links_cache = None # invalidate shared cache - return jsonify({"ok": True, "entry": entry, - "message": f"Moneycontrol link saved for " - f"{body.get('symbol')}."}) - - -# --------------------------------------------------------------------------- -# Date / refresh / export / config -# --------------------------------------------------------------------------- - -@api.post("/date") -def change_date(): - body = request.get_json(force=True) - result = _svc().change_date(body.get("date", "")) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/refresh") -def refresh(): - svc, crews, jobs = _svc(), _crews(), _jobs() - if svc.real_time_active: - _scheduler().stop() - - async def run(): - crew = crews.refresh_crew(svc.selected_date) - result = svc.refresh_dashboard() - _ = crew # crew retained for parity; refresh_dashboard drives reload - return result - - job_id = jobs.submit("Refreshing dashboard data...", run) - return jsonify({"ok": True, "job_id": job_id}) - - -@api.post("/export") -def export_csv(): - result = _svc().export_to_csv() - return jsonify(result), (200 if result.get("ok") else 500) - - -@api.get("/export/download") -def export_download(): - path = request.args.get("file", "") - p = Path(path) - if not p.exists() or p.parent != _svc().paths.log_dir: - return jsonify({"ok": False, "message": "File not found."}), 404 - return send_file(p, as_attachment=True) - - -@api.post("/config") -def set_config(): - body = request.get_json(force=True) - svc = _svc() - if "volume_cutoff" in body: - try: - v = float(body["volume_cutoff"]) - if v < 0: - raise ValueError - svc.volume_cutoff_threshold = v - except (TypeError, ValueError): - return jsonify({"ok": False, - "message": "Volume cutoff must be a " - "non-negative number."}), 400 - svc.persist_config() - return jsonify({"ok": True, - "volume_cutoff": svc.volume_cutoff_threshold}) - - -# --------------------------------------------------------------------------- -# Live data -# --------------------------------------------------------------------------- - -@api.post("/live/start") -def live_start(): - body = request.get_json(force=True) - svc = _svc() - result = svc.start_live(body.get("source"), body.get("refresh_rate")) - if not result.get("ok"): - return jsonify(result), 400 - if not svc.is_trading_hour(): - result["warning"] = ("Outside NSE trading hours (09:15–15:30 IST, " - "Mon–Fri). Prices may be stale.") - _scheduler().start(svc.live_refresh_rate) - return jsonify(result) - - -@api.post("/live/stop") -def live_stop(): - _scheduler().stop() - return jsonify(_svc().stop_live()) - - -@api.post("/live/refresh") -def live_refresh_now(): - svc, jobs = _svc(), _jobs() - - async def run(): - import asyncio - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, svc.load_live_data) - - job_id = jobs.submit("Fetching live data...", run) - return jsonify({"ok": True, "job_id": job_id}) - - -# --------------------------------------------------------------------------- -# Watchlists -# --------------------------------------------------------------------------- - -@api.get("/watchlists") -def watchlist_names(): - return jsonify({"names": _svc().watchlist_names, - "slots": WATCHLIST_SLOTS}) - - -@api.post("/watchlists/load/") -def watchlist_load(num): - result = _svc().load_watchlist(num) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/watchlists/load-name") -def watchlist_load_name(): - body = request.get_json(force=True) - result = _svc().load_dashboard_state_by_name(body.get("name", "")) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/watchlists/save") -def watchlist_save(): - result = _svc().save_dashboard() - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/watchlists/save-as") -def watchlist_save_as(): - body = request.get_json(force=True) - result = _svc().save_as_watchlist(body.get("slot"), body.get("name", "")) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/watchlists/add-selected") -def watchlist_add_selected(): - body = request.get_json(force=True) - result = _svc().add_selected_to_watchlist(body.get("slot")) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/watchlists/load-file") -def watchlist_load_file(): - """Arbitrary watchlist JSON: uploaded file or a known live-analysis name.""" - svc = _svc() - known = { - "maxvol50": svc.paths.max_vol_watchlist_file, - "maxprice50": svc.paths.max_price_watchlist_file, - "maxfall50": svc.paths.max_fall_watchlist_file, - "maxscore50": svc.paths.max_score_watchlist_file, - "maxreturn1w50": svc.paths.max_return_files.get("1W"), - "maxreturn1m50": svc.paths.max_return_files.get("1M"), - "maxreturn3m50": svc.paths.max_return_files.get("3M"), - } - name = request.form.get("known") or (request.json or {}).get("known") \ - if not request.files else None - if name: - path = known.get(name) - if not path: - return jsonify({"ok": False, "message": "Unknown analysis file."}), 400 - result = svc.load_watchlist_from_file(path) - return jsonify(result), (200 if result.get("ok") else 400) - f = request.files.get("file") - if not f: - return jsonify({"ok": False, "message": "No file supplied."}), 400 - dest = svc.paths.watchlist_dir / secure_filename(f.filename) - f.save(dest) - result = svc.load_watchlist_from_file(dest) - return jsonify(result), (200 if result.get("ok") else 400) - - -@api.post("/index/") -def load_index(name): - result = _svc().load_index_stocks(name.upper()) - return jsonify(result), (200 if result.get("ok") else 400) - - -# --------------------------------------------------------------------------- -# Analysis (crew-driven, job-based) -# --------------------------------------------------------------------------- - -def _analysis_job(description, crew_builder): - svc, jobs = _svc(), _jobs() - - async def run(): - ctx = await crew_builder().kickoff_async() - a = ctx["analysis"] - return svc.display_analysis_result( - a["symbols"], a["json_name"], a["cached"] - ) - - job_id = jobs.submit(description, run) - return jsonify({"ok": True, "job_id": job_id}) - - -@api.post("/analysis/price") -def analysis_price(): - body = request.get_json(force=True) - duration = body.get("duration") - change_type = body.get("change_type") - try: - threshold = float(body.get("threshold")) - except (TypeError, ValueError): - return jsonify({"ok": False, - "message": "Threshold must be a number."}), 400 - svc, crews = _svc(), _crews() - return _analysis_job( - f"Calculating {duration} {change_type} price changes...", - lambda: crews.price_analysis_crew( - duration, change_type, threshold, svc.selected_date - ), - ) - - -@api.post("/analysis/volume") -def analysis_volume(): - body = request.get_json(force=True) - duration = body.get("duration") - svc, crews = _svc(), _crews() - return _analysis_job( - f"Calculating {duration} volume movers...", - lambda: crews.volume_analysis_crew( - duration, svc.volume_cutoff_threshold - ), - ) - - -@api.post("/analysis/return") -def analysis_return(): - body = request.get_json(force=True) - duration = body.get("duration") - svc, crews = _svc(), _crews() - return _analysis_job( - f"Calculating {duration} max returns...", - lambda: crews.return_analysis_crew(duration, svc.selected_date), - ) - - -@api.post("/analysis/top-score") -def analysis_top_score(): - crews = _crews() - return _analysis_job("Loading top Moneycontrol scores...", - crews.top_score_crew) - - -# --------------------------------------------------------------------------- -# Portfolio / recommender -# --------------------------------------------------------------------------- - -@api.post("/portfolio/import") -def portfolio_import(): - svc, crews, jobs = _svc(), _crews(), _jobs() - - # Save the file from request context if uploaded - f = request.files.get("file") - filepath = None - if f: - session_token = get_session_token() - filename = secure_filename(f.filename) - dest = svc.paths.data_dir / f"uploaded_portfolio_{session_token}.csv" - f.save(dest) - filepath = str(dest) - else: - # Fallback to existing csv version of portfolio file if it exists - csv_path = Path(str(svc.paths.portfolio_file).replace('.parquet', '.csv')) - if csv_path.exists(): - filepath = str(csv_path) - - async def run(): - ctx = await crews.portfolio_crew(filepath).kickoff_async() - p = ctx["portfolio"] - - result = svc.load_portfolio_holdings(p["holdings"], p["unmapped"], p["broker_name"]) - - # Save the raw uploaded CSV to the parquet portfolio_file - if result.get("ok") and filepath: - try: - df_raw = pd.read_csv(filepath) - df_raw.to_parquet(svc.paths.portfolio_file, index=False) - except Exception as e: - logging.warning(f"Failed to save raw portfolio to parquet: {e}") - - # Trigger session backup to database/sync - svc.watchlists._backup() - - # Clean up the temporary uploaded CSV file - try: - Path(filepath).unlink(missing_ok=True) - except Exception as e: - logging.warning(f"Failed to delete temporary uploaded file {filepath}: {e}") - - return result - - job_id = jobs.submit("Importing portfolio...", run) - return jsonify({"ok": True, "job_id": job_id}) - - -@api.post("/recommend") -def recommend(): - svc, crews, jobs = _svc(), _crews(), _jobs() - symbols = [c.symbol for c in svc.cells] - if not symbols: - return jsonify({"ok": False, - "message": "No stocks on the board to analyse."}), 400 - - async def run(): - ctx = await crews.recommendation_crew(symbols).kickoff_async() - recs = ctx["recommendations"] - by_symbol = {r["symbol"]: r for r in recs} - for cell in svc.cells: - cell.recommendation = by_symbol.get(cell.symbol) - return {"ok": True, "recommendations": recs} - - job_id = jobs.submit("Running buy/sell recommender...", run) - return jsonify({"ok": True, "job_id": job_id}) - - -@api.post("/train") -def train(): - crews, jobs = _crews(), _jobs() - - async def run(): - ctx = await crews.training_crew().kickoff_async() - return {"ok": True, **ctx["training"]} - - job_id = jobs.submit("Training LSTM models for all symbols...", run) - return jsonify({"ok": True, "job_id": job_id}) - - -# --------------------------------------------------------------------------- -# Jobs -# --------------------------------------------------------------------------- - -@api.get("/jobs/") -def job_status(job_id): - job = _jobs().get(job_id) - if job is None: - return jsonify({"ok": False, "message": "Unknown job."}), 404 - return jsonify(_jobs().to_payload(job)) - - -# --------------------------------------------------------------------------- -# Utility scripts (interactive web terminal) -# --------------------------------------------------------------------------- - -@api.get("/scripts") -def scripts_list(): - return jsonify({"scripts": [ - {"label": label, "file": file} for label, file in UTILITY_SCRIPTS - ]}) - - -@api.post("/scripts/start") -def scripts_start(): - body = request.get_json(force=True) - file = body.get("file", "") - if file not in {f for _, f in UTILITY_SCRIPTS}: - return jsonify({"ok": False, "message": "Unknown script."}), 400 - svc = _svc() - run = _scripts().start(file, str(svc.paths.base_dir)) - logging.info(f"Started utility script {file} run={run.id}") - return jsonify({"ok": True, "run_id": run.id}) - - -@api.get("/scripts//output") -def scripts_output(run_id): - run = _scripts().get(run_id) - if run is None: - return jsonify({"ok": False, "message": "Unknown run."}), 404 - after = request.args.get("after", -1, type=int) - return jsonify({"ok": True, "lines": run.lines_since(after), - "finished": run.finished, - "returncode": run.returncode}) - - -@api.post("/scripts//input") -def scripts_input(run_id): - run = _scripts().get(run_id) - if run is None: - return jsonify({"ok": False, "message": "Unknown run."}), 404 - body = request.get_json(force=True) - ok = run.send_input(body.get("text", "")) - return jsonify({"ok": ok}) - - -@api.get("/status") -def server_status(): - svc = _svc() - exiting = getattr(svc, "is_exiting", False) - is_hf = "SPACE_ID" in os.environ - from ..core.nvidia_llm import get_nvidia_api_status - return jsonify({"ok": True, "exiting": exiting, "is_hf": is_hf, - "nvidia_status": get_nvidia_api_status()}) - - -@api.post("/shutdown") -def shutdown(): - is_hf = "SPACE_ID" in os.environ - if is_hf: - logging.info("Hugging Face mode: /shutdown cleans up current user session.") - userid = get_session_token() - extensions = current_app.extensions["nsedash"] - services = extensions["services"] - base_dir = extensions["base_dir"] - hf_sync = extensions.get("hf_sync") - from ..config import Paths - import shutil - - if userid and userid not in ("admin", "default"): - logging.info("Purging session '%s' assets on UI exit...", userid) - print(f"[SESSION] Disconnected (User Exit): {userid}", flush=True) - global total_disconnections_this_hour - total_disconnections_this_hour += 1 - - for tok in user_auth.SessionManager.get_userid_sessions(userid): - user_auth.SessionManager.destroy_session(tok) - - if userid in services: - del services[userid] - _active_sessions.pop(userid, None) - - paths = Paths(base_dir=base_dir, session_token=userid) - try: - for f in (paths.state_file, paths.config_file, paths.portfolio_file): - if f.exists(): - f.unlink() - if paths.watchlist_dir.exists(): - shutil.rmtree(paths.watchlist_dir) - except Exception as exc: - logging.error("Error purging session '%s' assets: %s", userid, exc) - - if hf_sync: - hf_sync.delete_session_files(userid) - - return jsonify({"ok": True, "message": "UI session closed successfully."}) - - logging.info("Initiating graceful shutdown via API...") - - # Mark as exiting - try: - _svc().is_exiting = True - except Exception: - pass - - # Trigger SIGINT process termination (handled by register_shutdown_handler in nsedash_app.py) - import sys - is_testing = current_app.testing or current_app.config.get("TESTING") or "pytest" in sys.modules - if not is_testing: - def kill_self(): - time.sleep(0.1) - logging.info("Triggering SIGINT signal for process shutdown.") - os.kill(os.getpid(), signal.SIGINT) - threading.Thread(target=kill_self, daemon=True).start() - else: - # Under tests, we manually run cleanup since we won't SIGINT the test process - logging.info("Testing mode: performing manual cleanup instead of SIGINT.") - try: - _scheduler().stop() - except Exception: - pass - try: - scripts = _scripts() - for run in list(scripts._runs.values()): - if run.process and run.process.poll() is None: - run.process.terminate() - try: - run.process.wait(timeout=1.0) - except Exception: - run.process.kill() - except Exception: - pass - try: - runtime = current_app.extensions["nsedash"]["runtime"] - runtime.shutdown() - except Exception: - pass - - return jsonify({"ok": True, "message": "Backend shutting down..."}) - - -# --------------------------------------------------------------------------- -# Admin API endpoints -# --------------------------------------------------------------------------- - -def is_admin_request() -> bool: - token = (request.headers.get("X-Session-Token") - or request.cookies.get("session_token") - or request.args.get("session_token")) - if not token: - return False - valid, userid = user_auth.SessionManager.validate_session(token) - if not valid: - return False - if userid == "admin": - return True - u = user_auth.UserDatabase.get_user(userid) - return bool(u and u.get("is_admin", False)) - - -@api.get("/admin/dashboard") -def admin_dashboard(): - if not is_admin_request(): - return jsonify({"error": "Unauthorized"}), 403 - - all_users = user_auth.UserDatabase.get_all_users() - - # Calculate some metrics - extensions = current_app.extensions["nsedash"] - service = extensions.get("service") or _svc() - - from ..config import NIFTY_STOCKS, SENSEX_STOCKS - total_symbols = len(set(NIFTY_STOCKS + SENSEX_STOCKS)) - - # Check yfinance connection status (parity with mlbench) - yfinance_ok = True - try: - import requests - r = requests.get( - "https://query1.finance.yahoo.com/v8/finance/chart/RELIANCE.NS?interval=1d&range=1d", - headers={"User-Agent": "Mozilla/5.0"}, - timeout=1.5 - ) - yfinance_ok = (r.status_code == 200) - except Exception: - yfinance_ok = False - - return jsonify({ - "ok": True, - "total_users": len(all_users), - "total_symbols": total_symbols, - "yfinance_connected": yfinance_ok, - "users": all_users, - }) - - -@api.post("/admin/action") -def admin_action(): - if not is_admin_request(): - return jsonify({"error": "Unauthorized"}), 403 - - body = request.get_json(force=True) - action = body.get("action") - target_uid = body.get("target_uid") - - admin_user = user_auth.SessionManager.validate_session( - request.headers.get("X-Session-Token") or request.cookies.get("session_token") - )[1] - - if action == "add_user": - username = body.get("username", "").strip() - role = body.get("role", "user") - if not username: - return jsonify({"ok": False, "message": "Username is required."}), 400 - is_adm = (role == "admin") - created = user_auth.UserDatabase.create_user(username, is_admin=is_adm) - if created: - user_auth.ActivityLogger.log_activity(admin_user, "add_user", "success", f"Created user {username} with role {role}") - return jsonify({"ok": True, "message": f"User '{username}' created successfully."}) - return jsonify({"ok": False, "message": "Failed to create user. User may already exist."}), 400 - - elif action == "qr": - qr_uri = user_auth.AuthenticationService.get_totp_qr(target_uid) - user = user_auth.UserDatabase.get_user(target_uid) or {} - secret = user.get("totp_secret", "Not Set") - return jsonify({"ok": True, "qr": qr_uri, "secret": secret}) - - elif action == "metrics": - user = user_auth.UserDatabase.get_user(target_uid) - if not user: - return jsonify({"ok": False, "message": "User not found"}), 404 - - # Load user's data to check symbols/watchlists - user_data = user_auth.get_user_data(target_uid) - state_data = user_data.get("state") or {} - symbols_count = len(state_data.get("symbols", [])) - watchlists_count = len(user_data.get("watchlists", {})) - - return jsonify({ - "ok": True, - "userid": target_uid, - "login_count": user.get("login_count", 0), - "last_login": user.get("last_login", "Never"), - "symbols_count": symbols_count, - "watchlists_count": watchlists_count, - }) - - elif action == "reset": - if target_uid == admin_user: - return jsonify({"ok": False, "message": "Cannot reset your own account while logged in."}), 400 - res = user_auth.UserDatabase.reset_totp(target_uid) - if res: - user_auth.SessionManager.clear_sessions_for_user(target_uid) - user_auth.ActivityLogger.log_activity(admin_user, "reset_account", "success", f"Full reset for {target_uid}") - return jsonify({"ok": True, "message": f"Full account reset for {target_uid}. They have been logged out."}) - return jsonify({"ok": False, "message": "Failed to reset account."}), 400 - - elif action == "lock": - if target_uid == admin_user: - return jsonify({"ok": False, "message": "Cannot lock your own admin account."}), 400 - user = user_auth.UserDatabase.get_user(target_uid) - if user: - new_status = not user.get("active", True) - user_auth.UserDatabase.update_user(target_uid, {"active": new_status}) - status_str = "unlocked" if new_status else "locked" - user_auth.ActivityLogger.log_activity(admin_user, "lock_user", "success", f"Toggled status of {target_uid} to {status_str}") - return jsonify({"ok": True, "message": f"User '{target_uid}' account status toggled to {status_str}."}) - return jsonify({"ok": False, "message": "User not found."}), 404 - - elif action == "delete": - if target_uid == admin_user: - return jsonify({"ok": False, "message": "Cannot delete your own admin account."}), 400 - res = user_auth.UserDatabase.delete_user(target_uid) - if res: - user_auth.ActivityLogger.log_activity(admin_user, "delete_user", "success", f"Permanently deleted user {target_uid}") - return jsonify({"ok": True, "message": f"User '{target_uid}' has been deleted."}) - return jsonify({"ok": False, "message": "Failed to delete user."}), 400 - - elif action == "mirror": - logs = user_auth.ActivityLogger.get_activity_logs(target_uid, limit=20) - return jsonify({"ok": True, "logs": logs}) - - elif action == "chat": - # Port chat functionality: return simulated message log with the user - msgs = [ - {"sender": "system", "text": f"Connected to {target_uid} session channel..."}, - {"sender": "system", "text": "No live messages in queue. Showing connection log."} - ] - return jsonify({"ok": True, "messages": msgs}) - - return jsonify({"ok": False, "message": "Unknown action."}), 400 - - -@api.get("/stocks//ai-analysis") -def get_ai_analysis(symbol: str): - """ - Get or generate AI-driven stock analysis in markdown format using the 5-pipeline - agentic research analyst (nsedash_research.py). Non-blocking background mode. - """ - token = get_session_token() - base_dir = current_app.extensions["nsedash"]["base_dir"] - force_refresh = request.args.get("force", "0") == "1" - - # Extract user-selected model and timeout - req_json = request.get_json(silent=True) or {} - selected_model = request.args.get("model") or req_json.get("model") or "google/diffusiongemma-26b-a4b-it" - timeout_raw = request.args.get("timeout") or req_json.get("timeout") or 5 - try: - timeout_mins = int(timeout_raw) - timeout_sec = timeout_mins * 60 if timeout_mins <= 60 else timeout_mins - except (ValueError, TypeError): - timeout_sec = 300 - - import sys - sys_path = str(Path(base_dir)) - if sys_path not in sys.path: - sys.path.insert(0, sys_path) - from nsedash_research import NSEMarketResearchAnalyst - - analyst = NSEMarketResearchAnalyst(base_dir=base_dir) - cache_file = analyst.aianalysis_dir / f"{symbol.upper()}_agentic.md" - - # If cached and not force_refresh, return immediately! - if not force_refresh and cache_file.exists(): - try: - mtime = cache_file.stat().st_mtime - if time.time() - mtime < 24 * 3600: - result = analyst.run_full_research_pipeline(symbol, force_refresh=False, selected_model=selected_model, timeout_seconds=timeout_sec) - return jsonify(result) - except Exception: - pass - - # Submit background job to prevent blocking Flask worker / UI thread - jobs = _jobs() - - async def run(): - res = analyst.run_full_research_pipeline(symbol, force_refresh=True, selected_model=selected_model, timeout_seconds=timeout_sec) - # Sync with HF dataset if enabled - from ..config import Paths - paths = Paths(base_dir=base_dir, session_token=token) - file_path = paths.aianalysis_dir / f"{symbol}.md" - hf_sync = current_app.extensions["nsedash"].get("hf_sync") - if hf_sync and file_path.exists(): - hf_sync.push_file_async(file_path) - return res - - job_id = jobs.submit(f"AI Agentic Research for {symbol}", run) - return jsonify({"ok": True, "job_id": job_id, "symbol": symbol, "status": "running", "cached": False, "selected_model": selected_model, "timeout_seconds": timeout_sec}) - - -@api.post("/stocks//ai-analysis/start") -def start_ai_analysis(symbol: str): - """Explicitly trigger fresh background AI agentic research.""" - return get_ai_analysis(symbol) - - - - diff --git a/nsedash/web/background.py b/nsedash/web/background.py deleted file mode 100644 index b1088a3bbff67866fb14f9bbdbcef0463329d185..0000000000000000000000000000000000000000 --- a/nsedash/web/background.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -nsedash.web.background -====================== -Async runtime for the web application. - -- ``AsyncRuntime``: a dedicated thread running an asyncio event loop. All - crews and long operations execute here, so Flask request handlers never - block on analysis, scraping, or training (the web equivalent of v4.5's - ``_run_long_operation`` daemon threads + wait dialogs). -- ``JobManager``: submit coroutine factories, poll status/result by job id. - The frontend polls ``/api/jobs/`` and shows a spinner — the UX role of - the v4.5 ``wait_dialog_decorator``. -- ``LiveRefreshScheduler``: asyncio task replicating the ``root.after`` - polling loop (fetch immediately, then every N minutes while active). -- ``ScriptRunner``: interactive subprocess execution with a scrollback - buffer and stdin injection — the web replacement for v4.5's - ``run_script_in_shell`` Tk shell window. -""" - -from __future__ import annotations - -import asyncio -import itertools -import logging -import os -import subprocess -import sys -import threading -import time -import uuid -from dataclasses import dataclass, field -from typing import Any, Callable, Coroutine - - -class AsyncRuntime: - """Dedicated asyncio loop in a daemon thread.""" - - def __init__(self): - self.loop = asyncio.new_event_loop() - self._thread = threading.Thread( - target=self._run, name="nsedash-async", daemon=True - ) - self._thread.start() - - def _run(self): - asyncio.set_event_loop(self.loop) - self.loop.run_forever() - - def submit(self, coro: Coroutine) -> "asyncio.Future": - return asyncio.run_coroutine_threadsafe(coro, self.loop) - - def shutdown(self): - self.loop.call_soon_threadsafe(self.loop.stop) - - -@dataclass -class Job: - id: str - description: str - status: str = "running" # running | done | error - result: Any = None - error: str | None = None - created_at: float = field(default_factory=time.time) - finished_at: float | None = None - - -class JobManager: - - def __init__(self, runtime: AsyncRuntime): - self.runtime = runtime - self._jobs: dict[str, Job] = {} - self._lock = threading.Lock() - - def submit(self, description: str, - coro_factory: Callable[[], Coroutine]) -> str: - job = Job(id=uuid.uuid4().hex[:12], description=description) - with self._lock: - self._jobs[job.id] = job - - async def runner(): - try: - job.result = await coro_factory() - job.status = "done" - except Exception as e: - logging.error(f"Job '{description}' failed: {e}") - job.status = "error" - job.error = str(e) - finally: - job.finished_at = time.time() - - self.runtime.submit(runner()) - return job.id - - def get(self, job_id: str) -> Job | None: - with self._lock: - return self._jobs.get(job_id) - - def to_payload(self, job: Job) -> dict: - return { - "id": job.id, "description": job.description, - "status": job.status, "result": job.result, "error": job.error, - } - - -class LiveRefreshScheduler: - """Replicates the v4.5 root.after() live polling loop with asyncio and triggers background nsemaxlive.py.""" - - def __init__(self, runtime: AsyncRuntime, refresh_callable: Callable[[], dict], base_dir: Any = None): - self.runtime = runtime - self.refresh_callable = refresh_callable - self.base_dir = str(base_dir) if base_dir else None - self._task: asyncio.Task | None = None - self._interval_min: int = 5 - - @property - def active(self) -> bool: - return self._task is not None and not self._task.done() - - def _run_maxlive_script(self): - if not self.base_dir: - return - script_path = os.path.join(self.base_dir, "nsemaxlive.py") - if os.path.exists(script_path): - try: - logging.info("[LiveScheduler] Triggering background nsemaxlive.py execution...") - res = subprocess.run([sys.executable, "-u", script_path], cwd=self.base_dir, timeout=300, capture_output=True, text=True) - if res.returncode == 0: - logging.info("[LiveScheduler] Background nsemaxlive.py finished successfully.") - else: - logging.warning(f"[LiveScheduler] Background nsemaxlive.py exited with code {res.returncode}: {res.stderr[:200]}") - except Exception as exc: - logging.error(f"[LiveScheduler] Error running background nsemaxlive.py: {exc}") - - def start(self, interval_min: int) -> None: - self.stop() - self._interval_min = max(1, int(interval_min)) - - async def poll_loop(): - loop = asyncio.get_running_loop() - while True: - try: - # Blocking scrape work runs in executor - await loop.run_in_executor(None, self.refresh_callable) - except Exception as e: - logging.error(f"Live refresh cycle failed: {e}") - - try: - # Run nsemaxlive.py in background executor - await loop.run_in_executor(None, self._run_maxlive_script) - except Exception as e: - logging.error(f"Live maxlive background script failed: {e}") - - await asyncio.sleep(self._interval_min * 60) - - def create(): - self._task = self.runtime.loop.create_task(poll_loop()) - - self.runtime.loop.call_soon_threadsafe(create) - - def stop(self) -> None: - task = self._task - self._task = None - if task and not task.done(): - self.runtime.loop.call_soon_threadsafe(task.cancel) - logging.info("Periodic live data refresh stopped.") - - -class ScriptRun: - """One interactive subprocess with buffered, tagged output lines.""" - - def __init__(self, script_name: str, script_dir: str): - self.id = uuid.uuid4().hex[:12] - self.script_name = script_name - self.script_dir = script_dir - self._lines: list[dict] = [] - self._seq = itertools.count() - self._lock = threading.Lock() - self.process: subprocess.Popen | None = None - self.returncode: int | None = None - self._start() - - def _log(self, message: str, tag: str = "stdout") -> None: - with self._lock: - self._lines.append( - {"n": next(self._seq), "text": message, "tag": tag} - ) - - def _start(self) -> None: - self._log(f"Starting script from directory: {self.script_dir}", "info") - self._log(f"Executing command: python -u {self.script_name}", "info") - self._log("-" * 80, "info") - - def run(): - try: - creationflags = 0 - if os.name == 'nt': - creationflags = subprocess.CREATE_NO_WINDOW - # AUDIT FIX M-11: use sys.executable, not bare 'python', - # which resolves differently (or not at all) per PATH. - self.process = subprocess.Popen( - [sys.executable, '-u', self.script_name], - cwd=self.script_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE, text=True, - encoding='utf-8', errors='replace', - bufsize=1, creationflags=creationflags, - ) - - def read_pipe(pipe, tag): - for line in iter(pipe.readline, ''): - self._log(line.rstrip("\n"), tag) - pipe.close() - - t_out = threading.Thread( - target=read_pipe, args=(self.process.stdout, "stdout"), - daemon=True) - t_err = threading.Thread( - target=read_pipe, args=(self.process.stderr, "error"), - daemon=True) - t_out.start(); t_err.start() - rc = self.process.wait() - t_out.join(); t_err.join() - self.returncode = rc - self._log("-" * 80, "info") - if rc == 0: - self._log( - f"Process '{self.script_name}' finished successfully " - f"(Exit Code: {rc}).", "success") - else: - self._log( - f"Process '{self.script_name}' finished with error " - f"(Exit Code: {rc}).", "error") - except FileNotFoundError: - self.returncode = -1 - self._log( - f"Error: Script '{self.script_name}' not found in " - f"'{self.script_dir}'.", "error") - except Exception as e: - self.returncode = -1 - self._log( - f"Unexpected error running '{self.script_name}': {e}", - "error") - finally: - self._log("Input disabled. Process has finished.", "info") - - threading.Thread(target=run, daemon=True).start() - - def send_input(self, text: str) -> bool: - text = (text or "").strip() - if not text or self.process is None or self.process.poll() is not None: - return False - try: - self._log(f"> {text}", "input") - self.process.stdin.write(text + '\n') - self.process.stdin.flush() - return True - except (IOError, BrokenPipeError) as e: - self._log(f"Error: Could not write to process. {e}", "error") - return False - - def lines_since(self, after: int = -1) -> list[dict]: - with self._lock: - return [ln for ln in self._lines if ln["n"] > after] - - @property - def finished(self) -> bool: - return self.returncode is not None - - -class ScriptRunner: - - def __init__(self): - self._runs: dict[str, ScriptRun] = {} - - def start(self, script_name: str, script_dir: str) -> ScriptRun: - run = ScriptRun(script_name, script_dir) - self._runs[run.id] = run - return run - - def get(self, run_id: str) -> ScriptRun | None: - return self._runs.get(run_id) diff --git a/nsedash/web/routes.py b/nsedash/web/routes.py deleted file mode 100644 index 7b9219d44058fb319b5987bfcf8edcb8a4616121..0000000000000000000000000000000000000000 --- a/nsedash/web/routes.py +++ /dev/null @@ -1,14 +0,0 @@ -import uuid -from flask import Blueprint, render_template, request, make_response - -pages = Blueprint("pages", __name__) - - -@pages.get("/") -def index(): - token = request.cookies.get("session_token") - response = make_response(render_template("index.html")) - if not token: - token = uuid.uuid4().hex[:12] - response.set_cookie("session_token", token, max_age=30*86400, httponly=True, samesite='Lax', secure=request.is_secure) - return response diff --git a/nsedash/web/static/app.js b/nsedash/web/static/app.js deleted file mode 100644 index 928e0e357bfe4a635716434c2c34b152496a0a05..0000000000000000000000000000000000000000 --- a/nsedash/web/static/app.js +++ /dev/null @@ -1,1548 +0,0 @@ -/* NSE Stock Dashboard — frontend controller. - * Mirrors every v4.5 desktop interaction against the REST API. */ - -"use strict"; - -const $ = (sel) => document.querySelector(sel); -const state = { board: null, charts: {}, livePoll: null }; - -function esc(s) { - if (s === null || s === undefined) return ""; - return String(s) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """) - .replace(/'/g, "'"); -} - -/* ---------------------------------------------------------------- utils */ - -async function api(path, opts = {}) { - const token = localStorage.getItem("session_token") || ""; - const headers = { - "Content-Type": "application/json", - "X-Session-Token": token, - ...(opts.headers || {}) - }; - const res = await fetch("/api" + path, { - ...opts, - headers, - }); - let body = {}; - try { body = await res.json(); } catch (e) { /* empty */ } - if (res.status === 401 && body.auth_required) { - showLoginBox(); - if (body.reason === "inactivity") { - toast("Session terminated due to inactivity.", "err"); - const loginError = $("#login-error"); - if (loginError) { - loginError.textContent = "Session terminated due to inactivity."; - loginError.classList.remove("hidden"); - } - } - return body; - } - if (!res.ok && body.message) toast(body.message, "err"); - return body; -} - -function toast(msg, kind = "") { - const el = document.createElement("div"); - el.className = "toast " + kind; - el.textContent = msg; - $("#toasts").appendChild(el); - setTimeout(() => el.remove(), 6000); -} - -function fmt(n, dp = 2) { - if (n === null || n === undefined || isNaN(n)) return "0.00"; - return Number(n).toLocaleString("en-IN", - { minimumFractionDigits: dp, maximumFractionDigits: dp }); -} - -function arrow(v) { return v > 0 ? "↑" : v < 0 ? "↓" : "↔"; } -function cls(v) { return v > 0 ? "gain" : v < 0 ? "loss" : "flat"; } - -function scoreClass(s) { - if (s >= 80) return "score-80"; - if (s >= 60) return "score-60"; - if (s >= 40) return "score-40"; - if (s >= 20) return "score-20"; - return "score-0"; -} - -/* Busy overlay + job polling (v4.5 wait dialog equivalent) */ -function busy(text) { - $("#busy-text").textContent = text || "Processing…"; - $("#busy").classList.remove("hidden"); -} -function unbusy() { $("#busy").classList.add("hidden"); } - -async function runJob(startPromise, description) { - busy(description); - try { - const start = await startPromise; - if (!start.job_id) { unbusy(); return start; } - while (true) { - await new Promise((r) => setTimeout(r, 700)); - const job = await api("/jobs/" + start.job_id); - if (job.status === "done") { unbusy(); return job.result || {}; } - if (job.status === "error") { - unbusy(); - toast(job.error || "Operation failed.", "err"); - return { ok: false, message: job.error }; - } - } - } catch (e) { - unbusy(); - toast(String(e), "err"); - return { ok: false }; - } -} - -/* ---------------------------------------------------------------- board */ - -async function loadBoard() { - state.board = await api("/board"); - render(); -} - -function render() { - const b = state.board; - if (!b) return; - - // Banner - const banner = $("#banner"); - banner.className = "banner " + (b.banner.mode === "live" ? "live" : "historic"); - $("#banner-text").textContent = b.banner.mode === "live" - ? `Live Data — ${b.banner.source} (every ${b.banner.refresh_rate} min)` - : "Historic Data"; - $("#banner-watchlist").textContent = - b.banner.watchlist + (b.watchlist_modified ? " *" : ""); - $("#live-toggle").checked = b.banner.mode === "live"; - $("#date-input").value = b.selected_date; - $("#date-input").max = new Date().toISOString().slice(0, 10); - - // Search datalist - const dl = $("#stock-list"); - dl.innerHTML = ""; - for (const key of b.search_keys) { - const o = document.createElement("option"); - o.value = key; - dl.appendChild(o); - } - $("#board-count").textContent = `${b.cells.length} stocks on board`; - - // Live index badges (NIFTY & SENSEX) - if (b.indices) { - if (b.indices.NIFTY) { - const n = b.indices.NIFTY; - const isPos = n.change >= 0; - const sym = isPos ? "▲" : "▼"; - const el = $("#nifty-val"); - if (el) { - el.textContent = `${sym} ${n.formatted_price} (${n.formatted_change})`; - el.className = `idx-pill ${isPos ? "positive" : "negative"}`; - } - } - if (b.indices.SENSEX) { - const s = b.indices.SENSEX; - const isPos = s.change >= 0; - const sym = isPos ? "▲" : "▼"; - const el = $("#sensex-val"); - if (el) { - el.textContent = `${sym} ${s.formatted_price} (${s.formatted_change})`; - el.className = `idx-pill ${isPos ? "positive" : "negative"}`; - } - } - } - - renderPortfolio(b.portfolio); - renderMenus(b); - renderGrid(b.cells); -} - -/* RTC Clock in IST Timezone */ -function updateRTC() { - try { - const options = { - timeZone: "Asia/Kolkata", - year: "numeric", - month: "short", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - hour12: true - }; - const istString = new Date().toLocaleString("en-IN", options) + " IST"; - const rtcEl = document.getElementById("rtc-time"); - if (rtcEl) rtcEl.textContent = istString; - } catch (e) { - console.error("RTC clock error:", e); - } -} -setInterval(updateRTC, 1000); -updateRTC(); - -function renderPortfolio(p) { - if (!p) return; - $("#pf-invested").textContent = fmt(p.invested); - $("#pf-current").textContent = fmt(p.current_value); - const day = $("#pf-day"); - day.textContent = `${fmt(p.day_gain_abs)} (${fmt(p.day_gain_pct)}%)`; - day.className = cls(p.day_gain_abs); - const abs = $("#pf-abs"); - abs.textContent = `${fmt(p.abs_returns_abs)} (${fmt(p.abs_returns_pct)}%)`; - abs.className = cls(p.abs_returns_abs); -} - -/* ---------------------------------------------------------------- menus */ - -function menuButton(label, fn) { - const btn = document.createElement("button"); - btn.textContent = label; - btn.setAttribute("role", "menuitem"); - btn.addEventListener("click", () => { closeMenus(); fn(); }); - return btn; -} - -function renderMenus(b) { - // Watchlists: load 1..20, load from file, live analysis quick loads - const wl = $("#menu-watchlists"); - wl.innerHTML = ""; - b.watchlist_names.forEach((name, i) => { - const isPrivate = i < 5; - const label = isPrivate ? `🔒 ${i + 1}. ${name}` : `${i + 1}. ${name}`; - const btn = menuButton(label, () => loadWatchlist(i + 1)); - if (isPrivate) { - btn.style.color = "#10b981"; - btn.style.fontWeight = "bold"; - } - wl.appendChild(btn); - }); - wl.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - wl.appendChild(menuButton("Load watchlist from file…", pickWatchlistFile)); - - // Max/Min price analysis - const mm = $("#menu-maxmin"); - mm.innerHTML = ""; - for (const d of ["1D", "1W", "1M", "3M"]) { - for (const t of ["Max", "Min"]) { - mm.appendChild(menuButton(`${d} ${t} Price Change`, - () => priceAnalysis(d, t))); - } - } - - // Volume analysis - const vol = $("#menu-volume"); - vol.innerHTML = ""; - for (const d of ["1D", "1W", "1M", "3M"]) { - vol.appendChild(menuButton(`${d} Max Volume`, () => volumeAnalysis(d))); - } - vol.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - vol.appendChild(menuButton( - `Set Volume Cutoff (now ${fmt(b.volume_cutoff, 0)})`, setVolumeCutoff)); - - // Live analysis file loads (top 3 menu items only) - const la = $("#menu-liveanalysis"); - la.innerHTML = ""; - la.appendChild(menuButton("Top 50 Volume Gainers (maxvol50)", - () => loadKnownFile("maxvol50"))); - la.appendChild(menuButton("Top 50 Price Gainers (maxprice50)", - () => loadKnownFile("maxprice50"))); - la.appendChild(menuButton("Top 50 Price Losers (maxfall50)", - () => loadKnownFile("maxfall50"))); - - // Simulation - const sim = $("#menu-simulation"); - sim.innerHTML = ""; - sim.appendChild(menuButton("Top 50 Score (maxscore50)", - () => loadKnownFile("maxscore50"))); - sim.appendChild(menuButton("Top Score (Moneycontrol)", topScore)); - for (const d of ["1W", "1M", "3M"]) { - sim.appendChild(menuButton(`${d} Max Return`, () => returnAnalysis(d))); - } - sim.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - sim.appendChild(menuButton("Buy/Sell Recommender", recommend)); - - // Utilities - loadUtilitiesMenu(); -} - -async function loadUtilitiesMenu() { - const u = $("#menu-utilities"); - if (!u) return; - - // Immediately render static items so Admin Control Panel is instantly available - u.innerHTML = ""; - u.appendChild(menuButton("Deep Learning: Train All Models", trainModels)); - u.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - u.appendChild(menuButton("Admin Control Panel", openAdminPanel)); - - try { - const res = await api("/scripts"); - if (res && res.scripts) { - u.innerHTML = ""; - (res.scripts || []).forEach((s) => { - u.appendChild(menuButton(s.label, () => startScript(s))); - }); - u.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - u.appendChild(menuButton("Deep Learning: Train All Models", trainModels)); - u.appendChild(Object.assign(document.createElement("div"), { className: "sep" })); - u.appendChild(menuButton("Admin Control Panel", openAdminPanel)); - } - } catch (err) { - console.error("Error loading utility scripts:", err); - } -} - -function closeMenus() { - document.querySelectorAll(".menu.open") - .forEach((m) => { - m.classList.remove("open"); - const btn = m.querySelector(".menu-btn"); - if (btn) btn.setAttribute("aria-expanded", "false"); - }); -} -document.querySelectorAll(".menu-btn").forEach((btn) => { - btn.setAttribute("aria-haspopup", "true"); - btn.setAttribute("aria-expanded", "false"); - btn.addEventListener("click", (e) => { - if (btn.id === "btn-exit") { - e.stopPropagation(); - triggerExit(); - return; - } - const menu = e.target.closest(".menu"); - const wasOpen = menu.classList.contains("open"); - closeMenus(); - if (!wasOpen) { - menu.classList.add("open"); - btn.setAttribute("aria-expanded", "true"); - } - e.stopPropagation(); - }); -}); -document.addEventListener("click", closeMenus); - -/* ---------------------------------------------------------------- cards */ - -function renderGrid(cells) { - // Destroy old charts to avoid canvas leaks - Object.values(state.charts).forEach((c) => c.destroy()); - state.charts = {}; - const grid = $("#grid"); - grid.innerHTML = ""; - for (const c of cells) grid.appendChild(cardEl(c)); -} - -/* ------------------------------------------------------------- AI Agentic Research */ -let currentAISymbol = null; -let currentAIAnalysisText = ""; - -async function openAIAnalysisModal(symbol, forceRefresh = false) { - currentAISymbol = symbol; - $("#modal-ai").classList.remove("hidden"); - $("#ai-title").innerHTML = `🤖 Agentic AI Stock Research: ${esc(symbol)}`; - - // Show spinner initially - $("#ai-content").innerHTML = ``; - - const selectedModel = $("#select-primary-llm") ? $("#select-primary-llm").value : "google/diffusiongemma-26b-a4b-it"; - const selectedTimeout = $("#select-llm-timeout") ? $("#select-llm-timeout").value : "5"; - - $("#ai-meta-model").innerHTML = `🤖 LLM: ${esc(selectedModel)}`; - $("#ai-meta-time").innerHTML = `⏱️ Time: --s (Max ${selectedTimeout}m)`; - $("#ai-meta-tokens").innerHTML = `📊 Tokens: In: -- | Out: -- | Total: --`; - - try { - const url = `/stocks/${encodeURIComponent(symbol)}/ai-analysis?model=${encodeURIComponent(selectedModel)}&timeout=${encodeURIComponent(selectedTimeout)}${forceRefresh ? '&force=1' : ''}`; - const payload = await api(url); - - if (payload && payload.ok) { - if (payload.analysis) { - // Immediate cached / completed display! - renderAIAnalysisReport(payload); - } else if (payload.job_id && payload.status === "running") { - // Background job submitted! - $("#ai-content").innerHTML = `
- -

5-Pipeline AI Agentic Research in Progress

-

Running 5 research pipelines for ${esc(symbol)} in background using ${esc(selectedModel)} (Timeout: ${selectedTimeout} mins).
You can close this modal and continue navigating NSEDASH. Live status is shown at bottom right.

-
`; - pollAIAgentBackground(symbol, payload.job_id); - } - } else { - $("#ai-content").innerHTML = `Error: ${esc((payload && payload.message) || "Failed to start AI analysis")}`; - } - } catch (err) { - $("#ai-content").innerHTML = `Error: ${esc(err.message)}`; - } -} - -function renderAIAnalysisReport(payload) { - currentAIAnalysisText = payload.analysis || ""; - $("#ai-content").innerHTML = marked.parse(currentAIAnalysisText); - - const modelName = payload.llm_model || "z-ai/glm-5.2"; - const elapsed = payload.elapsed_seconds !== undefined ? `${payload.elapsed_seconds}s` : "--s"; - const pt = payload.prompt_tokens || 0; - const ct = payload.completion_tokens || 0; - const tt = payload.total_tokens || (pt + ct); - - $("#ai-meta-model").innerHTML = `🤖 LLM: ${esc(modelName)}`; - $("#ai-meta-time").innerHTML = `⏱️ Time: ${elapsed}`; - $("#ai-meta-tokens").innerHTML = `📊 Tokens: In: ${fmt(pt, 0)} | Out: ${fmt(ct, 0)} | Total: ${fmt(tt, 0)}`; -} - -function pollAIAgentBackground(symbol, jobId) { - const widget = $("#ai-bg-widget"); - const spinner = $("#ai-bg-spinner"); - const text = $("#ai-bg-text"); - const viewBtn = $("#ai-bg-view-btn"); - - if (!widget) return; - widget.classList.remove("hidden"); - spinner.style.display = "block"; - text.textContent = `🤖 AI Agentic Research Running for ${symbol}...`; - viewBtn.style.display = "none"; - - const startTime = Date.now(); - const pollInterval = setInterval(async () => { - // Timeout safeguard after 310s - if (Date.now() - startTime > 310000) { - clearInterval(pollInterval); - text.textContent = `❌ AI Research for ${symbol} timed out`; - spinner.style.display = "none"; - setTimeout(() => widget.classList.add("hidden"), 8000); - return; - } - - try { - const job = await api("/jobs/" + jobId); - if (!job) return; - - if (job.status === "done") { - clearInterval(pollInterval); - const res = job.result || {}; - text.textContent = `✔ AI Agentic Research Completed for ${symbol}!`; - spinner.style.display = "none"; - viewBtn.style.display = "inline-block"; - viewBtn.onclick = () => { - openAIAnalysisModal(symbol, false); - widget.classList.add("hidden"); - }; - - toast(`AI Agentic Research report ready for ${symbol}`, "ok"); - - // If modal is currently open for this symbol, render immediately! - if (currentAISymbol === symbol && !$("#modal-ai").classList.contains("hidden")) { - renderAIAnalysisReport(res); - } - } else if (job.status === "error") { - clearInterval(pollInterval); - text.textContent = `❌ AI Research failed for ${symbol}`; - spinner.style.display = "none"; - toast(job.error || `AI Research failed for ${symbol}`, "err"); - setTimeout(() => widget.classList.add("hidden"), 8000); - } - } catch (e) { - console.error("Error polling AI job:", e); - } - }, 1500); -} - -function cardEl(c) { - const card = document.createElement("div"); - card.className = "card"; - const ch = c.changes || {}; - const mc = c.mc || {}; - const pl = (c.quantity * c.current_cmp) - (c.quantity * c.price); - const plPct = c.quantity * c.price - ? (pl / (c.quantity * c.price)) * 100 : 0; - - card.innerHTML = ` - -

${esc(c.symbol)}

-
${esc(c.security || "")} — ${esc(c.tran_date || "")}
-
Prev / Curr - ${fmt(c.prev_close)} / ${fmt(c.close_price)}
-
MCap / FV / Score - ${fmt(mc.mcap)} / ${fmt(mc.fv, 0)} / - ${fmt(mc.score, 0)}
-
52wk Hi (Δ) - ${fmt(c.hi_52wk)} (${fmt(c.hi_diff)}%)
-
52wk Lo (Δ) - ${fmt(c.lo_52wk)} (${fmt(c.lo_diff)}%)
-
ATH / ATL - ${fmt(mc.ath)} / ${fmt(mc.atl)}
-
- ${["1D", "1W", "1M", "3M"].map((k) => - `${k} ${arrow(ch[k])} ${fmt(ch[k])}%` - ).join("")} -
-
- ${(c.volume_changes || []).map((v, i) => - `V${i + 1} ${fmt(v)}%` - ).join("")} -
- - ${c.live ? `
- LIVE ${fmt(c.live.price)} - (${arrow(c.live.pct_change)} ${fmt(c.live.pct_change)}%) -  H ${fmt(c.live.day_high)} L ${fmt(c.live.day_low)} -  Vol ${arrow(c.live.vol_change)} ${fmt(c.live.vol_change)}% -
${esc(c.live.updated_at)}
-
` : ""} -
- - -
-
- Cost ${fmt(c.quantity * c.price)} · - CMP ${fmt(c.quantity * c.current_cmp)} · - P&L ${fmt(pl)} (${fmt(plPct)}%) -
-
- - ${c.recommendation - ? `${c.recommendation.action} - (${fmt(c.recommendation.confidence)})` - : ""} -
- - -
-
`; - - card.querySelector(".rm").addEventListener("click", async () => { - await api("/stocks/" + encodeURIComponent(c.symbol), { method: "DELETE" }); - loadBoard(); - }); - card.querySelector(".sel").addEventListener("change", (e) => { - api("/stocks/" + encodeURIComponent(c.symbol) + "/select", { - method: "POST", - body: JSON.stringify({ selected: e.target.checked }), - }); - }); - const pushInputs = async () => { - const r = await api("/stocks/" + encodeURIComponent(c.symbol) + "/inputs", { - method: "POST", - body: JSON.stringify({ - quantity: card.querySelector(".in-qty").value || 0, - price: card.querySelector(".in-price").value || 0, - }), - }); - if (r.ok) { renderPortfolio(r.portfolio); loadBoard(); } - }; - card.querySelectorAll(".in-qty, .in-price").forEach((inp) => { - inp.addEventListener("change", pushInputs); - }); - card.querySelector(".ai-btn").addEventListener("click", () => { - openAIAnalysisModal(c.symbol, false); - }); - - card.querySelector(".mc-btn").addEventListener("click", async () => { - if (mc.link) { window.open(mc.link, "_blank"); return; } - const link = prompt( - `Enter the Moneycontrol URL for ${c.symbol} (${c.security || ""}):`); - if (link === null) return; - const r = await api("/mc-link", { - method: "POST", - body: JSON.stringify({ symbol: c.symbol, security: c.security, link }), - }); - if (r.ok) { toast(r.message, "ok"); loadBoard(); } - }); - - if (c.chart) drawChart(card.querySelector("canvas"), c); - return card; -} - -function drawChart(canvas, c) { - const ch = c.chart; - const datasets = [{ - label: "Close", data: ch.prices, borderColor: "#3b82c4", - borderWidth: 1.6, pointRadius: 0, tension: 0.1, yAxisID: "y", - }]; - const dmaColors = { 9: "#8e44ad", 20: "#d29922", 50: "#16a085", 200: "#e07b70" }; - for (const w of [9, 20, 50, 200]) { - const d = ch["dma_" + w]; - if (d) datasets.push({ - label: w + "DMA", data: d, borderColor: dmaColors[w], - borderWidth: 1, pointRadius: 0, tension: 0.1, yAxisID: "y", - }); - } - if (ch.volumes && ch.volumes.length) { - datasets.push({ - label: "Volume", data: ch.volumes, type: "bar", - backgroundColor: "rgba(125,136,148,.35)", yAxisID: "vol", - }); - } - state.charts[c.symbol] = new Chart(canvas, { - type: "line", - data: { labels: ch.dates, datasets }, - options: { - responsive: true, maintainAspectRatio: true, aspectRatio: 2.2, animation: false, - plugins: { legend: { display: false } }, - scales: { - x: { ticks: { maxTicksLimit: 5, color: "#7d8894" }, grid: { display: false } }, - y: { ticks: { color: "#7d8894" }, grid: { color: "#2a333f" } }, - vol: { position: "right", display: false }, - }, - }, - }); -} - -/* --------------------------------------------------------------- actions */ - -async function loadWatchlist(n) { - const r = await api("/watchlists/load/" + n, { method: "POST" }); - if (r.ok) toast(`Loaded ${r.loaded} stocks from '${r.name}'.`, "ok"); - loadBoard(); -} - -function pickWatchlistFile() { - const inp = document.createElement("input"); - inp.type = "file"; - inp.accept = ".json"; - inp.addEventListener("change", async () => { - if (!inp.files.length) return; - const fd = new FormData(); - fd.append("file", inp.files[0]); - busy("Loading watchlist file…"); - const res = await fetch("/api/watchlists/load-file", - { method: "POST", body: fd }); - const body = await res.json(); - unbusy(); - toast(body.message || "Done", body.ok ? "ok" : "err"); - loadBoard(); - }); - inp.click(); -} - -async function loadKnownFile(name) { - busy("Loading live analysis file…"); - try { - const body = await api("/watchlists/load-file", { - method: "POST", - body: JSON.stringify({ known: name }), - }); - unbusy(); - if (body) { - toast(body.message || "Done", body.ok ? "ok" : "err"); - } - loadBoard(); - } catch (e) { - unbusy(); - toast(String(e), "err"); - } -} - -async function priceAnalysis(duration, changeType) { - const t = prompt( - `Enter the minimum % change threshold for ${duration} ${changeType}:`, - changeType === "Max" ? "5" : "-5"); - if (t === null) return; - const r = await runJob(api("/analysis/price", { - method: "POST", - body: JSON.stringify({ duration, change_type: changeType, threshold: t }), - }), `Calculating ${duration} ${changeType} price changes…`); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -} - -async function volumeAnalysis(duration) { - const r = await runJob(api("/analysis/volume", { - method: "POST", body: JSON.stringify({ duration }), - }), `Calculating ${duration} volume movers…`); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -} - -async function returnAnalysis(duration) { - const r = await runJob(api("/analysis/return", { - method: "POST", body: JSON.stringify({ duration }), - }), `Calculating ${duration} max returns…`); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -} - -async function topScore() { - const r = await runJob(api("/analysis/top-score", { method: "POST" }), - "Loading top Moneycontrol scores…"); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -} - -async function recommend() { - const r = await runJob(api("/recommend", { method: "POST" }), - "Running buy/sell recommender…"); - if (r.recommendations) { - const top = r.recommendations.slice(0, 10) - .map((x) => `${x.symbol}: ${x.action} (${fmt(x.confidence)})`) - .join("\n"); - toast("Recommendations updated.\n" + top, "ok"); - } - loadBoard(); -} - -async function trainModels() { - if (!confirm("Train LSTM models for ALL symbols? This can take a long time.")) - return; - const r = await runJob(api("/train", { method: "POST" }), - "Training LSTM models…"); - if (r.ok) toast(`Training complete for ${r.trained_symbols} symbols.`, "ok"); - loadBoard(); -} - -async function setVolumeCutoff() { - const v = prompt("Minimum average volume for volume analysis:", - state.board ? state.board.volume_cutoff : 10000); - if (v === null) return; - const r = await api("/config", { - method: "POST", body: JSON.stringify({ volume_cutoff: v }), - }); - if (r.ok) { toast(`Volume cutoff set to ${fmt(r.volume_cutoff, 0)}.`, "ok"); loadBoard(); } -} - -/* Scripts terminal */ -let termRun = null, termTimer = null, termLast = -1; - -async function startScript(s) { - const r = await api("/scripts/start", { - method: "POST", body: JSON.stringify({ file: s.file }), - }); - if (!r.ok) return; - termRun = r.run_id; - termLast = -1; - $("#terminal-title").textContent = s.label + " — " + s.file; - $("#terminal-out").innerHTML = ""; - $("#modal-terminal").classList.remove("hidden"); - termTimer = setInterval(pollTerminal, 800); -} - -async function pollTerminal() { - if (!termRun) return; - const r = await api(`/scripts/${termRun}/output?after=${termLast}`); - if (!r.ok) return; - const out = $("#terminal-out"); - for (const ln of r.lines) { - termLast = ln.n; - const span = document.createElement("span"); - span.className = ln.tag; - span.textContent = ln.text + "\n"; - out.appendChild(span); - } - if (r.lines.length) out.scrollTop = out.scrollHeight; - if (r.finished && termTimer) { clearInterval(termTimer); termTimer = null; } -} - -$("#terminal-send").addEventListener("click", sendTerminal); -$("#terminal-in").addEventListener("keydown", (e) => { - if (e.key === "Enter") sendTerminal(); -}); -async function sendTerminal() { - const text = $("#terminal-in").value; - if (!text.trim() || !termRun) return; - await api(`/scripts/${termRun}/input`, { - method: "POST", body: JSON.stringify({ text }), - }); - $("#terminal-in").value = ""; -} -$("#terminal-close").addEventListener("click", () => { - $("#modal-terminal").classList.add("hidden"); - if (termTimer) clearInterval(termTimer); - termTimer = null; -}); - -/* ------------------------------------------------------------- toolbar */ - -$("#btn-add").addEventListener("click", async () => { - const key = $("#stock-search").value.trim(); - if (!key) return; - const r = await api("/stocks", { - method: "POST", body: JSON.stringify({ key }), - }); - if (r.ok) { $("#stock-search").value = ""; loadBoard(); } -}); -$("#stock-search").addEventListener("keydown", (e) => { - if (e.key === "Enter") $("#btn-add").click(); -}); - -$("#btn-sensex").addEventListener("click", async () => { - await api("/index/SENSEX", { method: "POST" }); loadBoard(); -}); -$("#btn-nifty").addEventListener("click", async () => { - await api("/index/NIFTY", { method: "POST" }); loadBoard(); -}); - -$("#btn-portfolio").addEventListener("click", () => { - const inp = document.createElement("input"); - inp.type = "file"; - inp.accept = ".csv"; - inp.addEventListener("change", async () => { - if (!inp.files.length) return; - const fd = new FormData(); - fd.append("file", inp.files[0]); - const token = localStorage.getItem("session_token") || ""; - const startPromise = fetch("/api/portfolio/import", { - method: "POST", - headers: { "X-Session-Token": token }, - body: fd - }).then(res => res.json()); - - const r = await runJob(startPromise, "Importing portfolio…"); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); - }); - inp.click(); -}); - -$("#btn-export").addEventListener("click", async () => { - const r = await api("/export", { method: "POST" }); - if (r.ok) { - toast(r.message, "ok"); - window.location = "/api/export/download?file=" + encodeURIComponent(r.file); - } -}); - -$("#date-input").addEventListener("change", async (e) => { - const r = await api("/date", { - method: "POST", body: JSON.stringify({ date: e.target.value }), - }); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -}); - -/* Live toggle + modal */ -$("#live-toggle").addEventListener("change", (e) => { - if (e.target.checked) { - $("#modal-live").classList.remove("hidden"); - } else { - stopLive(); - } -}); -$("#live-cancel").addEventListener("click", () => { - $("#modal-live").classList.add("hidden"); - $("#live-toggle").checked = false; -}); -async function fetchLiveDataInBackground() { - toast("Fetching live data in background…", "info"); - try { - const start = await api("/live/refresh", { method: "POST" }); - if (!start || !start.job_id) { - if (start && start.message) toast(start.message, "err"); - return; - } - (async () => { - const startTime = Date.now(); - while (Date.now() - startTime < 60000) { - await new Promise((r) => setTimeout(r, 1000)); - const job = await api("/jobs/" + start.job_id); - if (job.status === "done") { - const res = job.result || {}; - toast(res.message || `Live data updated (${res.updated || 0} stocks).`, "ok"); - loadBoard(); - break; - } - if (job.status === "error") { - toast(job.error || "Live data fetch failed.", "err"); - break; - } - } - })(); - } catch (e) { - toast(String(e), "err"); - } -} - -$("#live-ok").addEventListener("click", async () => { - $("#modal-live").classList.add("hidden"); - const r = await api("/live/start", { - method: "POST", - body: JSON.stringify({ - source: $("#live-source").value, - refresh_rate: $("#live-rate").value, - }), - }); - if (!r.ok) { $("#live-toggle").checked = false; return; } - if (r.warning) toast(r.warning); - fetchLiveDataInBackground(); - // Poll the board while live so scheduler updates reach the UI. - if (state.livePoll) clearInterval(state.livePoll); - state.livePoll = setInterval(loadBoard, 10000); -}); - -async function stopLive() { - if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; } - await api("/live/stop", { method: "POST" }); - loadBoard(); -} - -/* Bottom controls */ -$("#btn-refresh").addEventListener("click", async () => { - const r = await runJob(api("/refresh", { method: "POST" }), - "Refreshing dashboard data…"); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -}); - -$("#btn-save").addEventListener("click", async () => { - const r = await api("/watchlists/save", { method: "POST" }); - if (r.needs_save_as) { openSaveAs(); return; } - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -}); - -function openSaveAs() { - $("#saveas-name").value = state.board - ? state.board.banner.watchlist.replace(/\s*\*$/, "") : ""; - $("#modal-saveas").classList.remove("hidden"); -} -$("#btn-saveas").addEventListener("click", openSaveAs); -$("#saveas-cancel").addEventListener("click", () => - $("#modal-saveas").classList.add("hidden")); -$("#saveas-ok").addEventListener("click", async () => { - $("#modal-saveas").classList.add("hidden"); - const r = await api("/watchlists/save-as", { - method: "POST", - body: JSON.stringify({ - slot: Number($("#saveas-slot").value), - name: $("#saveas-name").value, - }), - }); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -}); - -$("#btn-add-selected").addEventListener("click", async () => { - const slot = prompt("Add selected stocks to watchlist slot (1–20):", "1"); - if (slot === null) return; - const r = await api("/watchlists/add-selected", { - method: "POST", body: JSON.stringify({ slot: Number(slot) }), - }); - if (r.message) toast(r.message, r.ok ? "ok" : "err"); - loadBoard(); -}); - -$("#btn-delete-all").addEventListener("click", async () => { - if (!confirm("Remove all stocks from the dashboard?")) return; - await api("/stocks", { method: "DELETE" }); - loadBoard(); -}); - -function showShutdownScreen() { - if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; } - if (state.heartbeat) { clearInterval(state.heartbeat); state.heartbeat = null; } - const screen = $("#shutdown-screen"); - if (screen) { - screen.classList.add("visible"); - } -} - -function updateEnvPill(isHf) { - const pill = $("#env-pill"); - if (!pill) return; - if (isHf) { - pill.textContent = "HuggingFace Spaces"; - pill.className = "env-pill hf"; - } else { - pill.textContent = "Local"; - pill.className = "env-pill local"; - } -} - -function startHeartbeat() { - state.consecutiveFailures = 0; - state.heartbeat = setInterval(async () => { - try { - const res = await fetch("/api/status"); - if (!res.ok) { - state.consecutiveFailures++; - if (state.consecutiveFailures >= 3) { - showShutdownScreen(); - } - return; - } - const body = await res.json().catch(() => ({})); - if (body.exiting) { - showShutdownScreen(); - } - updateEnvPill(body.is_hf); - state.consecutiveFailures = 0; - } catch (err) { - state.consecutiveFailures++; - if (state.consecutiveFailures >= 3) { - showShutdownScreen(); - } - } - }, 1000); -} - -async function triggerExit() { - if (!confirm("Are you sure you want to exit? This will shut down the server and stop the application.")) { - return; - } - showShutdownScreen(); - try { - await fetch("/api/shutdown", { method: "POST" }); - } catch (err) { - console.error("Shutdown initiated (fetch failed/aborted as server terminated).", err); - } -} - -/* Login, TOTP, and Admin Control Panel helper functions */ - -function showLoginBox() { - const splash = $("#splash-screen"); - if (splash) { - splash.classList.remove("hidden"); - } - const splashStatus = $("#splash-status"); - const loader = $(".splash-loader-container"); - const loginBox = $("#login-box"); - const totpBox = $("#totp-box"); - - if (loader) loader.classList.add("hidden"); - if (splashStatus) splashStatus.classList.add("hidden"); - if (totpBox) totpBox.classList.add("hidden"); - if (loginBox) loginBox.classList.remove("hidden"); -} - -function showTotpBox(userid, qrUrl = null, secretKey = null) { - const loginBox = $("#login-box"); - const totpBox = $("#totp-box"); - const qrContainer = $("#totp-qr-container"); - const qrImage = $("#totp-qr-image"); - const secretKeyText = $("#totp-secret-key"); - - if (loginBox) loginBox.classList.add("hidden"); - if (totpBox) totpBox.classList.remove("hidden"); - - if (secretKey && secretKeyText) { - secretKeyText.textContent = secretKey; - if (qrContainer) qrContainer.classList.remove("hidden"); - - if (qrUrl && qrImage) { - qrImage.src = qrUrl; - qrImage.style.display = "block"; - } else if (qrImage) { - qrImage.style.display = "none"; - } - } else if (qrContainer) { - qrContainer.classList.add("hidden"); - } -} - -let currentAdminActionUser = null; - -async function openAdminPanel() { - $("#modal-admin").classList.remove("hidden"); - await refreshAdminDashboard(); -} - -async function refreshAdminDashboard() { - const res = await api("/admin/dashboard"); - if (!res.ok) return; - - $("#stat-users").textContent = res.total_users; - $("#stat-symbols").textContent = res.total_symbols; - $("#stat-yfinance").textContent = res.yfinance_connected ? "Online" : "Offline"; - $("#stat-yfinance").style.color = res.yfinance_connected ? "var(--gain)" : "var(--loss)"; - - const userList = $("#admin-user-list"); - userList.innerHTML = ""; - - Object.values(res.users).forEach((user) => { - const isOnline = user.login_status === "online"; - const isActive = user.active; - const isUserAdmin = user.is_admin; - - const tr = document.createElement("tr"); - tr.style.borderBottom = "1px solid rgba(255, 255, 255, 0.08)"; - tr.innerHTML = ` - ${esc(user.userid)} - ${isOnline ? "Online" : "Offline"} - ${isUserAdmin ? "Admin" : "User"} - ${user.login_count} - ${user.last_login ? user.last_login.substring(0, 16).replace("T", " ") : "Never"} - ${isActive ? "Active" : "Locked"} - - - - - - - - - - `; - - // Wire actions - tr.querySelector(".btn-action-qr").onclick = () => runAdminAction("qr", user.userid); - tr.querySelector(".btn-action-metrics").onclick = () => runAdminAction("metrics", user.userid); - tr.querySelector(".btn-action-reset").onclick = () => { - if (confirm(`Are you sure you want to perform a full account reset (password + TOTP) for ${user.userid}? This will log them out immediately.`)) { - runAdminAction("reset", user.userid); - } - }; - tr.querySelector(".btn-action-lock").onclick = () => runAdminAction("lock", user.userid); - tr.querySelector(".btn-action-mirror").onclick = () => runAdminAction("mirror", user.userid); - tr.querySelector(".btn-action-chat").onclick = () => runAdminAction("chat", user.userid); - tr.querySelector(".btn-action-delete").onclick = () => { - if (confirm(`Are you sure you want to PERMANENTLY delete user ${user.userid}?`)) { - runAdminAction("delete", user.userid); - } - }; - - userList.appendChild(tr); - }); -} - -function hideAllAdminSidebarPanels() { - document.querySelectorAll(".sidebar-panel").forEach((p) => p.classList.add("hidden")); - const grid = document.querySelector(".admin-main-grid"); - if (grid) grid.classList.remove("has-sidebar"); -} - -function showAdminSidebarPanel(panelId) { - hideAllAdminSidebarPanels(); - const p = document.getElementById(panelId); - if (p) p.classList.remove("hidden"); - const grid = document.querySelector(".admin-main-grid"); - if (grid) grid.classList.add("has-sidebar"); -} - -async function runAdminAction(action, target_uid) { - currentAdminActionUser = target_uid; - hideAllAdminSidebarPanels(); - - if (action === "qr") { - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action, target_uid }) - }); - if (res.ok) { - if (res.qr) { - $("#admin-qr-display").innerHTML = ``; - } else { - $("#admin-qr-display").innerHTML = `
Pending first login by user.
(QR code will be provisioned then)
`; - } - $("#admin-secret-key").textContent = res.secret; - showAdminSidebarPanel("panel-qr"); - } - } else if (action === "metrics") { - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action, target_uid }) - }); - if (res.ok) { - $("#admin-metrics-display").innerHTML = ` -
User ID: ${res.userid}
-
Login Count: ${res.login_count}
-
Last Login: ${res.last_login ? res.last_login.substring(0, 16).replace("T", " ") : "Never"}
-
Symbols in State: ${res.symbols_count}
-
Watchlists Created: ${res.watchlists_count}
- `; - showAdminSidebarPanel("panel-metrics"); - } - } else if (action === "reset" || action === "lock" || action === "delete") { - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action, target_uid }) - }); - if (res.ok) { - toast(res.message, "ok"); - await refreshAdminDashboard(); - } - } else if (action === "mirror") { - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action, target_uid }) - }); - if (res.ok) { - const logs = res.logs || []; - const logText = logs.map((l) => `[${l.timestamp.substring(11, 19)}] ${l.action.toUpperCase()}: ${l.status} (${l.details || ""})`).join("\n"); - $("#admin-mirror-display").textContent = logText || "No activities logged yet."; - showAdminSidebarPanel("panel-mirror"); - } - } else if (action === "chat") { - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action, target_uid }) - }); - if (res.ok) { - $("#chat-user-title").textContent = `Chat with ${target_uid}`; - const chatDisplay = $("#admin-chat-display"); - chatDisplay.innerHTML = ""; - res.messages.forEach((m) => { - const div = document.createElement("div"); - div.className = "chat-msg " + m.sender; - div.innerHTML = `${esc(m.text)}`; - chatDisplay.appendChild(div); - }); - showAdminSidebarPanel("panel-chat"); - } - } -} - -/* boot */ -document.addEventListener("DOMContentLoaded", async () => { - const btnLoginSubmit = $("#btn-login-submit"); - const btnTotpSubmit = $("#btn-totp-submit"); - const loginUsernameIn = $("#login-username"); - const loginPasswordIn = $("#login-password"); - const totpIn = $("#totp-input"); - const loginError = $("#login-error"); - const totpError = $("#totp-error"); - const splash = $("#splash-screen"); - - let authUsername = null; - - // Handle Login submission - if (btnLoginSubmit) { - btnLoginSubmit.addEventListener("click", async () => { - loginError.classList.add("hidden"); - const userid = loginUsernameIn.value.trim(); - const password = loginPasswordIn.value; - if (!userid || !password) return; - - busy("Verifying credentials..."); - try { - const res = await fetch("/api/auth/login", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ userid, password }) - }); - const data = await res.json(); - unbusy(); - - if (res.ok && data.ok) { - authUsername = data.userid; - if (data.totp_required) { - if (data.message === "first_login" || data.message === "totp_setup") { - showTotpBox(data.userid, data.qr, data.secret); - } else { - showTotpBox(data.userid); - } - } - } else { - loginError.textContent = data.message || "Invalid credentials."; - loginError.classList.remove("hidden"); - } - } catch (err) { - unbusy(); - console.error("Login failed:", err); - loginError.textContent = "Login request failed: " + err.message; - loginError.classList.remove("hidden"); - } - }); - } - - // Handle TOTP submission - if (btnTotpSubmit) { - btnTotpSubmit.addEventListener("click", async () => { - totpError.classList.add("hidden"); - const code = totpIn.value.trim(); - if (!code) return; - - busy("Verifying TOTP code..."); - try { - const res = await fetch("/api/auth/totp", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ userid: authUsername, code: code }) - }); - const data = await res.json(); - unbusy(); - - if (res.ok && data.ok) { - localStorage.setItem("session_token", data.session_token); - - const splashStatus = $("#splash-status"); - if (splashStatus) { - splashStatus.classList.remove("hidden"); - splashStatus.textContent = "Loading market data..."; - } - const loader = $(".splash-loader-container"); - if (loader) loader.classList.remove("hidden"); - $("#totp-box").classList.add("hidden"); - - // Successfully authenticated! Run check - await runBootSequence(); - } else { - totpError.textContent = data.message || "Incorrect code."; - totpError.classList.remove("hidden"); - totpIn.value = ""; - } - } catch (err) { - unbusy(); - console.error("TOTP verification failed:", err); - } - }); - } - - // Keyboard support for Enter key - if (loginUsernameIn) { - [loginUsernameIn, loginPasswordIn].forEach((el) => { - el.addEventListener("keydown", (e) => { - if (e.key === "Enter") btnLoginSubmit.click(); - }); - }); - } - if (totpIn) { - totpIn.addEventListener("keydown", (e) => { - if (e.key === "Enter") btnTotpSubmit.click(); - }); - } - - // Admin Panel Event Handlers - $("#admin-close").onclick = () => $("#modal-admin").classList.add("hidden"); - $("#ai-close").onclick = () => $("#modal-ai").classList.add("hidden"); - - const aiRefreshBtn = $("#ai-refresh-btn"); - if (aiRefreshBtn) { - aiRefreshBtn.onclick = () => { - if (currentAISymbol) { - openAIAnalysisModal(currentAISymbol, true); - } - }; - } - - const aiCopyBtn = $("#ai-copy-btn"); - if (aiCopyBtn) { - aiCopyBtn.onclick = () => { - if (!currentAIAnalysisText) { - toast("No report content to copy", "err"); - return; - } - navigator.clipboard.writeText(currentAIAnalysisText).then(() => { - toast("AI Research Report copied to clipboard! 📋", "ok"); - }).catch((err) => { - toast("Failed to copy report: " + err, "err"); - }); - }; - } - $("#admin-btn-show-add").onclick = () => { - showAdminSidebarPanel("panel-add-user"); - }; - $("#admin-btn-create-submit").onclick = async () => { - const username = $("#add-username").value.trim(); - const role = $("#add-role").value; - if (!username) return; - - const res = await api("/admin/action", { - method: "POST", - body: JSON.stringify({ action: "add_user", username, role }) - }); - if (res.ok) { - toast(res.message, "ok"); - $("#add-username").value = ""; - hideAllAdminSidebarPanels(); - await refreshAdminDashboard(); - } - }; - - // Fake chat send button support - $("#chat-send-btn").onclick = () => { - const chatInput = $("#chat-input"); - const text = chatInput.value.trim(); - if (!text || !currentAdminActionUser) return; - - const chatDisplay = $("#admin-chat-display"); - const div = document.createElement("div"); - div.className = "chat-msg admin"; - div.innerHTML = `Admin: ${esc(text)}`; - chatDisplay.appendChild(div); - chatInput.value = ""; - chatDisplay.scrollTop = chatDisplay.scrollHeight; - - // Simulated reply - setTimeout(() => { - const reply = document.createElement("div"); - reply.className = "chat-msg user"; - reply.innerHTML = `${esc(currentAdminActionUser)}: Thanks admin! Got it.`; - chatDisplay.appendChild(reply); - chatDisplay.scrollTop = chatDisplay.scrollHeight; - }, 1000); - }; - - $("#chat-input").addEventListener("keydown", (e) => { - if (e.key === "Enter") $("#chat-send-btn").click(); - }); - - // Settings -> Change Password Event Handlers - const btnSettingsChangePassword = $("#btn-settings-change-password"); - if (btnSettingsChangePassword) { - btnSettingsChangePassword.addEventListener("click", () => { - closeMenus(); - $("#cp-current").value = ""; - $("#cp-new").value = ""; - $("#cp-confirm").value = ""; - $("#cp-error").classList.add("hidden"); - $("#modal-change-password").classList.remove("hidden"); - }); - } - - $("#cp-cancel").addEventListener("click", () => { - $("#modal-change-password").classList.add("hidden"); - }); - - $("#cp-submit").addEventListener("click", async () => { - const current_password = $("#cp-current").value; - const new_password = $("#cp-new").value; - const confirm_password = $("#cp-confirm").value; - const cpError = $("#cp-error"); - - if (new_password !== confirm_password) { - cpError.textContent = "New password and confirm password do not match."; - cpError.classList.remove("hidden"); - return; - } - - try { - const res = await api("/auth/change-password", { - method: "POST", - body: JSON.stringify({ current_password, new_password, confirm_password }) - }); - if (res.ok) { - toast(res.message || "Password updated successfully!", "ok"); - $("#modal-change-password").classList.add("hidden"); - } else { - cpError.textContent = res.message || "Failed to update password."; - cpError.classList.remove("hidden"); - } - } catch (err) { - cpError.textContent = "Error changing password: " + err; - cpError.classList.remove("hidden"); - } - }); - - // Mandatory Policy Password Change Submission - $("#force-cp-submit").addEventListener("click", async () => { - const new_password = $("#force-cp-new").value; - const confirm_password = $("#force-cp-confirm").value; - const forceCpError = $("#force-cp-error"); - - if (new_password !== confirm_password) { - forceCpError.textContent = "New password and confirm password do not match."; - forceCpError.classList.remove("hidden"); - return; - } - - try { - const res = await api("/auth/change-password", { - method: "POST", - body: JSON.stringify({ new_password, confirm_password }) - }); - if (res.ok) { - toast("Compliant password saved successfully!", "ok"); - $("#modal-force-password-change").classList.add("hidden"); - await runBootSequence(); - } else { - forceCpError.textContent = res.message || "Failed to update password."; - forceCpError.classList.remove("hidden"); - } - } catch (err) { - forceCpError.textContent = "Error saving password: " + err; - forceCpError.classList.remove("hidden"); - } - }); - - async function runBootSequence() { - const res = await fetch("/api/auth/status", { - headers: { "X-Session-Token": localStorage.getItem("session_token") || "" } - }); - const status = await res.json(); - - const splashStatus = $("#splash-status"); - if (status.nvidia_status) { - if (status.nvidia_status.success) { - if (splashStatus) splashStatus.textContent = "System Initialized — NVIDIA API connection tested and successful ✔"; - toast(`NVIDIA API connection tested & successful (${status.nvidia_status.source || 'Keyring'})`, "ok"); - } else { - if (splashStatus) splashStatus.textContent = "System Initialized — " + (status.nvidia_status.message || "NVIDIA API Key warning"); - } - } - - if (status.authenticated) { - $("#display-username").textContent = status.userid; - if (status.is_admin) { - $("#menu-utilities-container").classList.remove("hidden"); - loadUtilitiesMenu(); - } else { - $("#menu-utilities-container").classList.add("hidden"); - } - - if (status.force_password_change) { - $("#modal-force-password-change").classList.remove("hidden"); - return; - } - - await loadBoard(); - startHeartbeat(); - - if (splash) { - splash.classList.add("fade-out"); - setTimeout(() => { - splash.classList.add("hidden"); - splash.classList.remove("fade-out"); - }, 600); - } - } else { - showLoginBox(); - if (status.reason === "inactivity") { - toast("Session terminated due to inactivity.", "err"); - const loginError = $("#login-error"); - if (loginError) { - loginError.textContent = "Session terminated due to inactivity."; - loginError.classList.remove("hidden"); - } - } - } - } - - // Handle Logout button click - const btnLogout = $("#btn-logout"); - if (btnLogout) { - btnLogout.addEventListener("click", async () => { - busy("Logging out..."); - try { - await fetch("/api/auth/logout", { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-Session-Token": localStorage.getItem("session_token") || "" - } - }); - } catch (err) { - console.error("Logout request failed:", err); - } - localStorage.removeItem("session_token"); - - // Stop live refresh heartbeat if active - if (state.livePoll) { clearInterval(state.livePoll); state.livePoll = null; } - if (state.heartbeat) { clearInterval(state.heartbeat); state.heartbeat = null; } - - authUsername = null; - $("#login-username").value = ""; - $("#login-password").value = ""; - $("#totp-input").value = ""; - unbusy(); - showLoginBox(); - }); - } - - // Start checking auth status on page load - runBootSequence(); -}); - - diff --git a/nsedash/web/static/style.css b/nsedash/web/static/style.css deleted file mode 100644 index 8025ddacbc46be0a7638a5bc0b46a7792af463b8..0000000000000000000000000000000000000000 --- a/nsedash/web/static/style.css +++ /dev/null @@ -1,924 +0,0 @@ -/* NSE Stock Dashboard — market-terminal styling. - Numeric-heavy layout: tabular figures, market-convention colours. */ - -:root { - --bg: #0e1116; - --panel: #161b22; - --panel-2: #1c232d; - --border: #2a333f; - --text: #dbe2ea; - --muted: #94a3b8; - --gain: #4ade80; - --loss: #ff7675; - --flat: #a1a1aa; - --accent: #60a5fa; - --amber: #f59e0b; - --live: #1f7a3d; - --historic: #8a2f2a; -} - -* { box-sizing: border-box; } -html, body { margin: 0; } -body { - background: var(--bg); - color: var(--text); - font: 13px/1.45 "Segoe UI", system-ui, -apple-system, sans-serif; -} -b, .num { font-variant-numeric: tabular-nums; } -button { - background: var(--panel-2); - color: var(--text); - border: 1px solid var(--border); - border-radius: 4px; - padding: 5px 10px; - cursor: pointer; -} -button:hover { border-color: var(--accent); } -button:focus-visible, input:focus-visible, select:focus-visible { - outline: 2px solid var(--accent); - outline-offset: 1px; -} -button.primary { background: var(--accent); border-color: var(--accent); color: #fff; } -button.danger { border-color: var(--loss); color: #e07b70; } -input, select { - background: var(--panel); - color: var(--text); - border: 1px solid var(--border); - border-radius: 4px; - padding: 5px 8px; -} -.muted { color: var(--muted); } -.hidden { display: none !important; } - -/* Banner */ -.banner { - display: flex; - justify-content: space-between; - padding: 6px 14px; - font-weight: 600; - color: #fff; - letter-spacing: .02em; -} -.banner.live { background: var(--live); } -.banner.historic { background: var(--historic); } - -.env-pill { - display: inline-block; - padding: 2px 6px; - font-size: 10px; - font-weight: 700; - border-radius: 12px; - text-transform: uppercase; - letter-spacing: 0.05em; - line-height: 1; -} -.env-pill.local { - background: rgba(59, 130, 196, 0.25); - color: #5dade2; - border: 1px solid rgba(59, 130, 196, 0.4); -} -.env-pill.hf { - background: rgba(241, 196, 15, 0.25); - color: #f1c40f; - border: 1px solid rgba(241, 196, 15, 0.4); -} - -/* Menu bar */ -.menubar { - display: flex; - gap: 2px; - background: var(--panel); - border-bottom: 1px solid var(--border); - padding: 0 8px; -} -.menu { position: relative; } -.menu-btn { - border: none; - background: none; - border-radius: 0; - padding: 8px 12px; -} -.menu-btn:hover { background: var(--panel-2); } -.menu-drop { - position: absolute; - top: 100%; - left: 0; - min-width: 230px; - max-height: 60vh; - overflow: auto; - background: var(--panel-2); - border: 1px solid var(--border); - border-radius: 0 0 6px 6px; - display: none; - z-index: 40; -} -.menu.open .menu-drop { display: block; } -.menu-drop button { - display: block; - width: 100%; - text-align: left; - border: none; - border-radius: 0; - padding: 7px 12px; -} -.menu-drop button:hover { background: var(--accent); color: #fff; } -.menu-drop .sep { border-top: 1px solid var(--border); margin: 4px 0; } -.menu-drop .head { - padding: 6px 12px 2px; - font-size: 11px; - text-transform: uppercase; - color: var(--muted); -} - -/* Toolbar */ -.toolbar { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 10px; - padding: 8px 12px; - background: var(--panel); - border-bottom: 1px solid var(--border); -} -.toggle { display: flex; align-items: center; gap: 6px; } -.pf-summary { - display: flex; - gap: 16px; - padding: 4px 12px; - background: var(--panel-2); - border: 1px solid var(--border); - border-radius: 4px; - flex-wrap: wrap; -} -.pf-summary span { color: var(--muted); } -.pf-summary b { color: var(--text); margin-left: 4px; } -.pf-summary b.gain { color: var(--gain); } -.pf-summary b.loss { color: var(--loss); } - -/* Live Index Tickers & RTC Clock */ -.index-tickers { - display: flex; - align-items: center; - gap: 8px; -} -.index-badge { - display: flex; - align-items: center; - gap: 6px; - background: var(--panel-2); - border: 1px solid var(--border); - border-radius: 4px; - padding: 3px 8px; - cursor: pointer; - transition: all 0.2s ease; -} -.index-badge:hover { - border-color: var(--accent); - background: rgba(59, 130, 196, 0.15); -} -.idx-name { - font-size: 11px; - font-weight: 700; - color: #94a3b8; - letter-spacing: 0.04em; -} -.idx-pill { - font-size: 11px; - font-weight: 700; - font-variant-numeric: tabular-nums; - padding: 2px 6px; - border-radius: 3px; - background: rgba(255, 255, 255, 0.08); -} -.idx-pill.positive { - color: var(--gain); - background: rgba(39, 174, 96, 0.18); - border: 1px solid rgba(39, 174, 96, 0.3); -} -.idx-pill.negative { - color: var(--loss); - background: rgba(224, 123, 112, 0.18); - border: 1px solid rgba(224, 123, 112, 0.3); -} - -.date-rtc-wrap { - margin-left: auto; - display: flex; - align-items: center; - gap: 14px; -} -.rtc-clock { - display: flex; - align-items: center; - gap: 6px; - background: rgba(15, 23, 42, 0.7); - border: 1px solid var(--border); - border-radius: 4px; - padding: 4px 10px; - font-family: "JetBrains Mono", "Consolas", monospace; - font-size: 11px; - font-weight: 600; - color: #38bdf8; - box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.3); -} -.rtc-icon { - font-size: 12px; -} - -.datewrap { margin-left: 0; display: flex; align-items: center; gap: 6px; } - -/* Add row */ -.addrow { - display: flex; - gap: 8px; - align-items: center; - padding: 8px 12px; -} -#stock-search { flex: 0 1 420px; } - -/* Grid */ -.grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(320px, 420px)); - gap: 12px; - padding: 0 12px 60px; - justify-content: center; - max-width: 1680px; - margin: 0 auto; -} - -/* Card */ -.card { - background: var(--panel); - border: 1px solid var(--border); - border-radius: 8px; - padding: 10px 12px; - position: relative; -} -.card h4 { margin: 0; font-size: 14px; } -.card .sec { font-size: 11px; color: var(--muted); } -.card .rm { - position: absolute; - top: 8px; - right: 8px; - padding: 0 7px; - line-height: 20px; -} -.card .row { - display: flex; - justify-content: space-between; - gap: 8px; - margin: 2px 0; -} -.card .lbl { color: var(--muted); } -.gain { color: var(--gain); } -.loss { color: var(--loss); } -.flat { color: var(--flat); } -.amber { color: var(--amber); } -.score-80 { color: #2ecc71; } -.score-60 { color: #a3d977; } -.score-40 { color: var(--amber); } -.score-20 { color: #e67e22; } -.score-0 { color: var(--loss); } -.changes, .volchips { - display: flex; - gap: 6px; - flex-wrap: wrap; - margin: 4px 0; -} -.chip { - background: var(--panel-2); - border: 1px solid var(--border); - border-radius: 4px; - padding: 2px 6px; - font-size: 11px; -} -.card canvas { width: 100%; height: auto; margin-top: 6px; } -.live-row { - margin-top: 6px; - padding: 5px 8px; - background: var(--panel-2); - border-left: 3px solid var(--live); - border-radius: 4px; - font-size: 12px; -} -.inputs { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 6px; - margin-top: 8px; -} -.inputs label { font-size: 11px; color: var(--muted); } -.inputs input { width: 100%; } -.pl-row { margin-top: 6px; font-size: 12px; } -.card-foot { - display: flex; - justify-content: space-between; - align-items: center; - margin-top: 8px; -} -.reco { font-size: 12px; } - -/* Bottom controls */ -.controls { - position: fixed; - bottom: 0; - left: 0; - right: 0; - display: flex; - gap: 8px; - padding: 8px 12px; - background: var(--panel); - border-top: 1px solid var(--border); - z-index: 30; -} - -/* Overlays / modals */ -.overlay { - position: fixed; - inset: 0; - background: rgba(0,0,0,.55); - display: flex; - align-items: center; - justify-content: center; - z-index: 60; -} -.modal { - background: var(--panel); - border: 1px solid var(--border); - border-radius: 8px; - padding: 18px; - min-width: 320px; - display: flex; - flex-direction: column; - gap: 10px; -} -.modal h3 { margin: 0 0 4px; } -.modal label { display: flex; flex-direction: column; gap: 4px; } -.modal-actions { display: flex; justify-content: flex-end; gap: 8px; } -.busy-box { - background: var(--panel); - border: 1px solid var(--border); - border-radius: 8px; - padding: 22px 30px; - display: flex; - align-items: center; - gap: 14px; -} -.spinner { - width: 22px; - height: 22px; - border: 3px solid var(--border); - border-top-color: var(--accent); - border-radius: 50%; - animation: spin 0.9s linear infinite; -} -@keyframes spin { to { transform: rotate(360deg); } } -@media (prefers-reduced-motion: reduce) { - .spinner { animation-duration: 2.5s; } -} - -/* Terminal */ -.modal.terminal { width: min(860px, 92vw); } -#terminal-out { - background: #0a0d11; - border: 1px solid var(--border); - border-radius: 4px; - height: 50vh; - overflow: auto; - padding: 10px; - margin: 0; - font: 12px/1.5 "Consolas", "SFMono-Regular", monospace; - white-space: pre-wrap; -} -#terminal-out .info { color: var(--muted); } -#terminal-out .error { color: #e07b70; } -#terminal-out .success { color: var(--gain); } -#terminal-out .input { color: var(--accent); } -.terminal-input { display: flex; gap: 8px; } -.terminal-input input { flex: 1; } - -/* Toasts */ -#toasts { - position: fixed; - right: 14px; - bottom: 56px; - display: flex; - flex-direction: column; - gap: 8px; - z-index: 80; -} -.toast { - background: var(--panel-2); - border: 1px solid var(--border); - border-left: 3px solid var(--accent); - border-radius: 6px; - padding: 10px 14px; - max-width: 420px; - box-shadow: 0 4px 14px rgba(0,0,0,.4); -} -.toast.err { border-left-color: var(--loss); } -.toast.ok { border-left-color: var(--gain); } - -@media (max-width: 720px) { - .pf-summary { order: 5; width: 100%; } - .datewrap { margin-left: 0; } -} - -/* Splash Screen Styles */ -.splash-screen { - position: fixed; - top: 0; - left: 0; - width: 100vw; - height: 100vh; - background: radial-gradient(circle at center, #1b2230 0%, #0e1116 100%); - display: flex; - justify-content: center; - align-items: center; - z-index: 9999; - opacity: 1; - visibility: visible; - transition: opacity 0.6s cubic-bezier(0.4, 0, 0.2, 1), visibility 0.6s cubic-bezier(0.4, 0, 0.2, 1); -} - -.splash-screen.fade-out { - opacity: 0; - visibility: hidden; - pointer-events: none; -} - -.splash-content { - text-align: center; - animation: splashFadeIn 1s cubic-bezier(0.16, 1, 0.3, 1); - padding: 20px; -} - -.splash-logo { - font-size: 42px; - font-weight: 800; - letter-spacing: -0.5px; - margin-bottom: 8px; - color: #ffffff; -} - -.splash-brand-nse { - background: linear-gradient(135deg, #3b82c4 0%, #27ae60 100%); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - padding-right: 2px; -} - -.splash-brand-dash { - color: #ffffff; - text-shadow: 0 0 20px rgba(59, 130, 196, 0.3); -} - -.splash-subtitle { - font-size: 13px; - color: var(--muted); - letter-spacing: 2px; - text-transform: uppercase; - margin-bottom: 30px; -} - -.splash-loader-container { - width: 220px; - height: 4px; - background: var(--border); - border-radius: 2px; - margin: 0 auto 12px; - overflow: hidden; - position: relative; -} - -.splash-loader-bar { - width: 100%; - height: 100%; - background: linear-gradient(90deg, #3b82c4, #27ae60, #3b82c4); - background-size: 200% 100%; - position: absolute; - left: -100%; - animation: loadingProgress 2s infinite linear, gradientMove 3s infinite linear; -} - -.splash-status { - font-size: 11px; - color: var(--muted); - font-family: monospace; -} - -@keyframes splashFadeIn { - from { - opacity: 0; - transform: translateY(20px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -@keyframes loadingProgress { - 0% { left: -100%; } - 50% { left: 0%; } - 100% { left: 100%; } -} - -@keyframes gradientMove { - 0% { background-position: 0% 50%; } - 100% { background-position: 200% 50%; } -} - -.captcha-box { - margin-top: 24px; - padding: 20px; - border-radius: 12px; - background: rgba(255, 255, 255, 0.03); - border: 1px solid rgba(255, 255, 255, 0.08); - box-shadow: 0 4px 30px rgba(0, 0, 0, 0.2); - backdrop-filter: blur(5px); - -webkit-backdrop-filter: blur(5px); - animation: splashFadeIn 0.5s ease-out; -} -.captcha-box p { - margin: 0 0 14px 0; - font-size: 14px; - color: #ccc; - font-family: inherit; -} -.captcha-box .captcha-row { - display: flex; - gap: 12px; - align-items: center; - justify-content: center; -} -.captcha-box input { - width: 90px; - padding: 8px; - border-radius: 6px; - border: 1px solid rgba(255, 255, 255, 0.15); - background: rgba(255, 255, 255, 0.07); - color: #ffffff; - font-size: 16px; - text-align: center; - outline: none; - transition: border-color 0.3s; -} -.captcha-box input:focus { - border-color: #3b82c4; -} -.captcha-box button { - padding: 8px 18px; - border-radius: 6px; - background: var(--live); - color: #ffffff; - border: none; - cursor: pointer; - font-weight: 700; - transition: filter 0.3s; -} -.captcha-box button:hover { - filter: brightness(1.15); -} -.captcha-box .captcha-err { - color: var(--loss); - font-size: 12px; - margin: 10px 0 0 0; - font-weight: bold; -} - -/* Shutdown Screen Styles */ -.shutdown-screen { - position: fixed; - top: 0; - left: 0; - width: 100vw; - height: 100vh; - background: radial-gradient(circle at center, #2c1a1d 0%, #0e0b0c 100%); - display: flex; - justify-content: center; - align-items: center; - z-index: 10000; - opacity: 0; - visibility: hidden; - transition: opacity 0.5s ease, visibility 0.5s ease; -} - -.shutdown-screen.visible { - opacity: 1; - visibility: visible; -} - - -/* Login and TOTP styling */ -.login-box, .totp-box { - margin-top: 24px; - padding: 20px; - border-radius: 12px; - background: rgba(255, 255, 255, 0.03); - border: 1px solid rgba(255, 255, 255, 0.08); - box-shadow: 0 4px 30px rgba(0, 0, 0, 0.2); - backdrop-filter: blur(5px); - -webkit-backdrop-filter: blur(5px); - animation: splashFadeIn 0.5s ease-out; - min-width: 280px; -} -.login-box h3, .totp-box h3 { - margin-top: 0; - margin-bottom: 12px; - color: #fff; - font-size: 18px; -} -.login-row { - margin-bottom: 10px; -} -.login-row input { - width: 100%; - padding: 8px; - border-radius: 6px; - border: 1px solid rgba(255, 255, 255, 0.15); - background: rgba(255, 255, 255, 0.07); - color: #ffffff; - font-size: 14px; - outline: none; -} -.login-row input:focus { - border-color: #3b82c4; -} -.login-box button, .totp-box button { - width: 100%; - padding: 8px; - border-radius: 6px; - background: var(--live); - color: #ffffff; - border: none; - cursor: pointer; - font-weight: 700; - font-size: 14px; -} -.login-err { - color: var(--loss); - font-size: 12px; - margin: 10px 0 0 0; - font-weight: bold; -} -.totp-qr-container img { - border: 1px solid #fff; - border-radius: 6px; - background: white; - padding: 6px; - margin: 8px 0; - max-width: 140px; -} - -/* Admin Panel modal customization & resizability */ -.modal.admin-panel { - width: min(1200px, 96vw); - min-width: 600px; - max-width: 98vw; - height: auto; - min-height: 480px; - max-height: 94vh; - resize: both; - overflow: auto; - box-sizing: border-box; -} - -.admin-main-grid { - display: grid; - grid-template-columns: 1fr; - gap: 16px; - width: 100%; -} - -.admin-main-grid.has-sidebar { - grid-template-columns: minmax(0, 1fr) 320px; -} - -.admin-section.user-mgmt { - display: flex; - flex-direction: column; - width: 100%; - flex: 1; -} - -.admin-section.user-mgmt .table-wrap { - flex: 1; - max-height: 60vh; - overflow-y: auto; -} - -.admin-table th, .admin-table td { - border-bottom: 1px solid rgba(255, 255, 255, 0.08); -} -.admin-table tr:hover td { - background: rgba(255, 255, 255, 0.02); -} -.sidebar-panel { - animation: splashFadeIn 0.3s ease-out; -} -.chat-view { - background: #0f172a; - border: 1px solid rgba(255,255,255,0.1); - height: 120px; - overflow-y: auto; - border-radius: 4px; - padding: 6px; -} -.chat-msg { - margin-bottom: 4px; -} -.chat-msg.system { - color: #94a3b8; - font-style: italic; -} -.chat-msg.admin { - color: #3b82f6; - font-weight: bold; -} -.chat-msg.user { - color: #10b981; - font-weight: bold; -} - -/* ========================================== - Responsive UI Overrides for Mobile Screens - ========================================== */ -html, body { - overflow-x: hidden; -} - -@media (max-width: 768px) { - /* Swipeable Horizontal Menu Bar */ - .menubar { - overflow-x: auto; - white-space: nowrap; - flex-wrap: nowrap; - -webkit-overflow-scrolling: touch; - gap: 0; - } - .menubar .menu { - flex-shrink: 0; - } - .menu-user-display { - margin-left: 8px !important; - flex-shrink: 0; - } - .menu-btn { - padding: 8px 10px; - } - - /* Responsive Admin Control Panel */ - .stat-cards { - grid-template-columns: 1fr !important; - gap: 8px; - } - .admin-main-grid { - grid-template-columns: 1fr !important; - gap: 16px; - } - .table-wrap { - overflow-x: auto; - -webkit-overflow-scrolling: touch; - } - .admin-table { - min-width: 550px; - } -} - -@media (max-width: 600px) { - /* Toolbar Layout Adjustments */ - .toolbar { - padding: 8px; - gap: 6px; - } - .toolbar > button, .toolbar > label.toggle { - flex: 1 1 calc(50% - 6px); - } - .pf-summary { - width: 100%; - order: 4; - gap: 8px; - justify-content: space-around; - padding: 6px; - } - .pf-summary span { - font-size: 11px; - } - .datewrap { - width: 100%; - order: 5; - margin-left: 0; - display: flex; - justify-content: space-between; - align-items: center; - } - .datewrap input { - flex-grow: 1; - max-width: 70%; - } - - /* Swipeable Bottom Control Dock */ - .controls { - overflow-x: auto; - white-space: nowrap; - flex-wrap: nowrap; - -webkit-overflow-scrolling: touch; - gap: 6px; - padding: 6px 8px; - } - .controls button { - flex-shrink: 0; - } -} - -@media (max-width: 480px) { - /* Add Stock Row Stacking */ - .addrow { - flex-direction: column; - align-items: stretch; - gap: 6px; - padding: 8px; - } - #stock-search { - width: 100%; - flex: unset; - } - .addrow button { - width: 100%; - } - - /* High Density Grid and Card scaling */ - .grid { - grid-template-columns: 1fr; - padding: 0 8px 80px; - gap: 10px; - } - .card { - padding: 8px; - } - .card h4 { - font-size: 13px; - } - - /* Responsive Dialog Sizing */ - .modal { - padding: 14px; - min-width: unset; - width: 100%; - } -} - -/* Screen reader only helper */ -.sr-only { - position: absolute; - width: 1px; - height: 1px; - padding: 0; - margin: -1px; - overflow: hidden; - clip: rect(0, 0, 0, 0); - white-space: nowrap; - border: 0; -} - -/* AI button & modal styles */ -.ai-btn { - background: linear-gradient(135deg, #6c5ce7, #4834d4); - color: #fff; - border: none; - font-weight: 600; - transition: transform 0.2s ease, filter 0.2s ease; -} -.ai-btn:hover { - filter: brightness(1.2); - transform: translateY(-1px); -} -.ai-analysis-modal { - max-width: 800px; - width: 90%; -} -@keyframes spin { - 0% { transform: rotate(0deg); } - 100% { transform: rotate(360deg); } -} -/* Style markdown lists, headings, and tables in modal */ -.markdown-body h1, .markdown-body h2, .markdown-body h3 { - margin-top: 16px; - margin-bottom: 8px; - color: var(--accent); -} -.markdown-body p { - margin-bottom: 12px; -} -.markdown-body ul, .markdown-body ol { - padding-left: 20px; - margin-bottom: 12px; -} -.markdown-body li { - margin-bottom: 4px; -} - - - diff --git a/nsedash/web/templates/index.html b/nsedash/web/templates/index.html deleted file mode 100644 index 4cd9852f67965531fd72be0c78a2711fd1fe1d14..0000000000000000000000000000000000000000 --- a/nsedash/web/templates/index.html +++ /dev/null @@ -1,489 +0,0 @@ - - - - - -NSE Stock Dashboard - - - - - - - -

NSE Stock Dashboard

- - -
-
- -
Market Intelligence & Analytics
-
-
-
-
Initializing System...
- - - - - - -
-
- - - - - - - - -
- - -
- Invested 0.00 - Current 0.00 - Day's Gain 0.00 (0.00%) - Abs Returns 0.00 (0.00%) -
- - - - - -
-
- NIFTY - ▲ 24,227.50 (+242.15 / +1.01%) -
-
- SENSEX - ▲ 77,568.70 (+802.78 / +1.04%) -
-
- - -
-
- 🕒 --:--:-- IST -
- -
-
- - -
- - - - -
- - -
- - -
- - - - - -
- - - - - - - - - - - - - - - - - - - - - - - -
-
- -
System Shut Down
-
- The backend server has been terminated safely.
- You may now close this browser tab. -
-
-
- - - - - - - - -
- - - - diff --git a/nsedash_admin.py b/nsedash_admin.py deleted file mode 100644 index 5c1d04d306bdb5918a9401e7a28c36d2839d51d3..0000000000000000000000000000000000000000 --- a/nsedash_admin.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -""" -NSE Stock Dashboard (NSEDashboard) Administration Utility - -Console-based interactive application providing granular menu options for -user administration, password resets, privilege management, data pipeline utilities, -and system health checks. - -Adapted specifically for the nsedash project architecture (including session -management, directory initialization, and utility scripts execution). -""" - -import os -import sys -import argparse -import subprocess -from datetime import datetime - -# Setup paths so we can import nsedash modules -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -if SCRIPT_DIR not in sys.path: - sys.path.insert(0, SCRIPT_DIR) - -from werkzeug.security import generate_password_hash, check_password_hash - -# Import core nsedash components -from nsedash.core.user_auth import init as auth_init, UserDatabase, SessionManager -from nsedash.config import Paths, UTILITY_SCRIPTS - -# Ingest data processing modules -try: - import nsedown - import nsecorpact - import nseprocessor - import nsesymhist - import nsemaxlive -except ImportError as _imp_err: - pass - - -def get_base_dir(): - return os.environ.get("NSEDASH_BASE_DIR", os.getcwd()) - -def init_auth(): - base_dir = get_base_dir() - auth_init(base_dir) - -def print_splash(): - banner = r""" -====================================================================== - _ _ ____ _____ ____ _ ____ _ _ ____ ___ _ ____ ____ - | \ | / ___|| ____| _ \ / \ / ___|| | | | _ \ / _ \ / \ | _ \| _ \ - | \| \___ \| _| | | | / _ \\___ \| |_| | |_) | | | / _ \| |_) | | | | - | |\ |___) | |___| |_| / ___ \___) | _ | _ <| |_| / ___ \ _ <| |_| | - |_| \_|____/|_____|____/_/ \_\____/|_| |_|_| \_\\___/_/ \_\_| \_\____/ -====================================================================== -""" - print(banner) - print(" 🛠️ NSEDASH SYSTEM ADMIN CONSOLE") - print("======================================================================\n") - -def menu_reset_admin_password(): - print("\n===================================================================") - print(" 🔑 RESET SUPER ADMIN ('admin') PASSWORD") - print("===================================================================") - - init_auth() - user = UserDatabase.get_user('admin') - current_hash = user.get('password_hash') if user else None - print(f"[*] Step 1: Current Stored Password Hash in Database:\n - {current_hash or '(None / Not Set)'}") - - print("\n[*] Step 2: Enter New Password") - while True: - new_pwd = input("Enter new password for super admin 'admin' (press Enter without input to exit): ").strip() - if not new_pwd: - print("[*] No input provided. Exiting password reset loop.") - return - if len(new_pwd) < 8: - print("[!] Error: Password must be at least 8 characters long. Please try again.") - continue - break - - print("\n[*] Step 3: Hashing & storing new password in database...") - try: - pwd_hash = generate_password_hash(new_pwd, method='pbkdf2:sha256') - UserDatabase.update_user('admin', { - 'password_hash': pwd_hash, - 'active': True - }) - SessionManager.clear_sessions_for_user('admin') - print("[OK] New password hash stored in database.") - except Exception as e: - print(f"[!] Error storing password in database: {e}") - return - - print("\n[*] Step 4: Extracting stored password hash from database for verification...") - user = UserDatabase.get_user('admin') - reextracted_hash = user.get('password_hash') if user else None - print(f" - Re-extracted Hash: {reextracted_hash}") - - print("\n[*] Step 5: Comparing extracted password hash with entered new password...") - is_match = check_password_hash(reextracted_hash, new_pwd) if reextracted_hash else False - - print("\n===================================================================") - print(" ✅ SUPER ADMIN PASSWORD VERIFICATION CONFIRMATION") - print("===================================================================") - print(f" - Target User Account : admin") - print(f" - Extracted DB Hash : {reextracted_hash}") - print(f" - Password Match Check : {'PASSED (True)' if is_match else 'FAILED (False)'}") - print(" -----------------------------------------------------------------") - if is_match: - print(" ✅ Password verified, matches, and confirmed stored in database!") - else: - print(" ❌ ERROR: Extracted password does NOT match entered password!") - print("===================================================================\n") - -def menu_reset_any_user_password(): - print("\n--- Reset Any User Password ---") - init_auth() - target_user = input("Enter username: ").strip() - if not target_user: - print("[!] Operation cancelled. Username required.") - return - - user = UserDatabase.get_user(target_user) - if not user: - print(f"[!] User '{target_user}' does not exist in the database.") - return - - new_pwd = input(f"Enter new password for user '{target_user}': ").strip() - if len(new_pwd) < 8: - print("[!] Error: Password must be at least 8 characters long.") - return - - pwd_hash = generate_password_hash(new_pwd, method='pbkdf2:sha256') - try: - UserDatabase.update_user(target_user, { - 'password_hash': pwd_hash, - 'active': True - }) - SessionManager.clear_sessions_for_user(target_user) - print(f"[OK] Password for user '{target_user}' reset successfully. User has been logged out.") - except Exception as e: - print(f"[!] Error resetting password: {e}") - -def menu_reset_totp_for_user(): - print("\n--- Reset TOTP for User ---") - init_auth() - target_user = input("Enter username to reset TOTP: ").strip() - if not target_user: - print("[!] Operation cancelled. Username required.") - return - - user = UserDatabase.get_user(target_user) - if not user: - print(f"[!] User '{target_user}' does not exist in the database.") - return - - try: - UserDatabase.update_user(target_user, { - 'totp_secret': None, - 'totp_secret_encrypted': None - }) - SessionManager.clear_sessions_for_user(target_user) - print(f"[OK] TOTP for user '{target_user}' reset successfully. They will be prompted to set it up on their next login.") - except Exception as e: - print(f"[!] Error resetting TOTP: {e}") - -def menu_full_account_reset(): - print("\n--- Full Account Reset (Force Fresh Registration) ---") - init_auth() - target_user = input("Enter username for full reset: ").strip() - if not target_user: - print("[!] Operation cancelled. Username required.") - return - - user = UserDatabase.get_user(target_user) - if not user: - print(f"[!] User '{target_user}' does not exist in the database.") - return - - try: - UserDatabase.update_user(target_user, { - 'password_hash': None, - 'totp_secret': None, - 'totp_secret_encrypted': None - }) - SessionManager.clear_sessions_for_user(target_user) - print(f"[OK] Full account reset for '{target_user}'. They have been logged out and will be forced to register a new password and set up TOTP on their next login.") - except Exception as e: - print(f"[!] Error resetting account: {e}") - -def menu_create_new_user(): - print("\n===================================================================") - print(" ➕ CREATE NEW USER ACCOUNT") - print("===================================================================") - init_auth() - - userid = input("1. Enter Username (userid): ").strip() - if not userid: - print("[!] Error: Username is required.") - return - - if UserDatabase.get_user(userid): - print(f"[!] Error: User '{userid}' already exists.") - return - - is_admin_input = input("2. Make user an admin? (y/N): ").strip().lower() - is_admin = is_admin_input == 'y' - - print("\n[*] Processing user creation...") - try: - UserDatabase.create_user(userid, is_admin) - print("\n===================================================================") - print(" ✅ NEW USER ACCOUNT CREATED SUCCESSFULLY") - print("===================================================================") - print(f" 1. Username : {userid}") - print(f" 2. Is Admin : {is_admin}") - print("===================================================================\n") - except Exception as e: - print(f"[!] Error creating user account: {e}") - -def menu_database_health_and_stats(): - print("\n===================================================================") - print(" 📊 DATABASE HEALTH CHECK & REAL-TIME SYSTEM STATISTICS") - print("===================================================================") - init_auth() - from nsedash.core import user_auth - - db_file = user_auth._USERS_DB_FILE - db_exists = db_file and db_file.exists() - print(f" - Users DB File ({db_file}) : {'[FOUND]' if db_exists else '[MISSING]'}") - - if not db_exists: - print("[!] Database file missing. It will be created on first use.") - return - - try: - users = UserDatabase.get_all_users() - user_count = len(users) - admin_count = sum(1 for u in users.values() if u.get('is_admin')) - active_count = sum(1 for u in users.values() if u.get('active')) - online_count = sum(1 for u in users.values() if u.get('login_status') == 'online') - - print("-------------------------------------------------------------------") - print(f" - Total User Accounts : {user_count}") - print(f" - Active Accounts : {active_count}") - print(f" - Admin Accounts : {admin_count}") - print(f" - Online Users : {online_count}") - print("-------------------------------------------------------------------") - - print("\n--- User Details ---") - print(f"{'User ID':<15} | {'Active':<8} | {'Admin':<8} | {'Status':<10} | {'Last Login'}") - print("-" * 75) - for uid, udata in users.items(): - is_act = "Yes" if udata.get("active") else "No" - is_adm = "Yes" if udata.get("is_admin") else "No" - status = udata.get("login_status") or "offline" - last_login = udata.get("last_login") or "Never" - print(f"{uid:<15} | {is_act:<8} | {is_adm:<8} | {status:<10} | {last_login}") - print("-" * 75) - except Exception as e: - print(f"[!] Health check query error: {e}") - -def menu_initialize_directories(): - print("\n--- Initialize System Data Directories ---") - base_dir = get_base_dir() - paths = Paths(base_dir=base_dir) - print(f"[*] Base Directory: {paths.base_dir}") - try: - paths.ensure_dirs() - print("[OK] All required directories verified and initialized successfully.") - except Exception as e: - print(f"[!] Error initializing directories: {e}") - -def menu_cleanup_sessions(): - print("\n--- Clear Expired Inactive Sessions ---") - init_auth() - try: - cleaned_count = SessionManager.cleanup_expired() - print(f"[OK] Cleaned up {cleaned_count} expired session(s).") - except Exception as e: - print(f"[!] Error cleaning up sessions: {e}") - -def run_utility_script(script_name, script_file): - print(f"\n--- Running Utility: {script_name} ({script_file}) ---") - script_path = os.path.join(SCRIPT_DIR, script_file) - if not os.path.exists(script_path): - print(f"[!] Script {script_file} not found in {SCRIPT_DIR}.") - return - - try: - print(f"[*] Executing {script_file}...\n") - subprocess.run([sys.executable, script_path], check=True, cwd=SCRIPT_DIR) - print(f"\n[OK] Script {script_file} completed successfully.") - except subprocess.CalledProcessError as e: - print(f"\n[!] Script {script_file} failed with exit code {e.returncode}.") - except Exception as e: - print(f"\n[!] Error executing script: {e}") - -def run_data_pipeline(): - print("\n================================================================================") - print(" 🚀 RUNNING FULL DATA PIPELINE (1 -> 5 IN SEQUENCE)") - print("================================================================================") - pipeline_scripts = [ - ("Download Daily Data", "nsedown.py"), - ("Process Corporate Actions", "nsecorpact.py"), - ("Process Daily Data", "nseprocessor.py"), - ("Process Max / Min", "nsesymhist.py"), - ("Download Live Data", "nsemaxlive.py"), - ] - for idx, (label, script_file) in enumerate(pipeline_scripts, 1): - print(f"\n[Step {idx}/{len(pipeline_scripts)}] Running {label} ({script_file})...") - run_utility_script(label, script_file) - print("\n================================================================================") - print(" ✅ FULL DATA PIPELINE EXECUTION COMPLETED") - print("================================================================================") - -def apply_corporate_action_pipeline(): - print("\n================================================================================") - print(" 🔄 APPLY CORPORATE ACTIONS PIPELINE") - print("================================================================================") - base_dir = get_base_dir() - processed_dir = os.path.join(base_dir, "processed") - - files_to_delete = [ - os.path.join(processed_dir, "nse_processed_data.parquet"), - os.path.join(processed_dir, "nse_processed_data.csv"), - os.path.join(processed_dir, "nse_summary_data.parquet"), - os.path.join(processed_dir, "nse_summary_data.csv"), - ] - - print("[*] Step 1: Deleting existing processed and extract data files...") - deleted_count = 0 - for file_path in files_to_delete: - if os.path.exists(file_path): - try: - os.remove(file_path) - print(f" - Deleted: {os.path.basename(file_path)}") - deleted_count += 1 - except Exception as e: - print(f" [!] Error deleting {os.path.basename(file_path)}: {e}") - if deleted_count == 0: - print(" - No existing processed/extract files found to delete.") - - pipeline_scripts = [ - ("Download Daily Data", "nsedown.py"), - ("Process Corporate Actions", "nsecorpact.py"), - ("Process Daily Data", "nseprocessor.py"), - ] - - print("\n[*] Step 2: Running pipeline modules sequentially (1 -> 3)...") - for idx, (label, script_file) in enumerate(pipeline_scripts, 1): - print(f"\n[Step {idx}/{len(pipeline_scripts)}] Running {label} ({script_file})...") - run_utility_script(label, script_file) - - print("\n================================================================================") - print(" ✅ CORPORATE ACTION PIPELINE COMPLETED") - print("================================================================================") - -def menu_utility_scripts(): - while True: - print("\n================================================================================") - print(" ⚙️ UTILITY SCRIPTS & DATA PIPELINE ⚙️") - print("================================================================================") - - scripts = list(UTILITY_SCRIPTS) - scripts.append(("Push to HuggingFace / Backup", "push_to_hf.py")) - - for idx, (label, filename) in enumerate(scripts, 1): - print(f" [{idx}] {label} ({filename})") - - run_pipeline_idx = len(scripts) + 1 - corpact_pipeline_idx = len(scripts) + 2 - - print(" ------------------------------------------------------------------------------") - print(f" [{run_pipeline_idx}] Run Data Pipeline (Execute 1-5 sequentially)") - print(f" [{corpact_pipeline_idx}] Apply Corporate Action (Delete processed data & execute 1,2,3)") - print(" ------------------------------------------------------------------------------") - print(" [0] Return to Main Menu") - print("================================================================================") - - choice = input(f"Select an option (0-{corpact_pipeline_idx}): ").strip() - if choice == '0': - break - - try: - val = int(choice) - if 1 <= val <= len(scripts): - label, filename = scripts[val - 1] - run_utility_script(label, filename) - elif val == run_pipeline_idx: - run_data_pipeline() - elif val == corpact_pipeline_idx: - apply_corporate_action_pipeline() - else: - print("[!] Invalid option.") - except ValueError: - print("[!] Please enter a valid number.") - - input("\nPress Enter to continue...") - -def print_interactive_menu(): - print(""" -================================================================================ - 🎓 NSEDASH - INITIALIZATION & SYSTEM ADMIN CONSOLE 🎓 -================================================================================ - [1] User Management: Reset Super Admin ('admin') Password - [2] User Management: Reset Any User Account Password - [3] User Management: Reset User TOTP Authentication - [4] User Management: Full Account Reset (Force Fresh Registration) - [5] User Management: Create New User Account - ------------------------------------------------------------------------------ - [6] System Health: Display Database Health & Real-time Statistics - [7] System Health: Initialize System Data Directories - [8] System Health: Clear Expired Inactive Sessions - ------------------------------------------------------------------------------ - [9] Utility Scripts: Run Data Pipeline / Sync Tasks - ------------------------------------------------------------------------------ - [0] Exit Administrative Console -================================================================================ -""") - -def interactive_console_menu(): - print_splash() - while True: - print_interactive_menu() - try: - choice = input("Select an option (0-9): ").strip() - if choice == '1': - menu_reset_admin_password() - elif choice == '2': - menu_reset_any_user_password() - elif choice == '3': - menu_reset_totp_for_user() - elif choice == '4': - menu_full_account_reset() - elif choice == '5': - menu_create_new_user() - elif choice == '6': - menu_database_health_and_stats() - elif choice == '7': - menu_initialize_directories() - elif choice == '8': - menu_cleanup_sessions() - elif choice == '9': - menu_utility_scripts() - elif choice == '0': - print("[*] Exiting Administrative Console. Goodbye!") - break - else: - print("[!] Invalid option. Please select a number between 0 and 9.") - except (KeyboardInterrupt, EOFError): - print("\n[*] Exiting Administrative Console. Goodbye!") - break - input("\nPress Enter to return to the main menu...") - - -def main(): - parser = argparse.ArgumentParser(description="NSEDashboard Database Initialization & Management Utility") - parser.add_argument('--reset-admin-password', type=str, help='Reset super admin password') - args = parser.parse_args() - - if args.reset_admin_password: - init_auth() - pwd_hash = generate_password_hash(args.reset_admin_password, method='pbkdf2:sha256') - UserDatabase.update_user('admin', { - 'password_hash': pwd_hash, - 'active': True - }) - SessionManager.clear_sessions_for_user('admin') - print("[*] Admin password reset successfully.") - return - - # Default to interactive console application mode when executed in terminal - interactive_console_menu() - -if __name__ == '__main__': - main() diff --git a/nsedash_api.py b/nsedash_api.py deleted file mode 100644 index bc706326ec8f6f78b8fa6558ac30965dd38aa0ae..0000000000000000000000000000000000000000 --- a/nsedash_api.py +++ /dev/null @@ -1,653 +0,0 @@ -#!/usr/bin/env python3 -""" -NSEDash Data Source API Server -============================= - -This module provides high-performance, responsive endpoints to retrieve stock symbol data -from two local parquet files: -1. nse_processed_data.parquet (EOD stock metrics) on port 8006. -2. nse_extract_data.parquet (EOD metrics + engineered indicators) on port 8007. - -Key Architectural Decisions: ---------------------------- -- Concurrent Dual Apps: Runs two FastAPI applications concurrently using multiprocessing. -- In-Memory Indexing: Loads and caches Parquet datasets grouped by SYMBOL on startup for O(1) lookups. -- Hot-Reload Monitoring: Checks Parquet file modified times (mtime) in a background thread - and reloads data thread-safely using a Reentrant Lock (RLock) without causing service downtime. -- Gzip HTTP Compression: Employs GZipMiddleware to automatically compress JSON HTTP payloads. -- Binary Parquet Option: Allows raw binary Parquet outputs via the `format=parquet` query parameter. -- Transit Encryption: Secures transit data over HTTPS via self-signed SSL/TLS keys and certificates. -- API Key Access Control: Restricts access to data endpoints using headers or query parameters. -- Command-line configuration: Configures port bindings, base directories, and hosts via argparse. - -Endpoints: ----------- -* GET /api/data/{symbol} -* GET /api/data - - Query parameters: - * start_date (optional, format: YYYY-MM-DD) - * end_date (optional, format: YYYY-MM-DD) - * format (optional, options: 'json', 'parquet', default: 'json') - * api_key (optional, alternative auth query parameter) - -Usage: ------- - python nsedash_api.py [--base-dir BASE_DIR] [--host HOST] [--port-processed PORT] [--port-extract PORT] -""" - -import argparse -import io -import logging -import multiprocessing -import os -import signal -import sys -import threading -import time -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Optional - -import pandas as pd -from fastapi import FastAPI, HTTPException, Query, Response, Security, Depends -from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.security.api_key import APIKeyHeader, APIKeyQuery - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s" -) -logger = logging.getLogger("nsedash_api") - - -class ParquetStore: - """ - Thread-safe in-memory cache manager for a Parquet dataset. - - This class loads a Parquet file, indexes it by SYMBOL, and serves structured - slices by date. It monitors the Parquet file's modification time (mtime) - on disk and reloads it dynamically when changes are detected. - """ - - def __init__(self, filepath: Path): - """ - Initialize the ParquetStore and load data for the first time. - - Args: - filepath (Path): Path to the target Parquet file. - """ - self.filepath = Path(filepath) - self.lock = threading.RLock() - self.cache: Dict[str, pd.DataFrame] = {} - self.last_mtime: float = 0.0 - self.reload() - - def reload(self) -> None: - """ - Thread-safely reloads Parquet data from disk if the file mtime has changed. - - This method reads the file, parses transaction dates, handles categorical data, - sorts values, splits the DataFrame by SYMBOL, and performs an atomic swap of the cache. - """ - with self.lock: - if not self.filepath.exists(): - logger.error(f"[STORE] Parquet file not found: {self.filepath}") - return - - try: - current_mtime = os.path.getmtime(self.filepath) - except OSError as e: - logger.error(f"[STORE] Failed to read mtime for {self.filepath}: {e}") - return - - if current_mtime == self.last_mtime and self.cache: - # No change on disk, skip reload - return - - logger.info(f"[STORE] Loading parquet file: {self.filepath} (change detected)...") - start_time = time.time() - try: - # Load parquet from disk - df = pd.read_parquet(self.filepath) - - if "SYMBOL" not in df.columns or "TRAN_DATE" not in df.columns: - raise ValueError("Parquet file must contain 'SYMBOL' and 'TRAN_DATE' columns.") - - # Convert transaction date to datetime object - df["TRAN_DATE"] = pd.to_datetime(df["TRAN_DATE"]) - - # De-categoricalize for compatibility and stability - for col in df.select_dtypes(include=["category"]).columns: - df[col] = df[col].astype(str) - - # Ensure uppercase symbol mapping - df["SYMBOL"] = df["SYMBOL"].astype(str).str.strip().str.upper() - - # Chronological sorting for range filtering - df = df.sort_values(by=["SYMBOL", "TRAN_DATE"]) - - # Group by SYMBOL to build O(1) key cache - new_cache: Dict[str, pd.DataFrame] = {} - grouped = df.groupby("SYMBOL", observed=True) - for sym, group in grouped: - new_cache[str(sym)] = group.reset_index(drop=True) - - # Swap references atomically - self.cache = new_cache - self.last_mtime = current_mtime - elapsed = time.time() - start_time - print( - f"[STORE] Loaded {self.filepath.name} in {elapsed:.2f}s. " - f"Total symbols cached: {len(new_cache)}", - flush=True - ) - except Exception as e: - logger.error(f"[STORE] Failed loading Parquet file {self.filepath}: {e}") - print(f"[STORE ERROR] Failed loading Parquet file {self.filepath}: {e}", flush=True) - - def get_data(self, symbol: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> pd.DataFrame: - """ - Thread-safely retrieves and filters data for a specific stock symbol and date range. - - Args: - symbol (str): Stock ticker symbol. - start_date (Optional[str]): ISO start date YYYY-MM-DD. - end_date (Optional[str]): ISO end date YYYY-MM-DD. - - Returns: - pd.DataFrame: A filtered slice of data matching the criteria. - """ - with self.lock: - group = self.cache.get(symbol.upper()) - if group is None or group.empty: - return pd.DataFrame() - - mask = pd.Series(True, index=group.index) - if start_date: - mask &= (group["TRAN_DATE"] >= pd.to_datetime(start_date)) - if end_date: - mask &= (group["TRAN_DATE"] <= pd.to_datetime(end_date)) - - return group[mask].copy() - - -def start_monitoring_thread(store: ParquetStore, interval_seconds: int = 10) -> None: - """ - Spawns a daemon thread that periodically triggers ParquetStore check-and-reload. - - Args: - store (ParquetStore): Store instance to monitor. - interval_seconds (int): Polling frequency. - """ - def check_loop(): - while True: - time.sleep(interval_seconds) - try: - store.reload() - except Exception as e: - logger.error(f"[MONITOR] Error monitoring {store.filepath}: {e}") - - monitor_thread = threading.Thread(target=check_loop, daemon=True) - monitor_thread.start() - - -def make_app(store: ParquetStore, port: int, api_key: str) -> FastAPI: - """ - FastAPI Application factory configuring routes, logging, security, and middleware. - - Args: - store (ParquetStore): Data cache store to query against. - port (int): Listening port for logging representation. - api_key (str): Allowed API key value. - - Returns: - FastAPI: Ready-to-run web application. - """ - app = FastAPI( - title=f"NSEDash Data Source API - Port {port}", - description="Failsafe, responsive, and secure symbol data source over HTTPS." - ) - - # API key extractors - API_KEY_NAME = "X-API-Key" - api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) - api_key_query = APIKeyQuery(name="api_key", auto_error=False) - - def verify_api_key( - header_key: Optional[str] = Security(api_key_header), - query_key: Optional[str] = Security(api_key_query) - ) -> None: - """Validates API Key using headers or query parameters.""" - if header_key == api_key or query_key == api_key: - return - raise HTTPException( - status_code=403, - detail="Could not validate credentials. Invalid API Key. Provide X-API-Key header or api_key query parameter." - ) - - # Enable CORS to allow direct requests from browsers - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - - # Enable transparent Gzip encoding response compression - app.add_middleware(GZipMiddleware, minimum_size=1000) - - @app.middleware("http") - async def log_requests(request, call_next): - """Logs each API data request with port, duration, size, authentication, and status.""" - start_time = time.time() - response = await call_next(request) - process_time = (time.time() - start_time) * 1000 - - path = request.url.path - if path.startswith("/api/data"): - client_ip = request.client.host if request.client else "unknown" - status_code = response.status_code - content_length = response.headers.get("content-length", "unknown") - if content_length != "unknown": - size_str = f"{float(content_length)/1024:.2f}KB" - else: - size_str = "unknown" - - auth_status = "AUTHORIZED" if status_code != 403 else "UNAUTHORIZED" - - # Print request trace to terminal - print( - f"[API REQUEST] {request.method} {request.url.path} from {client_ip} | " - f"Status: {status_code} ({auth_status}) | Size: {size_str} | " - f"Time: {process_time:.2f}ms | Port: {port}", - flush=True - ) - - return response - - @app.get("/api/data/{symbol}", dependencies=[Depends(verify_api_key)]) - def get_data_by_symbol( - symbol: str, - start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"), - end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"), - format: str = Query("json", description="Response format: 'json' or 'parquet'") - ): - """ - Retrieves filtered EOD data for a stock symbol (requires API Key auth). - - Supports output format of either Gzip-compressed JSON or raw binary Parquet. - """ - # Validate date formats - if start_date: - try: - pd.to_datetime(start_date) - except Exception: - raise HTTPException(status_code=400, detail="Invalid start_date format. Use YYYY-MM-DD.") - if end_date: - try: - pd.to_datetime(end_date) - except Exception: - raise HTTPException(status_code=400, detail="Invalid end_date format. Use YYYY-MM-DD.") - - symbol = symbol.strip().upper() - df_slice = store.get_data(symbol, start_date, end_date) - - if format.lower() == "parquet": - # Output raw Parquet bytes in-memory buffer - buffer = io.BytesIO() - df_slice.to_parquet(buffer, index=False, compression="snappy") - buffer.seek(0) - return Response( - content=buffer.getvalue(), - media_type="application/octet-stream", - headers={ - "Content-Disposition": f"attachment; filename={symbol}_data.parquet" - } - ) - else: - # Default to JSON format - df_json = df_slice.copy() - if not df_json.empty: - df_json["TRAN_DATE"] = df_json["TRAN_DATE"].dt.strftime("%Y-%m-%d") - records = df_json.to_dict(orient="records") - return {"symbol": symbol, "count": len(records), "data": records} - - @app.get("/api/data", dependencies=[Depends(verify_api_key)]) - def get_data_query( - symbol: str = Query(..., description="NSE Stock Symbol"), - start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"), - end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"), - format: str = Query("json", description="Response format: 'json' or 'parquet'") - ): - """Alternative endpoint allowing parameters to be passed entirely via query parameters.""" - return get_data_by_symbol(symbol=symbol, start_date=start_date, end_date=end_date, format=format) - - @app.get("/api/indicators", dependencies=[Depends(verify_api_key)]) - def get_indicators(): - """Returns the list of indicator and feature columns present in the Parquet store.""" - with store.lock: - if not store.cache: - return {"indicators": []} - first_df = next(iter(store.cache.values())) - columns = list(first_df.columns) - exclude = {"SYMBOL", "TRAN_DATE"} - indicators = [col for col in columns if col not in exclude] - return {"indicators": indicators} - - @app.get("/status") - def status(): - """Returns internal state metrics of the Parquet store and cached items (No auth required).""" - return { - "status": "online", - "file": str(store.filepath), - "last_mtime": store.last_mtime, - "symbols_count": len(store.cache) - } - - return app - - -def ensure_ssl_credentials(base_dir: Path) -> tuple[Path, Path]: - """ - Ensures that self-signed SSL certificate and key files exist on disk. - If not, dynamically generates them under processed/security/ in a thread-safe way. - - Returns: - tuple[Path, Path]: Absolute paths to (cert_path, key_path). - """ - security_dir = base_dir / "processed" / "security" - security_dir.mkdir(parents=True, exist_ok=True) - - cert_path = security_dir / "api_self_signed.crt" - key_path = security_dir / "api_self_signed.key" - - if cert_path.exists() and key_path.exists(): - return cert_path, key_path - - logger.info("[SSL] Generating new self-signed SSL credentials for localhost...") - try: - import datetime - import ipaddress - from cryptography import x509 - from cryptography.x509.oid import NameOID - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.asymmetric import rsa - from cryptography.hazmat.primitives import serialization - - # 1. Generate private key - private_key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048 - ) - - # Write private key - key_path.write_bytes( - private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption() - ) - ) - - # 2. Build self-signed certificate - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, "IN"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Maharashtra"), - x509.NameAttribute(NameOID.LOCALITY_NAME, "Mumbai"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, "NSEDash"), - x509.NameAttribute(NameOID.COMMON_NAME, "localhost"), - ]) - - cert = ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(private_key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1)) - .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365)) - .add_extension( - x509.SubjectAlternativeName([ - x509.DNSName("localhost"), - x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")) - ]), - critical=False, - ) - .sign(private_key, hashes.SHA256()) - ) - - # Write certificate - cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) - logger.info(f"[SSL] Successfully generated certificate at {cert_path}") - except Exception as e: - logger.error(f"[SSL] Failed generating self-signed SSL credentials: {e}") - raise RuntimeError(f"Failed to establish SSL/TLS encryption for transit: {e}") - - return cert_path, key_path - - -def ensure_api_key(base_dir: Path) -> str: - """ - Ensures that a persistent API key exists for data source requests. - Loads it from processed/security/api_key.txt if present, otherwise generates a new one. - - Returns: - str: Secure API key token. - """ - security_dir = base_dir / "processed" / "security" - security_dir.mkdir(parents=True, exist_ok=True) - key_file = security_dir / "api_key.txt" - - if key_file.exists(): - try: - api_key = key_file.read_text(encoding="utf-8").strip() - if api_key: - return api_key - except Exception as e: - logger.error(f"[AUTH] Error reading API key file: {e}") - - # Generate a new secure token - import secrets - api_key = secrets.token_urlsafe(32) - try: - key_file.write_text(api_key, encoding="utf-8") - logger.info(f"[AUTH] Generated new API key at {key_file}") - except Exception as e: - logger.error(f"[AUTH] Failed to save API key to file: {e}") - - return api_key - - -def start_server_process( - base_dir: str, file_path: str, host: str, port: int, api_key: str, cert_path: str, key_path: str -) -> None: - """ - Subprocess entry point to spin up a single Port FastAPI Uvicorn listener. - - Args: - base_dir (str): Working directory path. - file_path (str): Relative path to source Parquet file under base_dir. - host (str): Bound interface address. - port (int): Assigned port. - api_key (str): Allowed API Key validation token. - cert_path (str): Path to self-signed SSL cert. - key_path (str): Path to self-signed SSL private key. - """ - print(f"[*] Initializing store on port {port} for: {file_path}", flush=True) - try: - # Load Parquet Store - store = ParquetStore(Path(base_dir) / file_path) - - # Start file modification tracking - start_monitoring_thread(store, interval_seconds=10) - - # Build FastAPI App - app = make_app(store, port, api_key) - - import uvicorn - # Run Uvicorn listening on port over HTTPS - uvicorn.run( - app, - host=host, - port=port, - ssl_keyfile=key_path, - ssl_certfile=cert_path, - log_level="warning" - ) - except KeyboardInterrupt: - pass - except Exception as e: - logger.error(f"[SERVER PROCESS ERROR] Failed to start on port {port}: {e}") - print(f"[!] Server Process Error on port {port}: {e}", flush=True) - - -def print_splash(host: str, port_processed: int, port_extract: int, base_dir: str, api_key: str) -> None: - """ - Prints a beautiful ANSI-colored ASCII splash screen banner on console. - - Args: - host (str): Base interface IP. - port_processed (int): Port of the processed dataset endpoint. - port_extract (int): Port of the feature extract dataset endpoint. - base_dir (str): Working base path. - api_key (str): Active API token key. - """ - is_tty = sys.stdout.isatty() - GREEN = "\033[92m" if is_tty else "" - CYAN = "\033[96m" if is_tty else "" - RESET = "\033[0m" if is_tty else "" - BOLD = "\033[1m" if is_tty else "" - YELLOW = "\033[93m" if is_tty else "" - - banner = rf"""{GREEN}{BOLD} -======================================================================== - _ _ ____ _____ ____ _ ____ _ _ _ ____ ___ - | \ | / ___|| ____| _ \ / \ / ___|| |_| | / \ | _ \_ _| - | \| \___ \| _| | | | / _ \\___ \| _ | / _ \ | |_) | | - | |\ |___) | |___| |_| / ___ \___) | | | | / ___ \| __/| | - |_| \_|____/|_____|____/_/ \_\____/|_| |_|/_/ \_\_| |___| -========================================================================{RESET} - * {CYAN}NSE Stock Data Source API Service (FastAPI v0.136 over HTTPS){RESET} - * Base Directory: {CYAN}{base_dir}{RESET} - * Processed Data: {CYAN}https://{host}:{port_processed}/api/data/{{symbol}}{RESET} - * Extract Data: {CYAN}https://{host}:{port_extract}/api/data/{{symbol}}{RESET} - * API Key (Auth): {YELLOW}{api_key}{RESET} - * Shutdown: {GREEN}Press Ctrl+C to stop services.{RESET} -======================================================================== -""" - print(banner, flush=True) - - -# Multiprocess worker references -processes: List[multiprocessing.Process] = [] - - -def signal_handler(sig, frame) -> None: - """SIGINT callback ensuring concurrent children processes are cleaned up.""" - print("\n[Shutdown] Terminating running API server instances...", flush=True) - for p in processes: - if p.is_alive(): - try: - p.terminate() - p.join(timeout=1.0) - if p.is_alive(): - p.kill() - except Exception: - pass - print("[Shutdown] Clean exit completed.") - sys.exit(0) - - -def main() -> None: - """Primary parser and process coordinator orchestration loop.""" - parser = argparse.ArgumentParser( - description="NSEDash stock symbol data source API server.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--base-dir", - default=None, - help="Base directory containing the processed Parquet files." - ) - parser.add_argument( - "--host", - default="0.0.0.0", - help="Host interface to bind the API servers to." - ) - parser.add_argument( - "--port-processed", - type=int, - default=8006, - help="Port to run the processed API server on." - ) - parser.add_argument( - "--port-extract", - type=int, - default=8007, - help="Port to run the extract API server on." - ) - args = parser.parse_args() - - # Resolve working base directory - base_dir = args.base_dir or os.environ.get("NSEDASH_BASE_DIR") or str(Path.cwd()) - base_path = Path(base_dir) - - # Relative paths under base directory - processed_file = "processed/nse_processed_data.parquet" - extract_file = "processed/nse_extract_data.parquet" - - # Pre-flight files verification - if not (base_path / processed_file).exists(): - print(f"[ERROR] Processed parquet file not found: {base_path / processed_file}", flush=True) - sys.exit(1) - if not (base_path / extract_file).exists(): - print(f"[ERROR] Extract parquet file not found: {base_path / extract_file}", flush=True) - sys.exit(1) - - # Generate or load SSL credentials - cert_path, key_path = ensure_ssl_credentials(base_path) - - # Generate or load API Key - api_key = ensure_api_key(base_path) - - # Expose key via env variable for tests or local inspection - os.environ["NSEDASH_API_KEY"] = api_key - - print_splash(args.host, args.port_processed, args.port_extract, base_dir, api_key) - - # Set signal traps - signal.signal(signal.SIGINT, signal_handler) - - # Use 'spawn' for safe cross-platform multiprocessing startup behaviour - multiprocessing.set_start_method("spawn", force=True) - - # Processed database process (port 8006) - p1 = multiprocessing.Process( - target=start_server_process, - args=(base_dir, processed_file, args.host, args.port_processed, api_key, str(cert_path), str(key_path)), - name="api-processed" - ) - # Extract database process (port 8007) - p2 = multiprocessing.Process( - target=start_server_process, - args=(base_dir, extract_file, args.host, args.port_extract, api_key, str(cert_path), str(key_path)), - name="api-extract" - ) - - processes.append(p1) - processes.append(p2) - - # Boot processes - p1.start() - p2.start() - - # Keep primary coordinator thread running - while True: - time.sleep(1) - - -if __name__ == "__main__": - main() diff --git a/nsedash_app.py b/nsedash_app.py deleted file mode 100644 index 1ca48fa592394b5e0d21af956870d1cb8238ae81..0000000000000000000000000000000000000000 --- a/nsedash_app.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -NSE Stock Dashboard (NSEDashboard) Application Entry Point. - -This module serves as the primary executable entry point for the NSE Stock -Dashboard Flask application. It parses command-line arguments to configure -the server settings (base directory, host interface, port, and debug mode), -initializes the Flask application with all required services, and starts -the Flask WSGI server with multithreading enabled. - -Features: -- Configurable base data directory via command line or environment variables. -- Integrated background task processing framework. -- Multi-threaded Flask server configuration to maintain frontend responsiveness. - -Usage: - python nsedash_app.py [--base-dir BASE_DIR] [--host HOST] [--port PORT] [--debug] - -Options: - --base-dir DIR Specify the directory containing the processed stock data files. - If not provided, the current working directory or the value of the - $NSEDASH_BASE_DIR environment variable is used. - --host HOST Host interface to bind the web server to (default: 127.0.0.1). - --port PORT Port to listen on (default: 5000). - --debug Run the Flask application in debug mode. -""" - -import argparse -import os -import signal -import sys -import time - -# --------------------------------------------------------------------------- -# CLI SWITCHES (mlbench pattern — parsed before argparse for env var side-effects) -# --------------------------------------------------------------------------- - -#: -hf Force HuggingFace mode: bind 0.0.0.0:7860, activate HF dataset sync -ENABLE_HF = '-hf' in sys.argv or '--hf' in sys.argv - -#: -noadmin Disable admin-only tools (script runner, data utilities) -ENABLE_NOADMIN = '-noadmin' in sys.argv or '--noadmin' in sys.argv - -if ENABLE_HF: - os.environ["HF_MODE"] = "1" - print("[*] HF mode (-hf): Enabled — binding 0.0.0.0:7860, HF dataset sync active.") - -if ENABLE_NOADMIN: - os.environ["ADMIN_MODE"] = "0" - print("[*] No-admin (-noadmin): Enabled — admin tools disabled.") - -# Disable CUDA inside Hugging Face Spaces mode to save resources -if "SPACE_ID" in os.environ or ENABLE_HF: - os.environ["CUDA_VISIBLE_DEVICES"] = "-1" - os.environ["HF_MODE"] = "1" # Also set from SPACE_ID detection - -from nsedash.web import create_app - - -def get_gpu_info() -> str: - gpu_info = "CPU Only (No GPU detected)" - try: - import torch - if torch.cuda.is_available(): - gpu_info = f"CUDA Active - {torch.cuda.get_device_name(0)}" - except ImportError: - pass - - try: - if gpu_info == "CPU Only (No GPU detected)": - import tensorflow as tf - gpus = tf.config.list_physical_devices('GPU') - if gpus: - gpu_info = f"TensorFlow Active - {len(gpus)} GPU(s)" - except ImportError: - pass - return gpu_info - - -def print_console_splash(host, port): - display_host = "localhost" if host in ("127.0.0.1", "0.0.0.0") else host - gpu_info = get_gpu_info() - is_hf = os.environ.get("HF_MODE") == "1" or "SPACE_ID" in os.environ - admin_enabled = os.environ.get("ADMIN_MODE", "1") == "1" - - nvidia_info = "Not Tested" - try: - from nsedash.core.nvidia_llm import get_nvidia_api_status - status = get_nvidia_api_status() - if status.get("success"): - nvidia_info = f"Tested & Successful ({status.get('source', 'Keyring/Env')})" - else: - nvidia_info = f"Failed/Warning ({status.get('message', 'Key missing')})" - except Exception as e: - nvidia_info = f"Error: {e}" - - banner = r""" -====================================================================== - _ _ ____ _____ ____ _ ____ _ _ ____ ___ _ ____ ____ - | \ | / ___|| ____| _ \ / \ / ___|| | | | _ \ / _ \ / \ | _ \| _ \ - | \| \___ \| _| | | | / _ \\___ \| |_| | |_) | | | / _ \| |_) | | | | - | |\ |___) | |___| |_| / ___ \___) | _ | _ <| |_| / ___ \ _ <| |_| | - |_| \_|____/|_____|____/_/ \_\____/|_| |_|_| \_\\___/_/ \_\_| \_\____/ -====================================================================== -""" - print(banner) - print(f" * NSE Stock Dashboard (Flask v5.0)") - print(f" * Running on: http://{display_host}:{port}/") - print(f" * GPU in use: {gpu_info}") - print(f" * NVIDIA API: {nvidia_info}") - print(f" * HF Mode: {'Enabled (HF dataset sync active)' if is_hf else 'Disabled (local mode)'}") - print(f" * Admin Mode: {'Enabled' if admin_enabled else 'Disabled (-noadmin)'}") - print(f" * Auth: userid + password + TOTP (mlbench pattern)") - print(f" * Press Ctrl+C in console to exit gracefully.") - print("======================================================================\n") - - -_shutdown_in_progress = False - -def register_shutdown_handler(app): - def handle_sigint(signum, frame): - global _shutdown_in_progress - if _shutdown_in_progress: - return - _shutdown_in_progress = True - - print("\n[Shutdown] Gracefully shutting down nsedash backend...") - - # 1. Mark as exiting so UI status calls see it - try: - service = app.extensions["nsedash"]["service"] - service.is_exiting = True - except Exception: - pass - - # 2. Wait a short moment for UI to poll/receive the exiting state - time.sleep(1.5) - - # 3. Clean up backend components - try: - scheduler = app.extensions["nsedash"]["scheduler"] - scheduler.stop() - except Exception: - pass - - try: - scripts = app.extensions["nsedash"]["scripts"] - for run in list(scripts._runs.values()): - if run.process and run.process.poll() is None: - run.process.terminate() - try: - run.process.wait(timeout=1.0) - except Exception: - run.process.kill() - except Exception: - pass - - try: - runtime = app.extensions["nsedash"]["runtime"] - runtime.shutdown() - except Exception: - pass - - print("[Shutdown] Backend shutdown complete. Exiting process.") - sys.exit(0) - - signal.signal(signal.SIGINT, handle_sigint) - - -def main(): - parser = argparse.ArgumentParser( - description="NSE Stock Dashboard (Flask)", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -CLI switches (mlbench pattern, checked before argparse): - -hf Force HuggingFace mode: 0.0.0.0:7860, HF dataset sync - -noadmin Disable admin tools (script runner, data utilities) -""", - ) - parser.add_argument("--base-dir", default=None, - help="Base data directory (default: CWD or $NSEDASH_BASE_DIR)") - - is_hf = ENABLE_HF or ("SPACE_ID" in os.environ) - default_host = "0.0.0.0" if is_hf else "127.0.0.1" - default_port = 7860 if is_hf else 5000 - - parser.add_argument("--host", default=default_host) - parser.add_argument("--port", type=int, default=default_port) - parser.add_argument("--debug", action="store_true") - parser.add_argument("--hf", action="store_true", help="Force HF mode (same as -hf switch)") - parser.add_argument("--noadmin", action="store_true", help="Disable admin tools (same as -noadmin switch)") - args = parser.parse_args() - - # argparse flags also set env vars (in case user passed --hf instead of -hf) - if args.hf: - os.environ["HF_MODE"] = "1" - if args.noadmin: - os.environ["ADMIN_MODE"] = "0" - - app = create_app(args.base_dir) - - # Register shutdown handler & show splash screen - register_shutdown_handler(app) - print_console_splash(args.host, args.port) - - # threaded=True: request handlers stay responsive while background - # crews run on the dedicated asyncio thread. - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) - - -if __name__ == "__main__": - main() diff --git a/nsedash_app.zip b/nsedash_app.zip index 3cb398e00e8dfeb672def32910206a2d1bbdaf5f..7ec7cafaea1df273ff5e336edb912907711e2cad 100644 --- a/nsedash_app.zip +++ b/nsedash_app.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8af521a8d35b3db6a137744d3f22231165db988c9478722c77c97eb6289d300e -size 969257 +oid sha256:2f1000977e348beba9d01290d3b32681fd777d81e2f33fa39d0a76b58ca3830b +size 971914 diff --git a/nsedash_start.py b/nsedash_start.py index 8461b35d77c59bf7b5712025a5250349757f279a..378d268f0cdbeaab0fcf640c58d0aa7e5eca287a 100644 --- a/nsedash_start.py +++ b/nsedash_start.py @@ -44,7 +44,7 @@ def _silent_loader(): if not app_script.exists(): app_script = Path("nsedash_app.py") - args = [sys.executable, str(app_script), "-hf"] + args = [sys.executable, str(app_script), "--hf"] try: os.execv(sys.executable, args) except Exception: diff --git a/nsedown.py b/nsedown.py deleted file mode 100644 index 3388ba9de066b7366b1c0547f3f5f989d149a797..0000000000000000000000000000000000000000 --- a/nsedown.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -""" -NSE Bhavcopy Bulk Downloader (Standalone CLI Utility) - -This script downloads NSE bhavcopy ZIP files from NSE archives. -It handles downloading files from Jan 1, 2020 to current date, -organizing them in year-wise folders, and maintaining download logs. - -Features: -- CLI Argument support (--days, --output, --verify) -- Interactive fallback if run without arguments -- Year-wise file organization -- Download status tracking and log files -- Existing file verification (avoids duplicate downloads) -- Support for Indian market holidays and Diwali Muhurat Trading sessions (up to 2026) -""" - -import argparse -import logging -import os -import sys -import time -import zipfile -from datetime import datetime, timedelta -from pathlib import Path -from typing import List, Set - -import requests - - -class NSEDownloader: - """ - NSE Bhavcopy file downloader class. - Handles downloading of bhavcopy files from NSE archives website. - """ - - def __init__(self, days_back: int = 0, downloads_dir: str = "downloads"): - """ - Initialize downloader. - Args: - days_back: Number of days to look back (0 for full download from 2020-01-01) - downloads_dir: Directory where files should be downloaded - """ - self.days_back = days_back - self.base_url = "https://nsearchives.nseindia.com/archives/equities/bhavcopy/pr" - self.base_dir = Path(downloads_dir) - self.downloads_dir = self.base_dir - - self.headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - } - - self.downloaded_files = [] - self.skipped_files = [] - self.failed_files = [] - - self._setup_trading_holidays() - self.setup_logging() - self.setup_directories() - - # Muhurat Trading Days (Diwali Laxmi Pujan) - # Note: These are Sunday/Saturday holiday dates when special trading sessions occur. - self.muhurat_dates = { - datetime(2020, 11, 14), # Diwali 2020 - datetime(2021, 11, 4), # Diwali 2021 - datetime(2022, 10, 24), # Diwali 2022 - datetime(2023, 11, 12), # Diwali 2023 - datetime(2024, 10, 31), # Diwali 2024 - datetime(2025, 10, 21), # Diwali 2025 - datetime(2026, 11, 8), # Diwali 2026 - } - - def _setup_trading_holidays(self): - """Setup NSE trading holidays from 2020 to 2026""" - self.trading_holidays = { - # 2020 Holidays - datetime(2020, 1, 26).date(), # Republic Day - datetime(2020, 2, 21).date(), # Mahashivratri - datetime(2020, 3, 10).date(), # Holi - datetime(2020, 4, 2).date(), # Ram Navami - datetime(2020, 4, 6).date(), # Mahavir Jayanti - datetime(2020, 4, 10).date(), # Good Friday - datetime(2020, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2020, 5, 1).date(), # Maharashtra Day - datetime(2020, 5, 25).date(), # Eid-Ul-Fitr - datetime(2020, 8, 15).date(), # Independence Day - datetime(2020, 10, 2).date(), # Gandhi Jayanti - datetime(2020, 11, 16).date(), # Diwali Balipratipada - datetime(2020, 11, 30).date(), # Gurunanak Jayanti - - # 2021 Holidays - datetime(2021, 1, 26).date(), # Republic Day - datetime(2021, 3, 11).date(), # Mahashivratri - datetime(2021, 3, 29).date(), # Holi - datetime(2021, 4, 2).date(), # Good Friday - datetime(2021, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2021, 4, 21).date(), # Ram Navami - datetime(2021, 5, 1).date(), # Maharashtra Day - datetime(2021, 5, 13).date(), # Eid-Ul-Fitr - datetime(2021, 8, 15).date(), # Independence Day - datetime(2021, 9, 10).date(), # Ganesh Chaturthi - datetime(2021, 10, 2).date(), # Gandhi Jayanti - datetime(2021, 11, 4).date(), # Diwali - datetime(2021, 11, 5).date(), # Diwali Balipratipada - datetime(2021, 11, 19).date(), # Gurunanak Jayanti - - # 2022 Holidays - datetime(2022, 1, 26).date(), # Republic Day - datetime(2022, 3, 1).date(), # Mahashivratri - datetime(2022, 3, 18).date(), # Holi - datetime(2022, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2022, 4, 15).date(), # Good Friday - datetime(2022, 5, 1).date(), # Maharashtra Day - datetime(2022, 5, 3).date(), # Eid-Ul-Fitr - datetime(2022, 8, 15).date(), # Independence Day - datetime(2022, 10, 2).date(), # Gandhi Jayanti - datetime(2022, 10, 5).date(), # Dussehra - datetime(2022, 10, 24).date(), # Diwali Laxmi Pujan - datetime(2022, 10, 26).date(), # Diwali Balipratipada - datetime(2022, 11, 8).date(), # Gurunanak Jayanti - - # 2023 Holidays - datetime(2023, 1, 26).date(), # Republic Day - datetime(2023, 3, 7).date(), # Holi - datetime(2023, 3, 30).date(), # Ram Navami - datetime(2023, 4, 4).date(), # Mahavir Jayanti - datetime(2023, 4, 7).date(), # Good Friday - datetime(2023, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2023, 4, 22).date(), # Eid-Ul-Fitr - datetime(2023, 5, 1).date(), # Maharashtra Day - datetime(2023, 8, 15).date(), # Independence Day - datetime(2023, 9, 19).date(), # Ganesh Chaturthi - datetime(2023, 10, 2).date(), # Gandhi Jayanti - datetime(2023, 10, 24).date(), # Dussehra - datetime(2023, 11, 14).date(), # Diwali Balipratipada - datetime(2023, 11, 27).date(), # Gurunanak Jayanti - - # 2024 Holidays - datetime(2024, 1, 26).date(), # Republic Day - datetime(2024, 3, 8).date(), # Maha Shivaratri - datetime(2024, 3, 25).date(), # Holi - datetime(2024, 3, 29).date(), # Good Friday - datetime(2024, 4, 11).date(), # Eid-Ul-Fitr - datetime(2024, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2024, 4, 17).date(), # Ram Navami - datetime(2024, 5, 1).date(), # Maharashtra Day - datetime(2024, 8, 15).date(), # Independence Day - datetime(2024, 10, 2).date(), # Gandhi Jayanti - datetime(2024, 10, 31).date(), # Diwali - datetime(2024, 11, 1).date(), # Diwali Balipratipada - datetime(2024, 11, 15).date(), # Gurunanak Jayanti - - # 2025 Holidays - datetime(2025, 1, 26).date(), # Republic Day - datetime(2025, 2, 26).date(), # Mahashivratri - datetime(2025, 3, 14).date(), # Holi - datetime(2025, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2025, 4, 18).date(), # Good Friday - datetime(2025, 5, 1).date(), # Maharashtra Day - datetime(2025, 8, 15).date(), # Independence Day - datetime(2025, 10, 2).date(), # Gandhi Jayanti - datetime(2025, 10, 20).date(), # Dussehra - datetime(2025, 10, 22).date(), # Diwali Balipratipada - datetime(2025, 11, 4).date(), # Gurunanak Jayanti - - # 2026 Holidays - datetime(2026, 1, 15).date(), # Municipal Corporation Election - Maharashtra - datetime(2026, 1, 26).date(), # Republic Day - datetime(2026, 2, 15).date(), # Mahashivratri - datetime(2026, 3, 3).date(), # Holi - datetime(2026, 3, 21).date(), # Id-Ul-Fitr - datetime(2026, 3, 26).date(), # Ram Navami - datetime(2026, 3, 31).date(), # Mahavir Jayanti - datetime(2026, 4, 3).date(), # Good Friday - datetime(2026, 4, 14).date(), # Dr.Ambedkar Jayanti - datetime(2026, 5, 1).date(), # Maharashtra Day - datetime(2026, 5, 28).date(), # Bakri Id - datetime(2026, 6, 26).date(), # Muharram - datetime(2026, 8, 15).date(), # Independence Day - datetime(2026, 9, 14).date(), # Ganesh Chaturthi - datetime(2026, 10, 2).date(), # Gandhi Jayanti - datetime(2026, 10, 20).date(), # Dussehra - datetime(2026, 11, 8).date(), # Diwali Laxmi Pujan (Muhurat Trading day, but regular holiday) - datetime(2026, 11, 10).date(), # Diwali Balipratipada - datetime(2026, 11, 24).date(), # Gurunanak Jayanti - datetime(2026, 12, 25).date(), # Christmas - - # Additional Missing Historical Holidays (2020-2024) - datetime(2020, 12, 25).date(), # Christmas - datetime(2021, 7, 21).date(), # Bakrid - datetime(2021, 8, 19).date(), # Muharram - datetime(2021, 10, 15).date(), # Dussehra - datetime(2022, 8, 9).date(), # Muharram - datetime(2022, 8, 31).date(), # Ganesh Chaturthi - datetime(2023, 6, 29).date(), # Bakrid - datetime(2023, 12, 25).date(), # Christmas - datetime(2024, 1, 22).date(), # Ram Mandir Inauguration - datetime(2024, 5, 20).date(), # Buddha Purnima - datetime(2024, 6, 17).date(), # Bakrid - datetime(2024, 7, 17).date(), # Muharram - datetime(2024, 11, 20).date(), # Id-E-Milad - datetime(2024, 12, 25).date(), # Christmas - } - - def _is_weekend(self, date: datetime) -> bool: - """Check if date is a weekend (Saturday=5, Sunday=6)""" - return date.weekday() >= 5 - - def _is_valid_trading_day(self, date: datetime) -> bool: - """Check if date is valid for trading including Muhurat""" - # Always include Muhurat trading days - # We check date.replace(...) to normalize the time part to 00:00:00 - normalized_dt = date.replace(hour=0, minute=0, second=0, microsecond=0) - if normalized_dt in self.muhurat_dates: - return True - - # For regular days, check weekend and holidays - return not (self._is_weekend(date) or date.date() in self.trading_holidays) - - def setup_logging(self): - """Configure logging settings""" - log_format = '%(asctime)s - %(levelname)s - %(message)s' - logging.basicConfig( - level=logging.INFO, - format=log_format, - handlers=[ - logging.FileHandler(self.base_dir / 'download_log.txt'), - logging.StreamHandler(sys.stdout) - ] - ) - self.logger = logging.getLogger(__name__) - - def setup_directories(self): - """Create necessary directories and files""" - self.base_dir.mkdir(exist_ok=True, parents=True) - - self.success_file = self.base_dir / "files_downloaded.txt" - self.failed_file = self.base_dir / "download_failed.txt" - - self.success_file.touch(exist_ok=True) - self.failed_file.touch(exist_ok=True) - - def _get_date_range(self) -> List[datetime]: - """Get list of valid trading dates based on lookback period""" - # Determine date range - end_date = datetime.now() - if self.days_back > 0: - start_date = end_date - timedelta(days=self.days_back) - else: - start_date = datetime(2020, 1, 1) - - # Get all valid dates including Muhurat - valid_dates = [] - current = start_date - while current <= end_date: - if self._is_valid_trading_day(current): - valid_dates.append(current) - current += timedelta(days=1) - - return valid_dates - - def get_download_path(self, date: datetime) -> Path: - """Get year-wise subfolder path for download""" - year_dir = self.base_dir / str(date.year) - year_dir.mkdir(exist_ok=True, parents=True) - return year_dir - - def download_file(self, date: datetime) -> bool: - """Download and verify single file with Muhurat handling""" - filename = f"PR{date.strftime('%d%m%y')}.zip" - year_dir = self.get_download_path(date) - file_path = year_dir / filename - - # Track if it's a Muhurat trading day - normalized_dt = date.replace(hour=0, minute=0, second=0, microsecond=0) - is_muhurat = normalized_dt in self.muhurat_dates - - # Check if already downloaded and valid - if file_path.exists(): - try: - with zipfile.ZipFile(file_path) as zf: - if zf.testzip() is None: - self.skipped_files.append({ - 'date': date, - 'file': filename, - 'path': file_path, - 'is_muhurat': is_muhurat - }) - return True - except: - file_path.unlink() - - # Download with retries - for attempt in range(3): - try: - url = f"{self.base_url}/{filename}" - response = requests.get(url, headers=self.headers, timeout=30) - response.raise_for_status() - - with open(file_path, 'wb') as f: - f.write(response.content) - - with zipfile.ZipFile(file_path) as zf: - if zf.testzip() is None: - self.downloaded_files.append({ - 'date': date, - 'file': filename, - 'path': file_path, - 'size': file_path.stat().st_size, - 'is_muhurat': is_muhurat - }) - return True - - file_path.unlink() - - except Exception as e: - if attempt == 2: - self.failed_files.append({ - 'date': date, - 'file': filename, - 'error': str(e), - 'is_muhurat': is_muhurat - }) - return False - time.sleep(2 ** attempt) - - def print_summary(self, duration: float): - """Print detailed download summary with Muhurat indication""" - print("\nDownload Summary") - print("═" * 60) - - print(f"Time Taken: {duration:.1f} seconds") - print(f"Download Mode: {'Incremental' if self.days_back > 0 else 'Full'}") - if self.days_back > 0: - print(f"Days Processed: {self.days_back}") - print("─" * 60) - - # Downloaded files - if self.downloaded_files: - print("\n✓ Successfully Downloaded:") - total_size = sum(f['size'] for f in self.downloaded_files) - - for file in self.downloaded_files: - muhurat_tag = "[Muhurat]" if file['is_muhurat'] else "" - print(f" • {file['file']} {muhurat_tag} ({file['size'] / 1024:.1f} KB)") - print(f" Total Size: {total_size / 1024 / 1024:.2f} MB") - - # Skipped files - if self.skipped_files: - print("\n→ Already Existed:") - for file in self.skipped_files: - muhurat_tag = " [Muhurat]" if file['is_muhurat'] else "" - print(f" • {file['file']}{muhurat_tag}") - - # Failed files - if self.failed_files: - print("\n✗ Failed Downloads:") - for file in self.failed_files: - print(f" • {file['file']} - {file['error']}") - - print("═" * 60) - - def run(self): - """Execute download process with improved tracking""" - try: - dates = self._get_date_range() - if not dates: - print("✓ No files to download") - return - - start_time = time.time() - total = len(dates) - - for i, date in enumerate(dates, 1): - self.download_file(date) - - # Update progress - print(f"\rProgress: {i}/{total} " - f"[{len(self.downloaded_files)}⬇ " - f"{len(self.skipped_files)}⤍ " - f"{len(self.failed_files)}✗]", - end='', flush=True) - - time.sleep(0.2) - - # Print detailed summary - self.print_summary(time.time() - start_time) - - except KeyboardInterrupt: - print("\n\n✗ Download interrupted") - except Exception as e: - print(f"\n\n✗ Error: {e}") - logging.error(f"Download failed: {e}") - - -def self_verify(): - """Verify that holiday rules are working as intended""" - downloader = NSEDownloader(days_back=1, downloads_dir="downloads") - - test_dates = [ - (datetime(2026, 1, 15), False, "Maharashtra Municipal Elections (Thursday)"), - (datetime(2026, 1, 26), False, "Republic Day (Monday)"), - (datetime(2026, 3, 3), False, "Holi (Tuesday)"), - (datetime(2026, 11, 8), True, "Diwali Muhurat Trading (Sunday)"), - (datetime(2026, 11, 10), False, "Diwali-Balipratipada (Tuesday)"), - (datetime(2026, 11, 11), True, "Regular Wednesday"), - (datetime(2026, 11, 15), False, "Regular Sunday"), - ] - - passed = True - print("Self Verification Results:") - print("=" * 60) - for dt, expected, desc in test_dates: - actual = downloader._is_valid_trading_day(dt) - status = "PASS" if actual == expected else "FAIL" - if actual != expected: - passed = False - print(f" {dt.strftime('%Y-%m-%d')} ({desc}): Expected={expected}, Actual={actual} -> {status}") - print("=" * 60) - - if passed: - print("All holiday logic checks passed successfully!") - sys.exit(0) - else: - print("Self-verification failed!") - sys.exit(1) - - -def main(): - parser = argparse.ArgumentParser(description="Standalone NSE Bhavcopy Downloader Utility") - parser.add_argument( - "-d", "--days", - type=int, - help="Number of lookback days (0 for full download from 2020-01-01)" - ) - parser.add_argument( - "-o", "--output", - type=str, - default="downloads", - help="Output directory for downloading bhavcopies (default: 'downloads')" - ) - parser.add_argument( - "--verify", - action="store_true", - help="Run self-verification tests for 2026 trading holiday logic" - ) - - args = parser.parse_args() - - if args.verify: - self_verify() - return - - days_back = args.days - if days_back is None: - try: - print("Running in interactive mode. Use -h or --help for command-line options.") - days_input = input("Enter number of days to look back (0 for full download): ").strip() - if not days_input: - days_back = 0 - else: - days_back = int(days_input) - except (KeyboardInterrupt, EOFError): - print("\nExiting.") - sys.exit(0) - except ValueError: - print("Invalid input. Using full download mode.") - days_back = 0 - - if days_back < 0: - print("Invalid days input. Setting to full download.") - days_back = 0 - - downloader = NSEDownloader(days_back=days_back, downloads_dir=args.output) - downloader.run() - - -if __name__ == "__main__": - main() diff --git a/nsemaxlive.py b/nsemaxlive.py deleted file mode 100644 index e8f7b955e02a51703277d11a924340fa31bbbbc1..0000000000000000000000000000000000000000 --- a/nsemaxlive.py +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/env python3 -""" -NSE Max Live Data Scraper & Watchlist Generator (nsemaxlive.py) - -Scrapes live market gainers, losers, and volume spurts directly from NSE India -using Selenium in headless Chrome, processes the raw CSV downloads, and generates -top 50 JSON watchlists for NSEDash. - -Features: -- Headless Chrome scraping for Advance, Decline, and Volume spurts -- Robust download wait & error recovery retries -- Processes scraped CSVs to generate JSON watchlists: - - nse_dashboard_maxvol50.json (Max Volume Portfolio) - - nse_dashboard_maxprice50.json (Max Price Gainers Portfolio) - - nse_dashboard_maxfall50.json (Max Price Fall Portfolio) -- Configurable source and target paths for script execution -""" - -import argparse -import json -import logging -import os -import random -import sys -import time -from datetime import datetime -from pathlib import Path -import pandas as pd - -try: - from selenium import webdriver - from selenium.common.exceptions import ( - ElementClickInterceptedException, - NoSuchElementException, - TimeoutException, - WebDriverException, - ) - from selenium.webdriver.chrome.options import Options - from selenium.webdriver.chrome.service import Service as ChromeService - from selenium.webdriver.common.by import By - from selenium.webdriver.support import expected_conditions as EC - from selenium.webdriver.support.ui import WebDriverWait - from webdriver_manager.chrome import ChromeDriverManager -except ImportError: - webdriver = None - class WebDriverException(Exception): pass - class TimeoutException(Exception): pass - class NoSuchElementException(Exception): pass - class ElementClickInterceptedException(Exception): pass - - -# Default path handling: file is placed in /source/python/sammax1/nsedata/ (same folder as nsedash_admin.py) -SCRIPT_DIR = Path(__file__).resolve().parent -DEFAULT_OUTPUT_DIR = SCRIPT_DIR / "processed" -DEFAULT_DOWNLOAD_DIR = SCRIPT_DIR / "tempfiles" - -# URLs and JavaScript functions for NSE data -SCRAPE_ELEMENTS = { - 'Advance': { - 'url': 'https://www.nseindia.com/market-data/advance', - 'js_function': "javascript:downloadCSV('Advance-download')", - 'download_button_xpath': "//*[@id='Advance-download']" - }, - 'Decline': { - 'url': 'https://www.nseindia.com/market-data/decline', - 'js_function': "javascript:downloadCSV('Decline-download')", - 'download_button_xpath': "//*[@id='Decline-download']" - }, - 'Volume': { - 'url': 'https://www.nseindia.com/market-data/volume-gainers-spurts', - 'js_function': "javascript:downloadCSVFile()", - 'download_button_xpath': "//*[@id='dwldcsv']" - } -} - - -def print_splash_screen(): - """Print stylized ASCII splash screen banner.""" - splash = r""" -================================================================================ - NSE MAX LIVE DATA SCRAPER & PROCESSOR -================================================================================ - Automated Selenium Headless Scraper for Live NSE India Market Data - Outputs: Max Volume (Top 50), Max Price Gainers (Top 50), Max Fall (Top 50) -================================================================================ -""" - print(splash) - - -class NseDataScraper: - """ - A robust web scraper for downloading CSV data from NSE India website. - It uses Selenium with a headless Chrome browser, incorporating retry mechanisms - and improved error handling. - """ - def __init__(self, download_path, output_path, min_sleep_time=0, max_sleep_time=3, max_retries=5, headless=True): - """ - Initializes the NseDataScraper with paths and sets up the headless Chrome WebDriver. - - Args: - download_path (str or Path): The directory where files will be temporarily downloaded by Chrome. - output_path (str or Path): The directory where processed (renamed) CSV files will be saved. - min_sleep_time (float): Minimum random sleep time in seconds. - max_sleep_time (float): Maximum random sleep time in seconds. - max_retries (int): Maximum download retry attempts per element. - headless (bool): Run Chrome browser in headless mode. - """ - self.download_path = os.path.abspath(str(download_path)) - self.output_path = os.path.abspath(str(output_path)) - self.driver = None - self.wait = None - self.max_retries = max_retries - self.min_sleep_time = min_sleep_time - self.max_sleep_time = max_sleep_time - self.headless = headless - - self._ensure_directories() - self._initialize_webdriver() - - def _ensure_directories(self): - """Ensures that the download and output directories exist.""" - try: - os.makedirs(self.download_path, exist_ok=True) - os.makedirs(self.output_path, exist_ok=True) - except OSError as e: - print(f"Error creating directories: {e}. Please check permissions.") - raise - - def _initialize_webdriver(self): - """Initializes the Selenium WebDriver with Chrome options.""" - if webdriver is None: - raise RuntimeError( - "Selenium or webdriver-manager is not installed in the Python environment. " - "Please run: pip install selenium webdriver-manager" - ) - try: - chrome_options = Options() - prefs = { - "download.default_directory": self.download_path, - "download.prompt_for_download": False, - "download.directory_upgrade": True, - "plugins.always_open_pdf_externally": True - } - chrome_options.add_experimental_option("prefs", prefs) - if self.headless: - chrome_options.add_argument("--headless=new") - chrome_options.add_argument("--no-sandbox") - chrome_options.add_argument("--disable-dev-shm-usage") - chrome_options.add_argument("--window-size=1920,1080") - chrome_options.add_argument("--start-maximized") - chrome_options.add_argument("--disable-gpu") - chrome_options.add_argument("--log-level=3") - chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") - chrome_options.add_argument("Accept-Language=en-US,en;q=0.9") - chrome_options.add_argument("Accept-Encoding=gzip, deflate, br") - chrome_options.add_argument("Accept=text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") - chrome_options.add_argument("Connection=keep-alive") - chrome_options.add_argument("Upgrade-Insecure-Requests=1") - - # Check for Linux system Chromium binary locations - for bin_path in ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"]: - if os.path.exists(bin_path): - chrome_options.binary_location = bin_path - break - - try: - driver_path = ChromeDriverManager().install() - service = ChromeService(driver_path) - self.driver = webdriver.Chrome(service=service, options=chrome_options) - except Exception: - # Fallback to default system driver if ChromeDriverManager install fails - self.driver = webdriver.Chrome(options=chrome_options) - - self.wait = WebDriverWait(self.driver, 30) - - except WebDriverException as e: - print(f"Error initializing WebDriver: {e}") - raise - except Exception as e: - print(f"An unexpected error occurred during WebDriver initialization: {e}") - raise - - def _wait_for_page_load_and_elements(self, url, download_button_xpath): - """ - Waits for the page to load by checking for the presence and clickability of key elements. - Additionally, introduces a random sleep time after elements are found. - """ - try: - self.wait.until(EC.presence_of_element_located((By.XPATH, "//img[@alt='csv']"))) - self.wait.until(EC.element_to_be_clickable((By.XPATH, download_button_xpath))) - - random_sleep_time = random.uniform(self.min_sleep_time, self.max_sleep_time) if self.max_sleep_time > 0 else 0.0 - if random_sleep_time > 0: - time.sleep(random_sleep_time) - return True, random_sleep_time - except TimeoutException: - print(f" Timeout: Essential elements not found on {url}.") - return False, 0.0 - except NoSuchElementException: - print(f" No such element: Required element not found on {url}.") - return False, 0.0 - except Exception as e: - print(f" An error occurred while waiting for page elements on {url}: {e}") - return False, 0.0 - - def _wait_for_download_completion(self, expected_filename_prefix, timeout=10): - """ - Waits for a file to appear and complete downloading in the download directory. - Returns the full path of the downloaded file or None if timed out. - """ - start_time = time.time() - downloaded_file = None - current_size = -1 - - while time.time() - start_time < timeout: - try: - list_of_files = os.listdir(self.download_path) - csv_files = [f for f in list_of_files if f.endswith('.csv') or f.endswith('.crdownload')] - - if not csv_files: - time.sleep(1) - continue - - latest_file_path = max([os.path.join(self.download_path, f) for f in csv_files], key=os.path.getmtime) - latest_file_name = os.path.basename(latest_file_path) - - if latest_file_name.endswith('.crdownload'): - current_size_new = os.path.getsize(latest_file_path) - if current_size_new == current_size: - downloaded_file = latest_file_path - break - current_size = current_size_new - time.sleep(1) - continue - - if latest_file_name.endswith('.csv'): - downloaded_file = latest_file_path - break - - except OSError as e: - print(f" OS Error while checking download directory: {e}") - time.sleep(1) - except Exception as e: - print(f" An unexpected error occurred during download check: {e}") - time.sleep(1) - - time.sleep(1) - - if downloaded_file: - return downloaded_file - else: - for f in os.listdir(self.download_path): - if f.endswith('.crdownload'): - try: - os.remove(os.path.join(self.download_path, f)) - except OSError as e: - print(f" Error removing incomplete download {f}: {e}") - return None - - def download_csv(self, name, url, js_function, download_button_xpath): - """ - Navigates to a URL, triggers a CSV download, and renames the downloaded file. - Includes retry logic and reports download metrics. - """ - successful_sleep_time = 0.0 - - for attempt in range(1, self.max_retries + 1): - print(f" Attempting to download {name} data... (Attempt {attempt}/{self.max_retries})", end='\r') - try: - if self.driver is None: - self._initialize_webdriver() - - self.driver.get(url) - page_load_success, current_sleep_time = self._wait_for_page_load_and_elements(url, download_button_xpath) - if not page_load_success: - raise Exception("Page elements not ready.") - successful_sleep_time = current_sleep_time - - self.driver.execute_script(js_function) - latest_downloaded_file = self._wait_for_download_completion(name) - - if latest_downloaded_file: - new_filename = os.path.join(self.output_path, f"{name}.csv") - if os.path.exists(new_filename): - os.remove(new_filename) - os.rename(latest_downloaded_file, new_filename) - print(f" Successfully downloaded and saved {name} as: {new_filename} (Sleep Used: {round(successful_sleep_time, 2)}s)") - return True - else: - raise Exception("File download failed or timed out.") - - except (WebDriverException, TimeoutException, NoSuchElementException, - ElementClickInterceptedException, Exception): - self.close() - time.sleep(2) - finally: - for f in os.listdir(self.download_path): - try: - os.remove(os.path.join(self.download_path, f)) - except OSError as e: - print(f" Error cleaning up download directory: {e}") - - print(f" Failed to download {name} data after {self.max_retries} attempts.") - return False - - def close(self): - """Closes the Selenium WebDriver session if it's open.""" - try: - if self.driver: - self.driver.quit() - self.driver = None - except WebDriverException as e: - print(f"Error closing WebDriver: {e}") - except Exception as e: - print(f"An unexpected error occurred while closing the browser: {e}") - - def scrape_all(self): - """ - Orchestrates the scraping of all defined data elements in a random order, - handling retries and reporting success/failure. - """ - results = {name: False for name in SCRAPE_ELEMENTS.keys()} - scrape_order = list(SCRAPE_ELEMENTS.keys()) - random.shuffle(scrape_order) - - print("\n--- Starting NSE Data Scraping Process ---") - print(f"Scraping order: {', '.join(scrape_order)}") - - for name in scrape_order: - element_info = SCRAPE_ELEMENTS[name] - success = self.download_csv( - name, - element_info['url'], - element_info['js_function'], - element_info['download_button_xpath'] - ) - results[name] = success - if success: - self.close() - - print("\n--- NSE Data Scraping Process Complete ---") - print("\nScraping Summary:") - for name, success in results.items(): - status = "SUCCESS" if success else "FAILED" - print(f"- {name}: {status}") - return results - - -def process_csv_to_json( - input_csv_path: str, - output_json_path: str, - portfolio_name: str, - symbol_col: str, - sort_col: str, - sort_ascending: bool = False, - top_n: int = 50, - header_clean_type: str = "strip_quotes_and_spaces" -) -> None: - """ - Processes a CSV file, extracts top N symbols based on sorting criteria, - and saves them to a JSON file in the specified watchlist format. - - Args: - input_csv_path (str): Path to the input CSV file. - output_json_path (str): Path to the output JSON file. - portfolio_name (str): Name of the portfolio for the JSON output. - symbol_col (str): The name of the column containing stock symbols. - sort_col (str): The name of the column to sort by. - sort_ascending (bool): True for ascending sort, False for descending. - top_n (int): Number of top symbols to extract. - header_clean_type (str): Type of header cleaning to perform. - "strip_quotes_and_spaces" for Volume.csv, - "strip_spaces" for Advance/Decline.csv. - """ - try: - df = pd.read_csv(input_csv_path) - - cleaned_columns = [] - for col in df.columns: - if header_clean_type == "strip_quotes_and_spaces": - cleaned_columns.append(col.strip().replace('"', '')) - elif header_clean_type == "strip_spaces": - cleaned_columns.append(col.strip()) - else: - cleaned_columns.append(col) - - df.columns = cleaned_columns - - if sort_col not in df.columns: - print(f"Error: Sort column '{sort_col}' not found in {input_csv_path} after cleaning. Available columns: {df.columns.tolist()}") - return - if symbol_col not in df.columns: - print(f"Error: Symbol column '{symbol_col}' not found in {input_csv_path} after cleaning. Available columns: {df.columns.tolist()}") - return - - df[sort_col] = pd.to_numeric(df[sort_col], errors='coerce') - df.dropna(subset=[sort_col], inplace=True) - - df_sorted = df.sort_values(by=sort_col, ascending=sort_ascending) - top_symbols_df = df_sorted.head(top_n) - - symbols_list = [] - for _, row in top_symbols_df.iterrows(): - symbols_list.append({ - "SYMBOL": str(row[symbol_col]).strip().upper(), - "Quantity": 0.0, - "Price": 0.0 - }) - - last_updated = datetime.now().isoformat() - - json_output = { - "name": portfolio_name, - "symbols": symbols_list, - "last_updated": last_updated - } - - output_dir = os.path.dirname(output_json_path) - if output_dir and not os.path.exists(output_dir): - os.makedirs(output_dir, exist_ok=True) - - with open(output_json_path, 'w') as f: - json.dump(json_output, f, indent=2) - - print(f"Processing completed: {os.path.basename(output_json_path)}") - - except FileNotFoundError: - print(f"Error: Input file not found at {input_csv_path}") - except KeyError as e: - print(f"Error: Missing expected column in {input_csv_path}: {e}") - except Exception as e: - print(f"An unexpected error occurred while processing {input_csv_path}: {e}") - - -def main(): - parser = argparse.ArgumentParser(description="NSE Max Live Data Scraper and Watchlist Processor") - parser.add_argument( - "-o", "--output-dir", - type=str, - default=str(DEFAULT_OUTPUT_DIR), - help=f"Target directory for output CSVs & watchlists (default: {DEFAULT_OUTPUT_DIR})" - ) - parser.add_argument( - "-t", "--temp-dir", - type=str, - default=str(DEFAULT_DOWNLOAD_DIR), - help=f"Temporary download directory for Chrome (default: {DEFAULT_DOWNLOAD_DIR})" - ) - parser.add_argument( - "--min-sleep", - type=float, - default=0.0, - help="Minimum sleep time during element wait (seconds)" - ) - parser.add_argument( - "--max-sleep", - type=float, - default=0.0, - help="Maximum sleep time during element wait (seconds)" - ) - parser.add_argument( - "--retries", - type=int, - default=5, - help="Maximum retry attempts per URL download (default: 5)" - ) - parser.add_argument( - "--no-headless", - action="store_true", - help="Run Chrome browser in visible mode (default: headless)" - ) - - args = parser.parse_args() - - print_splash_screen() - - download_folder = os.path.abspath(args.temp_dir) - output_folder = os.path.abspath(args.output_dir) - - scraper = None - try: - scraper = NseDataScraper( - download_path=download_folder, - output_path=output_folder, - min_sleep_time=args.min_sleep, - max_sleep_time=args.max_sleep, - max_retries=args.retries, - headless=not args.no_headless - ) - scrape_results = scraper.scrape_all() - - output_base_dir = os.path.join(output_folder, "watchlists") - os.makedirs(output_base_dir, exist_ok=True) - - print("\n--- Starting CSV Processing to JSON ---") - - # 1. Process Volume.csv -> nse_dashboard_maxvol50.json - if scrape_results.get('Volume'): - process_csv_to_json( - input_csv_path=os.path.join(output_folder, "Volume.csv"), - output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxvol50.json"), - portfolio_name="Max Volume Portfolio", - symbol_col="SYMBOL", - sort_col="TODAY - VOLUME", - sort_ascending=False, - top_n=50, - header_clean_type="strip_quotes_and_spaces" - ) - else: - print("Skipping Volume.csv processing as it was not successfully scraped.") - - # 2. Process Advance.csv -> nse_dashboard_maxprice50.json - if scrape_results.get('Advance'): - process_csv_to_json( - input_csv_path=os.path.join(output_folder, "Advance.csv"), - output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxprice50.json"), - portfolio_name="Max Price Portfolio", - symbol_col="Symbol", - sort_col="%chng", - sort_ascending=False, - top_n=50, - header_clean_type="strip_spaces" - ) - else: - print("Skipping Advance.csv processing as it was not successfully scraped.") - - # 3. Process Decline.csv -> nse_dashboard_maxfall50.json - if scrape_results.get('Decline'): - process_csv_to_json( - input_csv_path=os.path.join(output_folder, "Decline.csv"), - output_json_path=os.path.join(output_base_dir, "nse_dashboard_maxfall50.json"), - portfolio_name="Max Fall Portfolio", - symbol_col="Symbol", - sort_col="%chng", - sort_ascending=True, - top_n=50, - header_clean_type="strip_spaces" - ) - else: - print("Skipping Decline.csv processing as it was not successfully scraped.") - - print("\n--- CSV Processing to JSON Complete ---") - - except Exception as e: - print(f"\nAn unrecoverable error occurred during the overall process: {e}") - finally: - if scraper: - scraper.close() - print("\n--- Program Exited ---") - - -if __name__ == "__main__": - main() diff --git a/nsemaxmin.py b/nsemaxmin.py deleted file mode 100644 index c2fd6d25aa140d8d9b7f65b5177c4a5a98a3eba1..0000000000000000000000000000000000000000 --- a/nsemaxmin.py +++ /dev/null @@ -1,147 +0,0 @@ -import pandas as pd -import json -import os -from datetime import datetime - -def check_and_convert_csv_to_parquet(csv_path: str, parquet_path: str) -> None: - """If a CSV file is found, convert it to Parquet and delete the CSV.""" - if os.path.exists(csv_path): - try: - df = pd.read_csv(csv_path) - df.to_parquet(parquet_path, index=False) - os.remove(csv_path) - print(f"Auto-converted {csv_path} to {parquet_path} and removed CSV.") - except Exception as e: - print(f"Failed to auto-convert {csv_path} to Parquet: {e}") - -def process_parquet_to_json( - input_parquet_path: str, - output_json_path: str, - portfolio_name: str, - symbol_col: str, - sort_col: str, - sort_ascending: bool = False, - top_n: int = 50, - header_clean_type: str = "strip_quotes_and_spaces" # "strip_quotes_and_spaces" or "strip_spaces" -) -> None: - """ - Processes a Parquet file, extracts top N symbols based on sorting criteria, - and saves them to a JSON file in the specified format. - """ - # Check for CSV format and convert to Parquet if found - csv_path = input_parquet_path.replace(".parquet", ".csv") - check_and_convert_csv_to_parquet(csv_path, input_parquet_path) - - try: - df = pd.read_parquet(input_parquet_path) - - # Clean up column names based on the specified type - cleaned_columns = [] - for col in df.columns: - if header_clean_type == "strip_quotes_and_spaces": - cleaned_columns.append(col.strip().replace('"', '')) - elif header_clean_type == "strip_spaces": - cleaned_columns.append(col.strip()) - else: - cleaned_columns.append(col) # No cleaning - - df.columns = cleaned_columns - - # Ensure sort_col and symbol_col exist after cleaning - if sort_col not in df.columns: - print(f"Error: Sort column '{sort_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}") - return - if symbol_col not in df.columns: - print(f"Error: Symbol column '{symbol_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}") - return - - # Convert sort_col to numeric, handling errors by coercing to NaN - df[sort_col] = pd.to_numeric(df[sort_col], errors='coerce') - # Drop rows where sort_col is NaN (if conversion failed) - df.dropna(subset=[sort_col], inplace=True) - - # Sort the DataFrame - df_sorted = df.sort_values(by=sort_col, ascending=sort_ascending) - - # Select the top N symbols - top_symbols_df = df_sorted.head(top_n) - - # Prepare the list of symbols for JSON output - symbols_list = [] - for _, row in top_symbols_df.iterrows(): - symbols_list.append({ - "SYMBOL": row[symbol_col], - "Quantity": 0.0, - "Price": 0.0 - }) - - # Get current timestamp - last_updated = datetime.now().isoformat() - - # Create the final JSON structure - json_output = { - "name": portfolio_name, - "symbols": symbols_list, - "last_updated": last_updated - } - - # Create output directory if it doesn't exist - output_dir = os.path.dirname(output_json_path) - if output_dir and not os.path.exists(output_dir): - os.makedirs(output_dir) - - # Write to JSON file - with open(output_json_path, 'w') as f: - json.dump(json_output, f, indent=2) - - print(f"Successfully processed {input_parquet_path} and saved to {output_json_path}") - - except FileNotFoundError: - print(f"Error: Input file not found at {input_parquet_path}") - except KeyError as e: - print(f"Error: Missing expected column in {input_parquet_path}: {e}") - except Exception as e: - print(f"An unexpected error occurred while processing {input_parquet_path}: {e}") - -# --- Main execution --- -if __name__ == "__main__": - # Ensure the 'processed/watchlists' directory exists - output_base_dir = "./processed/watchlists" - if not os.path.exists(output_base_dir): - os.makedirs(output_base_dir) - - # 1. Process Volume.parquet - process_parquet_to_json( - input_parquet_path="./processed/Volume.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxvol50.json", - portfolio_name="Max Volume Portfolio", - symbol_col="SYMBOL", - sort_col="TODAY - VOLUME", - sort_ascending=False, # Descending for Max Volume - top_n=50, - header_clean_type="strip_quotes_and_spaces" - ) - - # 2. Process Advance.parquet - process_parquet_to_json( - input_parquet_path="./processed/Advance.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxprice50.json", - portfolio_name="Max Price Portfolio", - symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap - sort_col="%chng", - sort_ascending=False, # Descending for Max Price Change - top_n=50, - header_clean_type="strip_spaces" - ) - - # 3. Process Decline.parquet - process_parquet_to_json( - input_parquet_path="./processed/Decline.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxfall50.json", - portfolio_name="Max Fall Portfolio", - symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap - sort_col="%chng", - sort_ascending=True, # Ascending for Max Fall (most negative %chng) - top_n=50, - header_clean_type="strip_spaces" - ) diff --git a/nsemcupdate.py b/nsemcupdate.py deleted file mode 100644 index 70c0ea1881216d97fc28a272184f7eb3dff43775..0000000000000000000000000000000000000000 --- a/nsemcupdate.py +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env python3 -""" -Moneycontrol Data Updater Utility (nsemcupdate.py) - -This script maps active NSE symbols in the database to their Moneycontrol pages -and scrapes/updates financial metrics (MC_SCORE, MC_MCAP, MC_FV, MC_ATH, MC_ATL) -in the nse_dashboard_mclink.json file. -""" - -import os -import sys -import re -import json -import time -import random -import logging -import argparse -from pathlib import Path -import requests -from bs4 import BeautifulSoup - -# Setup basic logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout) - ] -) - -USER_AGENTS = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0' -] - -def get_headers(): - return { - 'User-Agent': random.choice(USER_AGENTS), - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'Connection': 'keep-alive', - 'Referer': 'https://www.moneycontrol.com/', - 'Upgrade-Insecure-Requests': '1' - } - -def normalize_name(name): - if not name: - return "" - name = name.lower() - # Remove common corporate suffixes - name = re.sub(r'\b(ltd|limited|co|corp|corporation|ind|industries|india|limited\b|&)\b', '', name) - # Remove all non-alphanumeric characters - return re.sub(r'[^a-z0-9]', '', name) - -def parse_float(val_str): - if not val_str: - return 0.0 - try: - cleaned = re.sub(r'[^\d.]', '', val_str.strip()) - return float(cleaned) if cleaned else 0.0 - except Exception: - return 0.0 - -def scrape_directory_pages(cache_path, delay=2.0): - """Scrapes A-Z and others directories and caches the links list.""" - logging.info("Starting Moneycontrol directory index pages scan...") - letters = [chr(c) for c in range(ord('A'), ord('Z') + 1)] + ['others'] - all_links = [] - - for index, letter in enumerate(letters): - url = f'https://www.moneycontrol.com/india/stockpricequote/{letter}' - logging.info(f"[{index + 1}/{len(letters)}] Scanning index page: {url}") - - try: - r = requests.get(url, headers=get_headers(), timeout=15) - r.raise_for_status() - except Exception as e: - logging.error(f"Error fetching directory page {url}: {e}") - time.sleep(delay) - continue - - soup = BeautifulSoup(r.text, 'html.parser') - links = soup.select('a.bl_12') - page_count = 0 - - for l in links: - href = l.get('href') - text = l.text.strip() - if href and 'https://www.moneycontrol.com/india/stockpricequote/' in href: - path_parts = [p for p in href.split('/') if p] - if len(path_parts) >= 2: - mc_code = path_parts[-1].strip() - mc_segment = path_parts[-2].strip() - all_links.append({ - "company_name": text, - "mc_link": href, - "mc_company_code": mc_code, - "mc_url_segment": mc_segment - }) - page_count += 1 - - logging.info(f" -> Extracted {page_count} company links from page '{letter}'") - time.sleep(delay + random.uniform(0.5, 1.5)) - - # Save to cache - try: - with open(cache_path, 'w') as f: - json.dump(all_links, f, indent=4) - logging.info(f"Directory cache saved to: {cache_path}") - except Exception as e: - logging.error(f"Failed to save directory cache: {e}") - - return all_links - -def load_directory_cache(cache_path, delay=2.0): - if cache_path.exists(): - try: - with open(cache_path, 'r') as f: - return json.load(f) - except Exception as e: - logging.warning(f"Failed to load directory cache: {e}. Re-scraping...") - return scrape_directory_pages(cache_path, delay) - -def get_nse_symbol_from_page(soup): - """Robustly extracts the NSE ticker symbol from the HTML page.""" - # Heuristic 1: clearfix list items - for li in soup.find_all('li'): - span = li.find('span') - p = li.find('p') - if span and p and span.text.strip().lower().rstrip(':') == 'nse': - return p.text.strip().upper() - - # Heuristic 2: search for NSE label string - for tag in soup.find_all(string=lambda x: x and x.strip().lower().rstrip(':') == 'nse'): - parent = tag.parent - sibling = parent.find_next_sibling() - if sibling: - return sibling.text.strip().upper() - - # Heuristic 3: JSON script section at the bottom of the page - for script in soup.find_all('script'): - if script.string and 'nseId' in script.string: - match = re.search(r'"nseId"\s*:\s*"([^"]+)"', script.string) - if match: - return match.group(1).strip().upper() - - return None - -def scrape_stock_details(mc_link): - """Scrapes detail metrics and returns verified details or None.""" - try: - r = requests.get(mc_link, headers=get_headers(), timeout=15) - r.raise_for_status() - except Exception as e: - logging.warning(f"Request failed for {mc_link}: {e}") - return None - - soup = BeautifulSoup(r.text, 'html.parser') - - # Extract & verify NSE symbol - nse_symbol = get_nse_symbol_from_page(soup) - - # Scrape metrics - score_div = soup.find(class_='score') or soup.select_one('div[class*="score"]') - mc_score = 0.0 - if score_div: - match = re.search(r'(\d+)', score_div.text) - if match: - mc_score = float(match.group(1)) - - mcap_td = soup.select_one('td.nsemktcap.bsemktcap') - mc_mcap = parse_float(mcap_td.text) if mcap_td else 0.0 - - fv_td = soup.select_one('td.nsefv.bsefv') - mc_fv = parse_float(fv_td.text) if fv_td else 0.0 - - ath_td = soup.select_one('td.nseLTH.bseLTH') or soup.select_one('td.nseH52.bseH52') - mc_ath = parse_float(ath_td.text) if ath_td else 0.0 - - atl_td = soup.select_one('td.nseLTL.bseLTL') or soup.select_one('td.nseL52.bseL52') - mc_atl = parse_float(atl_td.text) if atl_td else 0.0 - - return { - "NSE_SYMBOL": nse_symbol, - "MC_SCORE": mc_score, - "MC_MCAP": mc_mcap, - "MC_FV": mc_fv, - "MC_ATH": mc_ath, - "MC_ATL": mc_atl - } - -def main(): - parser = argparse.ArgumentParser(description="Update Moneycontrol related information for NSE stocks.") - parser.add_argument("--base-dir", help="Base directory containing processed stock data.") - parser.add_argument("--update-all", action="store_true", help="Force update all stocks, even those already successfully matched.") - parser.add_argument("--delay", type=float, default=2.0, help="Mean delay between network requests to prevent rate blocking.") - parser.add_argument("--limit", type=int, help="Limit the number of stocks scraped in this run.") - args = parser.parse_args() - - # Initialize Paths config - # We append workspace path to sys.path so we can import nsedash.config - workspace_dir = Path(__file__).resolve().parent - sys.path.insert(0, str(workspace_dir)) - - try: - from nsedash.config import Paths - except ImportError as e: - logging.error(f"Failed to import config. Ensure you run this script from the workspace root: {e}") - sys.exit(1) - - paths = Paths(base_dir=args.base_dir) - logging.info(f"Initialized base data directory: {paths.base_dir}") - - # Load active symbols from parquet - if not paths.input_file.exists(): - logging.error(f"Active stock processed data file not found: {paths.input_file}") - sys.exit(1) - - import pandas as pd - try: - df = pd.read_parquet(paths.input_file) - active_companies = df[['SYMBOL', 'SECURITY']].drop_duplicates().to_dict('records') - logging.info(f"Loaded {len(active_companies)} unique active stocks from database.") - except Exception as e: - logging.error(f"Failed to load active symbols: {e}") - sys.exit(1) - - # Load existing links - mc_links = {} - if paths.mc_link_file.exists(): - try: - with open(paths.mc_link_file, 'r') as f: - mc_links = json.load(f) - logging.info(f"Loaded {len(mc_links)} existing Moneycontrol links from: {paths.mc_link_file.name}") - except Exception as e: - logging.error(f"Failed to load mc_link file: {e}") - - # Determine which stocks need lookup or update - stocks_to_update = [] - for item in active_companies: - symbol = str(item['SYMBOL']).strip().upper() - security = str(item['SECURITY']).strip() - - # Check if already processed - details = mc_links.get(symbol, {}) - has_link = 'MC_LINK' in details and details['MC_LINK'] - is_completed = details.get('MC_STATUS', False) - - if args.update_all or not has_link or not is_completed: - stocks_to_update.append({ - "symbol": symbol, - "security": security, - "existing_details": details - }) - - if not stocks_to_update: - logging.info("All stocks are already up-to-date. Nothing to do!") - sys.exit(0) - - # Filter out limit - if args.limit: - stocks_to_update = stocks_to_update[:args.limit] - logging.info(f"Limiting scan to first {args.limit} stocks.") - - logging.info(f"Total stocks to update in this session: {len(stocks_to_update)}") - - # Load directory listings (cache or scrape) - cache_path = paths.data_dir / "nse_mc_directory_cache.json" - mc_directory = load_directory_cache(cache_path, args.delay) - logging.info(f"Moneycontrol directory loaded with {len(mc_directory)} links.") - - # Match missing links - matched_count = 0 - scraped_count = 0 - start_time = time.time() - - try: - for idx, stock in enumerate(stocks_to_update): - symbol = stock["symbol"] - security = stock["security"] - existing = stock["existing_details"] - - logging.info(f"[{idx+1}/{len(stocks_to_update)}] Processing: {symbol} - {security}") - - mc_link = existing.get("MC_LINK") - mc_code = existing.get("MC_COMPANY_CODE") - mc_name = existing.get("MC_COMPANY_NAME", security) - - # 1. Resolve Link if missing - if not mc_link: - # Find candidates using heuristics - candidates = [] - norm_sec = normalize_name(security) - norm_sym = normalize_name(symbol) - - for entry in mc_directory: - c_name = entry["company_name"] - c_code = entry["mc_company_code"] - c_seg = entry["mc_url_segment"] - c_link = entry["mc_link"] - - norm_c_name = normalize_name(c_name) - norm_c_seg = normalize_name(c_seg) - - # Heuristics checks - if norm_sym == normalize_name(c_code): - candidates.append((10, c_link, c_code, c_name)) - elif norm_sec == norm_c_name or norm_sec == norm_c_seg: - candidates.append((9, c_link, c_code, c_name)) - elif norm_sec in norm_c_name or norm_c_name in norm_sec: - candidates.append((8, c_link, c_code, c_name)) - elif norm_sec in norm_c_seg or norm_c_seg in norm_sec: - candidates.append((7, c_link, c_code, c_name)) - elif norm_sym in norm_c_name or norm_sym in norm_c_seg: - candidates.append((5, c_link, c_code, c_name)) - - # Sort by priority score descending - candidates.sort(key=lambda x: x[0], reverse=True) - - # Try candidate links - found = False - for priority, c_link, c_code, c_name in candidates: - logging.info(f" Trying candidate URL ({priority} pts): {c_link}") - - # Scraping & Verification - time.sleep(args.delay + random.uniform(0.1, 1.0)) - details = scrape_stock_details(c_link) - scraped_count += 1 - - if details: - extracted_sym = details["NSE_SYMBOL"] - if extracted_sym == symbol: - logging.info(f" --> Verified match! NSE symbol matches: {symbol}") - mc_link = c_link - mc_code = c_code - mc_name = c_name - - # Merge details - mc_links[symbol] = { - "MC_LINK": mc_link, - "MC_COMPANY_CODE": mc_code, - "MC_COMPANY_NAME": mc_name, - "MC_SCORE": details["MC_SCORE"], - "MC_MCAP": details["MC_MCAP"], - "MC_FV": details["MC_FV"], - "MC_ATH": details["MC_ATH"], - "MC_ATL": details["MC_ATL"], - "MC_STATUS": True - } - matched_count += 1 - found = True - break - else: - logging.info(f" --> Rejected. Page NSE symbol was: {extracted_sym}") - - if not found: - logging.warning(f" No verified Moneycontrol URL found for: {symbol}") - # If it had existing details, mark status as False to retry - if symbol in mc_links: - mc_links[symbol]["MC_STATUS"] = False - continue - - else: - # Stock already has a link, just scrape & update the metrics - logging.info(f" Scraping metrics from existing link: {mc_link}") - time.sleep(args.delay + random.uniform(0.1, 1.0)) - details = scrape_stock_details(mc_link) - scraped_count += 1 - - if details: - # Verified check - if details["NSE_SYMBOL"] and details["NSE_SYMBOL"] != symbol: - logging.warning(f" --> Warning: URL NSE symbol mismatch. Expected {symbol}, got {details['NSE_SYMBOL']}") - - mc_links[symbol] = { - "MC_LINK": mc_link, - "MC_COMPANY_CODE": mc_code, - "MC_COMPANY_NAME": mc_name, - "MC_SCORE": details["MC_SCORE"], - "MC_MCAP": details["MC_MCAP"], - "MC_FV": details["MC_FV"], - "MC_ATH": details["MC_ATH"], - "MC_ATL": details["MC_ATL"], - "MC_STATUS": True - } - matched_count += 1 - else: - logging.warning(f" Failed to update metrics for: {symbol}") - mc_links[symbol]["MC_STATUS"] = False - - # Periodic save (every 10 updates) - if matched_count % 10 == 0: - with open(paths.mc_link_file, 'w') as f: - json.dump(mc_links, f, indent=4, sort_keys=True) - logging.info("Periodic progress saved.") - - except KeyboardInterrupt: - logging.info("\n--- PAUSED BY USER ---") - logging.info("Saving progress before exit...") - - finally: - # Final save - with open(paths.mc_link_file, 'w') as f: - json.dump(mc_links, f, indent=4, sort_keys=True) - - elapsed = time.time() - start_time - logging.info(f"\nExecution finished.") - logging.info(f" - Elapsed Time: {elapsed:.2f} seconds") - logging.info(f" - Requests Executed: {scraped_count}") - logging.info(f" - Stocks Successfully Updated: {matched_count}") - logging.info(f" - Total entries now in json: {len(mc_links)}") - -if __name__ == "__main__": - main() diff --git a/nseprocessor.py b/nseprocessor.py deleted file mode 100644 index 9cd8fd9ce1104b601fdf5d8854c70f06a2c8a34b..0000000000000000000000000000000000000000 --- a/nseprocessor.py +++ /dev/null @@ -1,1258 +0,0 @@ -""" -NSE Bhavcopy Data Processor - -This script processes NSE bhavcopy files with proper DataFrame handling and enhanced logging. -Supports both fresh processing and incremental updates. - -Features: -- Date-ordered processing -- Proper DataFrame operations -- Enhanced logging -- Progress tracking -- Data validation -- Optional corporate action application for full processing runs - -Author: Your Name -Date: June 2025 (Updated) -""" - -import pandas as pd -import numpy as np -import zipfile -import re -import logging -from pathlib import Path -from datetime import datetime, timedelta -import time -from pathlib import Path -import logging -from tqdm import tqdm -import sys -import gc -import json -try: - import talib -except ImportError: - talib = None -try: - import pandas_ta_classic as ta -except ImportError: - try: - import pandas_ta as ta - except ImportError: - ta = None - - -import fastparquet -import warnings -from sklearn.preprocessing import MinMaxScaler -from sklearn.feature_selection import mutual_info_regression - -warnings.filterwarnings('ignore') -warnings.simplefilter('ignore', FutureWarning) - -def _calculate_indicators_for_group(group_data_tuple): - symbol, group, col_mapping = group_data_tuple - df = group.copy() - - # Rename columns to match pandas_ta expectations - df = df.rename(columns=col_mapping) - - # ===== MOMENTUM INDICATORS (14) ===== - df.ta.rsi(length=14, append=True) - df.ta.macd(fast=12, slow=26, signal=9, append=True) - stoch = df.ta.stoch(length=14, k=3, d=3, append=True) - df.ta.willr(length=14, append=True) - df.ta.cmo(length=14, append=True) - df.ta.cci(length=20, append=True) - df.ta.roc(length=12, append=True) - df.ta.mfi(length=14, append=True) - df.ta.fisher(length=9, append=True) - df.ta.kst(append=True) - df.ta.mom(length=10, append=True) - df.ta.tsi(long=25, short=13, append=True) - df.ta.uo(fast=7, medium=14, slow=28, append=True) - df.ta.ao(append=True) - - # ===== TREND INDICATORS (18) ===== - df.ta.sma(length=20, append=True) - df.ta.sma(length=50, append=True) - df.ta.sma(length=200, append=True) - df.ta.ema(length=12, append=True) - df.ta.ema(length=26, append=True) - df.ta.ema(length=50, append=True) - df.ta.ema(length=200, append=True) - df.ta.adx(length=14, append=True) - df.ta.aroon(length=14, append=True) - df.ta.supertrend(length=10, multiplier=3, append=True) - df.ta.psar(af0=0.02, af=0.2, append=True) - df.ta.alma(length=9, append=True) - df.ta.dema(length=10, append=True) - df.ta.tema(length=10, append=True) - df.ta.hma(length=9, append=True) - df.ta.kama(length=10, append=True) - df.ta.linreg(length=25, append=True) - - # ===== VOLATILITY INDICATORS (14) ===== - df['True_Range'] = np.maximum( - np.maximum( - df['High'] - df['Low'], - np.abs(df['High'] - df['Close'].shift()) - ), - np.abs(df['Low'] - df['Close'].shift()) - ) - - bbands = df.ta.bbands(length=20, std=2, append=True) - df.ta.atr(length=14, append=True) - df.ta.natr(length=14, append=True) - df.ta.kc(length=20, scalar=2, append=True) - df.ta.stdev(length=20, append=True) - - df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252) - df['Donchian_High'] = df['High'].rolling(20).max() - df['Donchian_Low'] = df['Low'].rolling(20).min() - df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2 - - hl_range = df['High'] - df['Low'] - mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum() - df['Mass_Index'] = mass_index - - df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - - atr_sum = df['True_Range'].rolling(14).sum() - high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9)) - df['CHOP'] = 100 * high_low / np.log10(14) - - # ===== VOLUME INDICATORS (13) ===== - df.ta.obv(append=True) - df.ta.ad(append=True) - df.ta.adosc(fast=3, slow=10, append=True) - df.ta.cmf(length=20, append=True) - - df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum() - df['Volume_SMA_20'] = df['Volume'].rolling(20).mean() - - klinger_fast = df['Volume'].ewm(span=34).mean() - klinger_slow = df['Volume'].ewm(span=55).mean() - df['KO'] = klinger_fast - klinger_slow - - distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2 - box_height = df['High'] - df['Low'] - eom = distance_moved / (box_height / df['Volume']) - df['EOM'] = eom.rolling(14).mean() - - df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume'] - df['EFI'] = df['EFI'].ewm(span=13).mean() - - tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2) - vm_plus = np.abs(df['High'] - df['Low'].shift(1)) - vm_minus = np.abs(df['Low'] - df['High'].shift(1)) - df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum() - df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum() - - # ===== OSCILLATORS & ADDITIONAL ===== - df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['Median_Price'] = (df['High'] + df['Low']) / 2 - df['HL2'] = (df['High'] + df['Low']) / 2 - df['OC2'] = (df['Open'] + df['Close']) / 2 - df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4 - - df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['R1'] = 2 * df['Pivot'] - df['Low'] - df['S1'] = 2 * df['Pivot'] - df['High'] - df['R2'] = df['Pivot'] + (df['High'] - df['Low']) - df['S2'] = df['Pivot'] - (df['High'] - df['Low']) - - weights = np.arange(1, 21) - df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights)) - df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean() - df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum() - - df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20'] - - df['Price_vs_SMA20'] = df['Close'] - df['SMA_20'] - df['Price_vs_EMA12'] = df['Close'] - df['EMA_12'] - df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26'] - df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200'] - - df['RSI_Divergence'] = df['RSI_14'].diff() - df['Volume_Divergence'] = df['Volume'].diff() - - df['Momentum_Strength'] = df['RSI_14'].rolling(5).std() - df['Trend_Strength'] = df['ADX_14'] - - df['Close_Change'] = df['Close'].pct_change() - df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close'] - df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open'] - df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean() - - # Revert column names to uppercase - inv_col_mapping = {v: k for k, v in col_mapping.items()} - df = df.rename(columns=inv_col_mapping) - - # Add sentiment & news features (4 features) - df['Sentiment'] = 0.0 - df['News_Count'] = 0.0 - df['News_Sentiment'] = 0.0 - df['News_Impact'] = 0.0 - - # Add hybrid strategy indicators (4 features) - df['VIX'] = 15.0 - df['Vol_20MA'] = df['NET_TRDQTY'].rolling(window=20).mean().bfill().fillna(0.0) - df['Prev_High'] = df['HIGH_PRICE'].shift(1).bfill().fillna(0.0) - df['Breakout_Signal'] = ((df['CLOSE_PRICE'] > df['Prev_High']) & (df['NET_TRDQTY'] > df['Vol_20MA'])).astype(float) - - return df - -class NSEDataProcessor: - """Process NSE bhavcopy data with optimized display and relative paths""" - - def __init__(self): - self.setup_paths() - self.setup_logging() - self.initialize_metrics() - - # Define schema (existing schema code) - self.schema = { - 'SYMBOL': 'category', - 'SECURITY': 'category', - 'SERIES': 'category', - 'TRAN_DATE': 'datetime64[ns]', - 'OPEN_PRICE': 'float32', - 'HIGH_PRICE': 'float32', - 'LOW_PRICE': 'float32', - 'CLOSE_PRICE': 'float32', - 'PREV_CL_PR': 'float32', - 'NET_TRDVAL': 'float64', - 'NET_TRDQTY': 'int64', - 'TRADES': 'int64', - 'HI_52_WK': 'float32', - 'LO_52_WK': 'float32', - 'HI_DIFF': 'float32', - 'LO_DIFF': 'float32', - 'DAILY_RETURN': 'float32', - 'VOLATILITY': 'float32' - # Additional feature engineering columns, added runtime and saved to extract file - } - - self.string_cols = ['MKT', 'SERIES', 'SYMBOL', 'SECURITY'] - self.numeric_cols = [ - 'PREV_CL_PR', 'OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', - 'CLOSE_PRICE', 'NET_TRDVAL', 'NET_TRDQTY', 'TRADES', - 'HI_52_WK', 'LO_52_WK' - ] - self.corporate_actions = [] # Initialize corporate actions list - - def setup_paths(self): - """Setup using relative paths""" - try: - self.base_dir = Path.cwd() - self.downloads_dir = self.base_dir / 'downloads' - self.processed_dir = self.base_dir / 'processed' - self.corpact_file = self.processed_dir / 'nse_dashboard_corpact.json' - self.extract_file = self.processed_dir / 'nse_extract_data.parquet' - self.parquet_path = self.processed_dir / 'nse_processed_data.parquet' - - if not self.downloads_dir.exists(): - raise FileNotFoundError(f"Downloads directory not found: {self.downloads_dir}") - - self.processed_dir.mkdir(parents=True, exist_ok=True) - - except Exception as e: - print(f"Path setup failed: {str(e)}") - sys.exit(1) - - def setup_logging(self): - """Configure minimal logging""" - try: - log_format = '%(asctime)s - %(levelname)s - %(message)s' - logging.basicConfig( - filename=self.downloads_dir / 'bulk_data_processing.log', - level=logging.INFO, - format=log_format - ) - self.logger = logging.getLogger(__name__) - - except Exception as e: - print(f"Logging setup failed: {str(e)}") - sys.exit(1) - - def initialize_metrics(self): - """Initialize processing metrics""" - self.metrics = { - 'start_time': None, - 'files_found': 0, - 'files_processed': 0, - 'files_failed': 0, - 'rows_processed': 0, - 'processing_errors': [], - 'processed_files': [], - 'failed_files': [], - 'start_datetime': None, - 'end_datetime': None, - 'duration': None, - 'batch_metrics': [], - 'corporate_actions_applied_count': 0 # New metric for corporate actions - } - - def get_latest_processed_date(self): - """Get latest processed date from existing output""" - try: - if self.parquet_path.exists(): - df = pd.read_parquet(self.parquet_path, columns=['TRAN_DATE']) - latest_date = pd.to_datetime(df['TRAN_DATE']).max() - self.logger.info(f"Latest processed date: {latest_date.strftime('%Y-%m-%d')}") - print(f"Found existing data - processing files newer than {latest_date.strftime('%Y-%m-%d')}") - return latest_date - else: - self.logger.info("No existing data found - performing fresh processing") - print("Starting fresh processing - no existing data found") - return None - except Exception as e: - self.logger.error(f"Failed to get latest date: {str(e)}") - return None - - def extract_date_from_filename(self, filename): - """Extract and validate date from filename (supports PdDDMMYY, PRDDMMYY, pdDDMMYYYY)""" - try: - name = Path(filename).stem - - # Match old format: Pd or PR + ddmmyy - match_old = re.search(r'(?i)(Pd|PR)(\d{2})(\d{2})(\d{2})', name) - - # Match new format: pd + ddmmyyyy - match_new = re.search(r'(?i)pd(\d{2})(\d{2})(\d{4})', name) - - if match_new: - day, month, year = match_new.groups() - elif match_old: - day, month, year = match_old.groups()[1:] # skip prefix - year = str(2000 + int(year)) # convert yy to yyyy - else: - return None - - date_obj = datetime(int(year), int(month), int(day)) - - if date_obj.year < 2000 or date_obj > datetime.now(): - raise ValueError(f"Date out of valid range: {date_obj}") - - return date_obj.strftime('%Y-%m-%d') - - except Exception as e: - self.logger.error(f"Date extraction failed for {filename}: {str(e)}") - return None - - def process_zip_file(self, zip_path): - """Process ZIP file with error handling""" - try: - with zipfile.ZipFile(zip_path) as zf: - csv_files = [f for f in zf.namelist() - if re.match(r'^pd\d{6}(\d{2})?\.csv$', Path(f).name, re.IGNORECASE)] - - if not csv_files: - self.logger.warning(f"No matching CSV in {zip_path}") - return None - - csv_file = csv_files[0] - tran_date = self.extract_date_from_filename(Path(csv_file).name) - if not tran_date: - return None - - with zf.open(csv_file) as f: - df = pd.read_csv(f, thousands=',') - return self.process_dataframe(df, tran_date) - - except Exception as e: - self.logger.error(f"ZIP processing failed - {zip_path}: {str(e)}") - self.metrics['files_failed'] += 1 - return None - - def process_dataframe(self, df, tran_date): - """Process DataFrame with proper indexing""" - try: - # Create explicit copy - df = df.copy() - - # Clean string columns - for col in self.string_cols: - if col in df.columns: - df.loc[:, col] = df[col].astype(str).str.strip() - - # Filter rows and create new copy - mask = ( - (df['SERIES'].str.upper() == 'EQ') & - df['MKT'].notna() - ) - mask_index = ( - df['MKT'].str.upper() == 'Y' - ) - df_index = df[mask_index].copy() - df = df[mask].copy() - - # Create SYMBOL column for df_index - if not df_index.empty: - df_index.loc[:, 'SYMBOL'] = df_index['SECURITY'].str.upper().str.replace(r'\s+','', regex=True) - df_index = df_index[df_index['SYMBOL'].notna()] - df_index = df_index.drop_duplicates(subset=['SYMBOL']) - - # Concat df_index to df - df = pd.concat([df, df_index], ignore_index=True) - - # Add transaction date - df.loc[:, 'TRAN_DATE'] = pd.to_datetime(tran_date) - - # Process numeric columns - for col in self.numeric_cols: - if col in df.columns: - df.loc[:, col] = pd.to_numeric( - df[col].astype(str).str.replace(',', ''), - errors='coerce' - ) - - # Drop rows with missing numerics - df = df.dropna(subset=self.numeric_cols) - - if df.empty: - return None - - # Calculate indicators and validate - df = self.add_technical_indicators(df) - df = self.validate_schema(df) - - if df is not None: - self.metrics['rows_processed'] += len(df) - - return df - - except Exception as e: - self.logger.error(f"DataFrame processing error: {str(e)}") - return None - - def add_technical_indicators(self, df): - """Calculate technical indicators with proper dtype handling""" - try: - df = df.copy() - - # Initialize indicator columns with np.nan - indicators = ['HI_DIFF', 'LO_DIFF', 'DAILY_RETURN', 'VOLATILITY'] - for col in indicators: - df[col] = np.nan - - # Helper function for safe calculation - def calculate_pct_change(a, b, divisor): - """Calculate percentage change with proper dtype handling""" - result = ( - ((a.astype('float32') - b.astype('float32')) / - divisor.astype('float32') * 100) - .round(2) - .astype('float32') - ) - return result - - # Calculate HI_DIFF - mask = df['HI_52_WK'] > 0 - if mask.any(): - df.loc[mask, 'HI_DIFF'] = calculate_pct_change( - df.loc[mask, 'HI_52_WK'], - df.loc[mask, 'CLOSE_PRICE'], - df.loc[mask, 'HI_52_WK'] - ) - - # Calculate LO_DIFF - mask = df['LO_52_WK'] > 0 - if mask.any(): - df.loc[mask, 'LO_DIFF'] = calculate_pct_change( - df.loc[mask, 'CLOSE_PRICE'], - df.loc[mask, 'LO_52_WK'], - df.loc[mask, 'LO_52_WK'] - ) - - # Calculate DAILY_RETURN - mask = df['PREV_CL_PR'] > 0 - if mask.any(): - df.loc[mask, 'DAILY_RETURN'] = calculate_pct_change( - df.loc[mask, 'CLOSE_PRICE'], - df.loc[mask, 'PREV_CL_PR'], - df.loc[mask, 'PREV_CL_PR'] - ) - - # Calculate VOLATILITY - mask = df['OPEN_PRICE'] > 0 - if mask.any(): - high_low_diff = (df.loc[mask, 'HIGH_PRICE'] - df.loc[mask, 'LOW_PRICE']) - df.loc[mask, 'VOLATILITY'] = calculate_pct_change( - high_low_diff, - pd.Series(0, index=high_low_diff.index), - df.loc[mask, 'OPEN_PRICE'] - ) - - # Drop rows where any indicator is missing - return df.dropna(subset=indicators) - - except Exception as e: - self.logger.error(f"Error calculating indicators: {str(e)}") - return df - - def _create_forward_target(self, group, lookahead_days=1): - """ - Creates a forward-looking regression target: the next closing price after `lookahead_days`. - Optionally allows for volatility computation or future annotation. - Returns a DataFrame with 'TARGET_PRICES' column containing single future prices. - """ - # Ensure chronological order - group = group.sort_values(by='TRAN_DATE').copy() - - # Extract future single-step targets - close_prices = group['CLOSE_PRICE'].values - future_targets = [] - - for i in range(len(close_prices)): - if i + lookahead_days < len(close_prices): - future_targets.append(close_prices[i + lookahead_days]) - else: - future_targets.append(np.nan) - - group['TARGET_PRICES'] = future_targets - - # Drop rows where TARGET_PRICES is NaN - group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True) - - return group - - def _create_simple_target(self, group, lookahead_days=1, method="single"): - """ - Generates regression targets for each row based on future closing prices. - - Parameters: - - group: DataFrame containing at least 'TRAN_DATE' and 'CLOSE_PRICE' - - lookahead_days: Number of days to look ahead for target generation - - method: 'single' for next-day price, 'mean' for average of next N days - - Returns: - - Modified DataFrame with a new column 'TARGET_PRICES' - """ - # Ensure chronological order - group = group.sort_values(by='TRAN_DATE').copy() - - # Extract close prices - close_prices = group['CLOSE_PRICE'].values - - # Generate targets using vectorized slicing - if method == "single": - target = np.roll(close_prices, -lookahead_days) - elif method == "mean": - target = np.array([ - np.mean(close_prices[i+1:i+1+lookahead_days]) - if i + lookahead_days < len(close_prices) else np.nan - for i in range(len(close_prices)) - ]) - else: - raise ValueError(f"Unsupported method: {method}") - - # Assign and clean - group['TARGET_PRICES'] = target - group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True) - - return group - - - def calculate_mlbench_indicators(self, df): - df = df.copy() - - # ===== MOMENTUM INDICATORS (14) ===== - df.ta.rsi(length=14, append=True) - df.ta.macd(fast=12, slow=26, signal=9, append=True) - stoch = df.ta.stoch(length=14, k=3, d=3, append=True) - df.ta.willr(length=14, append=True) - df.ta.cmo(length=14, append=True) - df.ta.cci(length=20, append=True) - df.ta.roc(length=12, append=True) - df.ta.mfi(length=14, append=True) - df.ta.fisher(length=9, append=True) - df.ta.kst(append=True) - df.ta.mom(length=10, append=True) - df.ta.tsi(long=25, short=13, append=True) - df.ta.uo(fast=7, medium=14, slow=28, append=True) - df.ta.ao(append=True) - - # ===== TREND INDICATORS (18) ===== - df.ta.sma(length=20, append=True) - df.ta.sma(length=50, append=True) - df.ta.sma(length=200, append=True) - df.ta.ema(length=12, append=True) - df.ta.ema(length=26, append=True) - df.ta.ema(length=50, append=True) - df.ta.ema(length=200, append=True) - df.ta.adx(length=14, append=True) - df.ta.aroon(length=14, append=True) - df.ta.supertrend(length=10, multiplier=3, append=True) - df.ta.psar(af0=0.02, af=0.2, append=True) - df.ta.alma(length=9, append=True) - df.ta.dema(length=10, append=True) - df.ta.tema(length=10, append=True) - df.ta.hma(length=9, append=True) - df.ta.kama(length=10, append=True) - df.ta.linreg(length=25, append=True) - - # ===== VOLATILITY INDICATORS (14) ===== - df['True_Range'] = np.maximum( - np.maximum( - df['High'] - df['Low'], - np.abs(df['High'] - df['Close'].shift()) - ), - np.abs(df['Low'] - df['Close'].shift()) - ) - - bbands = df.ta.bbands(length=20, std=2, append=True) - df.ta.atr(length=14, append=True) - df.ta.natr(length=14, append=True) - df.ta.kc(length=20, scalar=2, append=True) - df.ta.stdev(length=20, append=True) - - df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252) - df['Donchian_High'] = df['High'].rolling(20).max() - df['Donchian_Low'] = df['Low'].rolling(20).min() - df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2 - - hl_range = df['High'] - df['Low'] - mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum() - df['Mass_Index'] = mass_index - - df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - - atr_sum = df['True_Range'].rolling(14).sum() - high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9)) - df['CHOP'] = 100 * high_low / np.log10(14) - - # ===== VOLUME INDICATORS (13) ===== - df.ta.obv(append=True) - df.ta.ad(append=True) - df.ta.adosc(fast=3, slow=10, append=True) - df.ta.cmf(length=20, append=True) - - df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum() - df['Volume_SMA_20'] = df['Volume'].rolling(20).mean() - - klinger_fast = df['Volume'].ewm(span=34).mean() - klinger_slow = df['Volume'].ewm(span=55).mean() - df['KO'] = klinger_fast - klinger_slow - - distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2 - box_height = df['High'] - df['Low'] - eom = distance_moved / (box_height / df['Volume']) - df['EOM'] = eom.rolling(14).mean() - - df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume'] - df['EFI'] = df['EFI'].ewm(span=13).mean() - - tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2) - vm_plus = np.abs(df['High'] - df['Low'].shift(1)) - vm_minus = np.abs(df['Low'] - df['High'].shift(1)) - df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum() - df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum() - - # ===== OSCILLATORS & ADDITIONAL ===== - df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['Median_Price'] = (df['High'] + df['Low']) / 2 - df['HL2'] = (df['High'] + df['Low']) / 2 - df['OC2'] = (df['Open'] + df['Close']) / 2 - df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4 - - df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['R1'] = 2 * df['Pivot'] - df['Low'] - df['S1'] = 2 * df['Pivot'] - df['High'] - df['R2'] = df['Pivot'] + (df['High'] - df['Low']) - df['S2'] = df['Pivot'] - (df['High'] - df['Low']) - - weights = np.arange(1, 21) - df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights)) - df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean() - df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum() - - df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20'] - - df['Price_vs_SMA20'] = df['Close'] - df['SMA_20'] - df['Price_vs_EMA12'] = df['Close'] - df['EMA_12'] - df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26'] - df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200'] - - df['RSI_Divergence'] = df['RSI_14'].diff() - df['Volume_Divergence'] = df['Volume'].diff() - - df['Momentum_Strength'] = df['RSI_14'].rolling(5).std() - df['Trend_Strength'] = df['ADX_14'] - - df['Close_Change'] = df['Close'].pct_change() - df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close'] - df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open'] - df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean() - - return df - - def create_features(self): - logging.info("Calculating technical indicators (105 total) and other features...") - - col_mapping = { - 'OPEN_PRICE': 'Open', - 'HIGH_PRICE': 'High', - 'LOW_PRICE': 'Low', - 'CLOSE_PRICE': 'Close', - 'NET_TRDQTY': 'Volume' - } - - # Ensure correct column types - float_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'NET_TRDQTY', 'PREV_CL_PR'] - for col in float_columns: - if col in self.df.columns: - self.df[col] = self.df[col].astype('float64') - - grouped = self.df.groupby('SYMBOL', observed=False) - tasks = [(symbol, group, col_mapping) for symbol, group in grouped if len(group) >= 200] - - import multiprocessing - # Use available CPU cores minus 2 for efficiency and system responsiveness - num_cores = max(1, multiprocessing.cpu_count() - 2) - logging.info(f"Processing indicators in parallel using {num_cores} cores...") - - processed_groups = [] - with multiprocessing.Pool(processes=num_cores) as pool: - for group_res in tqdm(pool.imap(_calculate_indicators_for_group, tasks), total=len(tasks), desc="Calculating indicators per symbol"): - processed_groups.append(group_res) - - self.df = pd.concat(processed_groups, ignore_index=True) - - # Add NIFTY50 Index features (2 features) - logging.info("Checking for NIFTY50 index data...") - if 'NIFTY50' in self.df['SYMBOL'].values: - logging.info("Calculating NIFTY50 index features...") - nifty_df = self.df[self.df['SYMBOL'] == 'NIFTY50'].copy() - nifty_df = nifty_df[['TRAN_DATE', 'CLOSE_PRICE']].rename(columns={'CLOSE_PRICE': 'NIFTY50_CLOSE'}) - self.df = self.df.merge(nifty_df, on='TRAN_DATE', how='left') - self.df['NIFTY50_RETURN'] = self.df.groupby('SYMBOL', observed=False)['NIFTY50_CLOSE'].pct_change() * 100 - self.df['NIFTY50_RETURN'] = self.df['NIFTY50_RETURN'].fillna(0.0) - else: - self.df['NIFTY50_CLOSE'] = 0.0 - self.df['NIFTY50_RETURN'] = 0.0 - - # Replace infinity and NaNs - self.df.replace([np.inf, -np.inf], np.nan, inplace=True) - # Safe fillna/bfill for numeric columns - numeric_cols = self.df.select_dtypes(include=[np.number]).columns - self.df[numeric_cols] = self.df[numeric_cols].bfill().fillna(0.0) - - def create_target(self): - # Add a column for the symbol name - self.df['SYMBOL'] = self.df['SYMBOL'].astype('category') - # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE - self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True) - self.df.reset_index(drop=True, inplace=True) - - # Calculate and populate RECOMMEND column using the chosen strategy - # Options: 'strategy' (Ultra Profit Maximizer), 'forward' (future price movement), 'simple' (basic up/down/flat) - target_creator_type = 'simple' # Options: 'forward', 'simple' - logging.info(f"Generating Type: {target_creator_type} target price for model training...") - - from tqdm import tqdm - tqdm.pandas(desc="Generating target prices per symbol") - if target_creator_type == 'forward': - self.df = self.df.groupby('SYMBOL').progress_apply( - self._create_forward_target, - lookahead_days=1 - ).reset_index(drop=True) - elif target_creator_type == 'simple': - self.df = self.df.groupby('SYMBOL', observed=False).progress_apply( - self._create_simple_target, - lookahead_days=1 - ).reset_index(drop=True) - - # Verify if TARGET_PRICES column is created - if 'TARGET_PRICES' not in self.df.columns: - logging.error("TARGET_PRICES column not found after target creation...") - - def run_correlation_analysis(self, df): - try: - print("\nRunning correlation analysis.") - numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() - - # Drop rows with NaNs to ensure clean correlation - clean_df = df[numeric_cols].dropna() - - # Compute correlation matrix - corr_matrix = clean_df.corr() - - # Focus on correlation with CLOSE_PRICE - target_corr = corr_matrix['CLOSE_PRICE'].sort_values(ascending=False) - - print("\n📈 Correlation with CLOSE_PRICE:") - print(target_corr.to_string(float_format=lambda x: f"{x:.4f}")) - - # Optionally, show top 5 positively and negatively correlated features - print("\n⬆️ Top 5 positively correlated features:") - print(target_corr.drop('CLOSE_PRICE').head(5).to_string(float_format=lambda x: f"{x:.4f}")) - - print("\n⬇️ Top 5 negatively correlated features:") - print(target_corr.drop('CLOSE_PRICE').tail(5).to_string(float_format=lambda x: f"{x:.4f}")) - - # Mutual Information Analysis - print("\nCalculating Mutual Information Scores.") - X = clean_df.drop(columns=['CLOSE_PRICE'], errors='ignore') - y = clean_df['CLOSE_PRICE'] - scaler = MinMaxScaler() - X_scaled = scaler.fit_transform(X) - - mi_scores = mutual_info_regression(X_scaled, y) - mi_series = pd.Series(mi_scores, index=X.columns).sort_values(ascending=False) - - print("\n📊 Mutual Information Scores:") - print(mi_series.to_string(float_format=lambda x: f"{x:.4f}")) - - except Exception as e: - logging.error(f"Correlation analysis failed: {e}") - - - def save_extract_file(self): - # Final cleanup and save to Parquet - print("Cleaning up data before saving...") - logging.info("Saving processed data to Parquet file...") - self.df.dropna(inplace=True) - print("Sorting data by SYMBOL and TRAN_DATE...") - self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True) - self.df.reset_index(drop=True, inplace=True) - print(f"Saving to {self.extract_file.name} (compression='snappy')...") - self.df.to_parquet(self.extract_file, index=False, compression='snappy') - print("Saving extract file completed successfully.") - logging.info(f"Processed data saved to {self.extract_file}") - - def load_data(self, label_creator_type = 'simple'): - """ - Loads stock data from CSV, performs date filtering, and calculates - various technical indicators and lag features. - """ - logging.info("Loading stock data... This may take a moment.") - try: - # Define data types for efficient memory usage and accurate loading - dtypes = { - 'SYMBOL': 'category', 'TRAN_DATE': str, 'OPEN_PRICE': 'float32', - 'HIGH_PRICE': 'float32', 'LOW_PRICE': 'float32', 'PREV_CL_PR': 'float32', - 'CLOSE_PRICE': 'float32', 'NET_TRDVAL': 'float32', 'NET_TRDQTY': 'float32', - 'TRADES': 'float32', 'HI_52_WK': 'float32', 'LO_52_WK': 'float32', - 'HI_DIFF': 'float32', 'LO_DIFF': 'float32' - } - - # Load data from Parquet - self.df = pd.read_parquet( - self.parquet_path, - columns=list(dtypes.keys()) - ) - - self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce') - self.df.dropna(subset=['TRAN_DATE'], inplace=True) - - self.first_date = self.df['TRAN_DATE'].min() - self.last_date = self.df['TRAN_DATE'].max() - - # Filter data for the last two years - scale_data = False - training_days = 365 * 2 # Years of data to consider for scaling - if scale_data: - self.scaled_date = self.last_date - timedelta(days=training_days) - logging.info(f"Processing transaction dates and filtering data from: {self.scaled_date.strftime('%Y-%m-%d')}...") - mask = self.df['TRAN_DATE'] >= self.scaled_date - latest_data = self.df[mask].copy() - self.df = latest_data - else: - logging.info(f"Processing transaction dates and filtering data from: {self.first_date.strftime('%Y-%m-%d')}...") - - # Sort data by SYMBOL and TRAN_DATE - self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True) - - is_ti = getattr(self, 'ti_only', False) or not self.extract_file.exists() - if is_ti: - # Create feature set from dataframe - logging.info(f"Generating features and labels for model training on complete dataset...") - self.create_features() - print("Generating target prices.") - self.create_target() - # Drop null values from dataframe and drop index - self.df.dropna(inplace=True) - print("Saving extract file.") - self.save_extract_file() - print("Extract data generated and saved successfully.") - else: - logging.info("Skipping technical indicators recalculation (use -ti switch to recalculate).") - - logging.info("Data loading and preprocessing completed successfully...") - # Show correlation analysis results - #self.run_correlation_analysis(self.df) - # Log the final DataFrame statistics - logging.info(f"Data loaded with {len(self.df)} rows and {len(self.df.columns)} columns.") - # Show all the column from dataframe - logging.info(f"Data columns: {', '.join(self.df.columns)}") - logging.info(f"Data contains {self.df['SYMBOL'].nunique()} unique symbols.") - logging.info(f"Data last date: {self.last_date.strftime('%Y-%m-%d')}, Data first date: {self.first_date.strftime('%Y-%m-%d')}") - logging.info(f"Data contains {self.df['TRAN_DATE'].nunique()} unique transaction dates.") - # Show the first and lst few rows of the DataFrame for verification - logging.info(f"First 5 rows of the DataFrame:\n{self.df.head()}") - logging.info(f"Last 5 rows of the DataFrame:\n{self.df.tail()}") - - except FileNotFoundError: - logging.error(f"Error: Input file not found at {self.input_file}. Please ensure 'nse_processed_data.csv' exists.") - sys.exit(1) # Exit if the critical input file is missing - except Exception as e: - logging.error(f"Failed to load and preprocess data: {e}") - sys.exit(1) # Exit on general data processing failure - - def apply_corporate_actions(self, df): - """ - Adjusts historical prices (OPEN_PRICE,HIGH_PRICE,LOW_PRICE,CLOSE_PRICE,PREV_CL_PR,HI_52_WK,LO_52_WK) - in the DataFrame based on corporate actions. Returns the adjusted DataFrame and the count of symbols for which actions were applied. - """ - adjusted_symbols_count = 0 - - # Load corporate actions from JSON file - try: - if not self.corpact_file.exists(): - self.logger.info(f"Corporate actions file not found: {self.corpact_file}. Skipping corporate actions.") - return df, 0 # Return original DataFrame if file not found - - with open(self.corpact_file, 'r') as f: - self.corporate_actions = json.load(f) - - if not self.corporate_actions: - self.logger.info("No corporate actions found in file.") - return df, 0 # Return original DataFrame if no actions are defined - except FileNotFoundError: - self.logger.error(f"Corporate actions file not found: {self.corpact_file}") - return df, 0 - except json.JSONDecodeError: - self.logger.error(f"Error decoding corporate actions JSON from {self.corpact_file}. Check JSON format.") - return df, 0 - except Exception as e: - self.logger.error(f"Failed to load corporate actions: {str(e)}") - return df, 0 - - df_adjusted = df.copy() - df_adjusted['TRAN_DATE'] = pd.to_datetime(df_adjusted['TRAN_DATE']) # Ensure datetime type - - print("Applying corporate actions...", end='', flush=True) - # Sort corporate actions by date in descending order to apply older actions first - # This ensures that if multiple actions for the same symbol happen, the adjustments cascade correctly - self.corporate_actions.sort(key=lambda x: datetime.strptime(x['date'], '%Y-%m-%d'), reverse=True) - - for action in tqdm(self.corporate_actions, desc="Applying corporate actions"): - symbol = action['symbol'] - action_date = datetime.strptime(action['date'], '%Y-%m-%d') - ratio_str = action['ratio'] - - # Corrected part: Check if 'type' key exists and its value is not None - if 'type' not in action or action['type'] is None: - self.logger.warning(f"Corporate action for {symbol} on {action_date.strftime('%Y-%m-%d')} is missing 'type' or 'type' is null. Skipping.") - continue - - action_type = action['type'].lower() # Now it's safe to call .lower() - - # Parse ratio (e.g., "1:2" for split, "1:1" for bonus) - try: - if action_type == "split": - old_shares, new_shares = map(int, ratio_str.split(':')) - adjustment_factor = new_shares / old_shares - elif action_type == "bonus": - bonus_shares, held_shares = map(int, ratio_str.split(':')) - adjustment_factor = (held_shares + bonus_shares) / held_shares - else: - self.logger.warning(f"Unsupported corporate action type: {action_type} for {symbol} on {action_date.strftime('%Y-%m-%d')}. Skipping.") - continue - except ValueError: - self.logger.error(f"Invalid ratio format for {symbol} {action_type}: {ratio_str}. Skipping.") - continue - - # Select data for the specific symbol and dates *before* the corporate action - # Prices before the action date need to be adjusted - mask = (df_adjusted['SYMBOL'] == symbol) & (df_adjusted['TRAN_DATE'] < action_date) - - if df_adjusted[mask].empty: - continue - - # Apply adjustment to relevant price columns - # HI_52_WK and LO_52_WK are adjusted here as well - price_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'PREV_CL_PR', 'HI_52_WK', 'LO_52_WK'] - - for col in price_columns: - if col in df_adjusted.columns: - # Apply adjustment. Rounding to 2 decimal places for prices. - df_adjusted.loc[mask, col] = (df_adjusted.loc[mask, col] / adjustment_factor).round(2).astype('float32') - - adjusted_symbols_count += 1 - self.logger.info(f"Applied {action_type} action for {symbol} on {action_date.strftime('%Y-%m-%d')}. Factor: {adjustment_factor:.2f}") - - print("Done") - return df_adjusted, adjusted_symbols_count - - def validate_schema(self, df): - """Validate and convert DataFrame against schema with safe type conversion""" - try: - validated_df = pd.DataFrame() - - for col, dtype in self.schema.items(): - if col in df.columns: - try: - # Handle integer columns safely - if dtype in ['int32', 'int64']: - # First convert to float to handle NaN values - validated_df[col] = pd.to_numeric(df[col], errors='coerce') - # Then convert to integer, replacing NaN with 0 - validated_df[col] = validated_df[col].fillna(0).astype(dtype) - else: - validated_df[col] = df[col].astype(dtype) - except Exception as e: - self.logger.warning(f"Type conversion failed for {col} to {dtype}: {str(e)}. Keeping original type.") - # If conversion fails, keep original column but ensure it's copied - validated_df[col] = df[col] - - return validated_df - - except Exception as e: - self.logger.error(f"Schema validation error: {str(e)}") - return None - - def save_processed_data(self, df): - """Save processed data with proper sorting and validation""" - try: - if df.empty: - raise ValueError("No data to save") - - # Check if this is fresh processing - is_fresh_save = not self.parquet_path.exists() - - # Ensure proper date format - df['TRAN_DATE'] = pd.to_datetime(df['TRAN_DATE']) - - # If existing file, merge with new data - if not is_fresh_save: - self.logger.info("Reading existing data for merge...") - existing_df = pd.read_parquet(self.parquet_path) - if not pd.api.types.is_datetime64_any_dtype(existing_df['TRAN_DATE']): - existing_df['TRAN_DATE'] = pd.to_datetime(existing_df['TRAN_DATE']) - - # Concatenate and remove duplicates - self.logger.info("Merging with existing data...") - df = pd.concat([existing_df, df], ignore_index=True) - df = df.drop_duplicates(subset=['TRAN_DATE', 'SYMBOL'], keep='last') - - # Convert SYMBOL to category for efficient sorting - df['SYMBOL'] = df['SYMBOL'].astype('category') - - # Sort the final dataset - self.logger.info("Sorting output data...") - df = df.sort_values( - by=['TRAN_DATE', 'SYMBOL'], - ascending=[True, True] - ).reset_index(drop=True) - - print("\nSaving processed data...", end='', flush=True) - with tqdm(total=1, desc="Saving outputs") as pbar: - # Save Parquet - df.to_parquet(self.parquet_path, index=False) - pbar.update(1) - - # Log summary - self.logger.info( - f"{'Created new' if is_fresh_save else 'Updated'} output files\n" - f"Total records: {len(df):,}\n" - f"Date range: {df['TRAN_DATE'].min():%Y-%m-%d} to " - f"{df['TRAN_DATE'].max():%Y-%m-%d}\n" - f"Files:\n{self.parquet_path}" - ) - - # Print summary - print(f"\nData {'created' if is_fresh_save else 'updated'} successfully") - print(f"Records: {len(df):,}") - print(f"Date range: {df['TRAN_DATE'].min():%Y-%m-%d} to " - f"{df['TRAN_DATE'].max():%Y-%m-%d}") - - # Return final DataFrame - return df - - except Exception as e: - self.logger.error(f"Failed to save output: {str(e)}") - raise - return None - - def process_all_files(self): - """ - Process files with simplified progress display, - and optionally apply corporate actions for full processing runs. - """ - self.metrics['start_time'] = time.time() - - # Determine if this is a full processing run - latest_date = self.get_latest_processed_date() - is_full_processing = (latest_date is None) - - # Find and sort ZIP files - zip_files = [] - for zip_path in self.downloads_dir.rglob("*.zip"): - try: - date_str = self.extract_date_from_filename(zip_path.name) - if date_str: - file_date = pd.to_datetime(date_str) - if latest_date is None or file_date > latest_date: - zip_files.append((file_date, zip_path)) - except Exception: - # Log the error but continue if a single file has issues - self.logger.warning(f"Could not extract date or process filename for {zip_path.name}. Skipping.") - continue - - zip_files.sort() - self.new_dates = [d for d, _ in zip_files] - zip_files = [p for _, p in zip_files] - - if not zip_files: - print("No new files to process.") - if is_full_processing: # If no files and it's a full run, maybe apply corporate actions to existing data - print("However, performing corporate action check on existing data (if any).") - try: - # Load existing data if available to apply corporate actions - if self.parquet_path.exists(): - final_df = pd.read_parquet(self.parquet_path) - if not pd.api.types.is_datetime64_any_dtype(final_df['TRAN_DATE']): - final_df['TRAN_DATE'] = pd.to_datetime(final_df['TRAN_DATE']) - final_df, applied_count = self.apply_corporate_actions(final_df) - self.metrics['corporate_actions_applied_count'] = applied_count - if applied_count > 0: - print(f"\nApplied {applied_count} corporate actions to existing data.") - self.save_processed_data(final_df) # Save updated data - else: - print("No new corporate actions applied to existing data.") - else: - print("No existing data found for corporate action application.") - except Exception as e: - self.logger.error(f"Error during corporate action application on existing data: {str(e)}") - print(f"Error applying corporate actions to existing data: {str(e)}") - # Check if extract file exists, if not then try to load data - if not self.extract_file.exists(): - print("No extract file found, generating now.") - try: - self.load_data() - except Exception as e: - self.logger.error(f"Error loading extract data: {str(e)}") - print(f"Error loading extract data: {str(e)}") - elif getattr(self, 'ti_only', False): - print("Recalculating technical indicators for the complete data sets...") - try: - self.load_data() - except Exception as e: - self.logger.error(f"Error recalculating technical indicators: {str(e)}") - print(f"Error recalculating technical indicators: {str(e)}") - return - - print(f"Processing {len(zip_files)} files...", end='', flush=True) - - # Process in batches - batch_size = 10 - processed_batches = [] - - with tqdm(total=len(zip_files), desc="Processing ZIP files", bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt}') as pbar: - for i in range(0, len(zip_files), batch_size): - batch = zip_files[i:i + batch_size] - batch_data = [] - - for zip_file in batch: - try: - df = self.process_zip_file(zip_file) - if df is not None and not df.empty: - batch_data.append(df) - self.metrics['files_processed'] += 1 - except Exception as e: - logging.warning(f"Failed to process {zip_file}: {e}") - finally: - pbar.update(1) - - if batch_data: - batch_df = pd.concat(batch_data, ignore_index=True) - processed_batches.append(batch_df) - gc.collect() - - if processed_batches: - print("\nMerging data...", end='', flush=True) - final_df = pd.concat(processed_batches, ignore_index=True) - # Sorting will happen inside save_processed_data as well if merging with existing. - # However, for consistency before corporate actions, we can sort here too. - final_df = final_df.sort_values(['TRAN_DATE', 'SYMBOL']) - print("Done") - - # Apply corporate actions if it's a full processing run - if is_full_processing: - print("It's a full processing run. Checking for corporate actions...") - final_df, applied_count = self.apply_corporate_actions(final_df) - self.metrics['corporate_actions_applied_count'] = applied_count - if applied_count > 0: - print(f"Successfully applied {applied_count} corporate actions.") - self.logger.info(f"Applied {applied_count} corporate actions during full processing.") - else: - print("No corporate actions found or applied.") - self.logger.info("No corporate actions found or applied during full processing.") - - # Save processed data - df = self.save_processed_data(final_df) - - # Reload data to ensure consistency - self.load_data() - - """ - # Update self.df and create features/target - if df is not None: - self.df = df - self.create_features() - self.create_target() - self.save_extract_file() - else: - print("No data to save after processing.") - self.logger.warning("No data to save after processing.") - """ - - duration = time.time() - self.metrics['start_time'] - self.metrics['duration'] = duration - self.metrics['end_datetime'] = datetime.now() - start_dt = datetime.fromtimestamp(self.metrics['start_time']).strftime('%Y-%m-%d %H:%M:%S') - - print(f"\n--- Processing Summary ---") - print(f"Recalculate Technical Indicators (-ti): {'Yes (Complete Datasets)' if getattr(self, 'ti_only', False) else ('Yes (Extract File Missing)' if not self.extract_file.exists() else 'No (Skipped)')}") - print(f"Started: {start_dt}") - print(f"Ended: {self.metrics['end_datetime'].strftime('%Y-%m-%d %H:%M:%S')}") - print(f"Duration: {self.metrics['duration']:.1f}s") - print(f"Files Found: {len(zip_files)}") # This is the count of files considered for processing - print(f"Files Processed (new/updated): {self.metrics['files_processed']}") - print(f"Files Failed: {self.metrics['files_failed']}") - print(f"Rows Processed: {self.metrics['rows_processed']:,}") - if is_full_processing: - print(f"Corporate Actions Applied: {self.metrics['corporate_actions_applied_count']}") - print(f"--------------------------") - - -def main(): - import argparse - parser = argparse.ArgumentParser(description="NSE Data Processor with Technical Indicators switch") - parser.add_argument("-ti", "--ti", action="store_true", help="Recalculate technical indicators only for the current data file") - args = parser.parse_args() - - try: - processor = NSEDataProcessor() - processor.ti_only = args.ti - processor.process_all_files() - except KeyboardInterrupt: - print("\nProcessing interrupted by user.") - except Exception as e: - print(f"\nAn unhandled error occurred: {str(e)}") - logging.exception("An unhandled error occurred in main execution block.") - sys.exit(1) - - -if __name__ == "__main__": - main() - - diff --git a/nsereco/__init__.py b/nsereco/__init__.py deleted file mode 100644 index 0dffc14aecc19c6c507083bd4973d262a5cc2d4b..0000000000000000000000000000000000000000 --- a/nsereco/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -import sys -import warnings -sys.original_warn = warnings.warn - -from flask import Flask - -# Configure TensorFlow to only use CPU to avoid CUDA compatibility issues with compute capability 12.0 -try: - import tensorflow as tf - tf.config.set_visible_devices([], 'GPU') -except Exception: - pass - -def create_app(): - app = Flask(__name__) - - # Import and register routes - from . import routes - app.register_blueprint(routes.bp) - - return app - diff --git a/nsereco/agent_workflow.py b/nsereco/agent_workflow.py deleted file mode 100644 index 92cc2d6f944e6e70708cdd88883bcb29f6526710..0000000000000000000000000000000000000000 --- a/nsereco/agent_workflow.py +++ /dev/null @@ -1,229 +0,0 @@ -import os -import json -import logging -import asyncio -from typing import Dict, TypedDict, Any -from pathlib import Path -from concurrent.futures import ThreadPoolExecutor - -from langchain_core.language_models.fake import FakeListLLM -from langchain_core.tools import tool -from crewai import Agent, Task, Crew, Process -from langgraph.graph import StateGraph, START, END - -# Import the managers -from nsereco.routes import get_managers - -# Define the state for LangGraph -class WorkflowState(TypedDict): - task_type: str - num_symbols: int - worker_threads: int - results: str - status: str - -# Create a Fake LLM for CrewAI to run offline -fake_llm = FakeListLLM(responses=["I will now execute the python function."]) - -# ========================================== -# CrewAI Agents & Tools Definition -# ========================================== - -@tool -def execute_optuna_tuning_tool(num_symbols: int, max_workers: int) -> str: - """Tool to execute Optuna tuning prioritizing NIFTY stocks.""" - sm, rec = get_managers() - tracker_path = sm.data_dir / 'symbol_master_tracker.json' - - # Load tracker - if not tracker_path.exists(): - return "symbol_master_tracker.json not found." - - with open(tracker_path, 'r') as f: - tracker = json.load(f) - - # Filter pending tuning - pending = [sym for sym, data in tracker.items() if data.get("tuning_status") == "PENDING"] - # Prioritize nifty stocks - nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is True] - non_nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is False] - - target_symbols = (nifty_pending + non_nifty_pending)[:num_symbols] - - if not target_symbols: - return "No pending symbols for tuning." - - logging.info(f"Agent executing Optuna tuning for {len(target_symbols)} symbols: {target_symbols}") - - from nsereco.task_manager import task_manager - task_manager.update_progress(0, len(target_symbols)) - - # Run tuning (this runs sequentially or using its internal threads) - for idx, sym in enumerate(target_symbols): - try: - tracker[sym]["tuning_status"] = "IN_PROGRESS" - with open(tracker_path, 'w') as f: - json.dump(tracker, f, indent=4) - # Recommender function for tuning actually tunes all passed symbols, but we can pass just one at a time, or all. - # wait, run_optuna_tuning takes num_symbols. It will pick from NIFTY stocks. - # To force it to tune our target_symbols, we need to adapt it, but since we are controlling it, we can just use train_symbol or run_optuna_tuning. - # Actually, run_optuna_tuning does randomly pick from NIFTY. Let's just update run_optuna_tuning to accept a list of symbols if we wanted, or we just let it run. - rec.run_optuna_tuning(sm, num_trials=20, num_symbols=1, max_workers=max_workers, tuning_symbols=[sym]) - task_manager.update_progress(idx + 1, len(target_symbols)) - except Exception as e: - logging.error(f"Error tuning {sym}: {e}") - tracker[sym]["tuning_status"] = "FAILED" - with open(tracker_path, 'w') as f: - json.dump(tracker, f, indent=4) - task_manager.update_progress(idx + 1, len(target_symbols)) - - return f"Optuna tuning completed for {target_symbols}" - -@tool -def execute_model_training_tool(num_symbols: int, max_workers: int) -> str: - """Tool to execute Model training using best hyperparameters prioritizing NIFTY stocks.""" - sm, rec = get_managers() - tracker_path = sm.data_dir / 'symbol_master_tracker.json' - - if not tracker_path.exists(): - return "symbol_master_tracker.json not found." - - with open(tracker_path, 'r') as f: - tracker = json.load(f) - - # Filter tuning COMPLETED and training PENDING - pending = [sym for sym, data in tracker.items() - if data.get("tuning_status") == "COMPLETED" and data.get("training_status") == "PENDING"] - - # Prioritize nifty stocks - nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is True] - non_nifty_pending = [sym for sym in pending if tracker[sym].get("nifty_indicator") is False] - - target_symbols = (nifty_pending + non_nifty_pending)[:num_symbols] - - if not target_symbols: - return "No pending symbols for training." - - logging.info(f"Agent executing Model training for {len(target_symbols)} symbols: {target_symbols}") - - from nsereco.task_manager import task_manager - task_manager.update_progress(0, len(target_symbols)) - - for idx, sym in enumerate(target_symbols): - try: - tracker[sym]["training_status"] = "IN_PROGRESS" - with open(tracker_path, 'w') as f: - json.dump(tracker, f, indent=4) - - # Fetch best parameters - best_params = tracker[sym].get("optuna_best_parameters") - if not best_params: - best_params = rec.best_hyperparameters - - # We use the train_symbol or run_model_training_all_stocks logic here - sym_df = sm.df[sm.df['SYMBOL'] == sym].copy() - rec.train_model(sym, sym_df, sm, best_params, "agent_training") - - # Mark as completed - tracker[sym]["training_status"] = "COMPLETED" - with open(tracker_path, 'w') as f: - json.dump(tracker, f, indent=4) - task_manager.update_progress(idx + 1, len(target_symbols)) - - except Exception as e: - logging.error(f"Error training {sym}: {e}") - tracker[sym]["training_status"] = "FAILED" - with open(tracker_path, 'w') as f: - json.dump(tracker, f, indent=4) - task_manager.update_progress(idx + 1, len(target_symbols)) - - return f"Model training completed for {target_symbols}" - -# Define Agents -tuning_agent = Agent( - role='Hyperparameter Tuning Optimizer', - goal='Optimize hyperparameters for stock prediction models.', - backstory='You are an expert AI responsible for tuning stock models to perfection.', - tools=[], - allow_delegation=False, - llm="fake" -) - -training_agent = Agent( - role='Model Trainer', - goal='Train predictive models using the best hyperparameters.', - backstory='You are an expert AI data scientist responsible for training robust models.', - tools=[], - allow_delegation=False, - llm="fake" -) - -# ========================================== -# LangGraph Workflow Definition -# ========================================== - -async def optuna_tuning_node(state: WorkflowState) -> WorkflowState: - num_symbols = state['num_symbols'] - worker_threads = state['worker_threads'] - - # Execute the tool directly - result = execute_optuna_tuning_tool.invoke({"num_symbols": num_symbols, "max_workers": worker_threads}) - - return {"results": result, "status": "COMPLETED"} - -async def model_training_node(state: WorkflowState) -> WorkflowState: - num_symbols = state['num_symbols'] - worker_threads = state['worker_threads'] - - result = execute_model_training_tool.invoke({"num_symbols": num_symbols, "max_workers": worker_threads}) - - return {"results": result, "status": "COMPLETED"} - -def route_task(state: WorkflowState): - if state['task_type'] == 'tune': - return 'optuna_tuning_node' - elif state['task_type'] == 'train': - return 'model_training_node' - return END - -# Build LangGraph -workflow = StateGraph(WorkflowState) -workflow.add_node("optuna_tuning_node", optuna_tuning_node) -workflow.add_node("model_training_node", model_training_node) -workflow.add_conditional_edges(START, route_task) -workflow.add_edge("optuna_tuning_node", END) -workflow.add_edge("model_training_node", END) - -app = workflow.compile() - -# ========================================== -# Orchestrator Function -# ========================================== - -def run_agent_workflow(task_type: str, num_symbols: int, worker_threads: int): - """ - Entry point to trigger the workflow. Runs asynchronously. - """ - import asyncio - - async def _run(): - initial_state = { - "task_type": task_type, - "num_symbols": num_symbols, - "worker_threads": worker_threads, - "results": "", - "status": "STARTED" - } - - logging.info(f"Starting Agent Workflow: Task={task_type}, Symbols={num_symbols}") - final_state = await app.ainvoke(initial_state) - logging.info(f"Workflow Finished. Result: {final_state['results']}") - - try: - loop = asyncio.get_running_loop() - # If we are in an already running loop (like within a web worker thread sometimes), we can create a task - loop.create_task(_run()) - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(_run()) diff --git a/nsereco/data.py b/nsereco/data.py deleted file mode 100644 index f2ea24c046493c53ec30a591ec3a235004698208..0000000000000000000000000000000000000000 --- a/nsereco/data.py +++ /dev/null @@ -1,749 +0,0 @@ -import pandas as pd -import logging -from pathlib import Path -from datetime import datetime, timedelta -import json -import gc -import warnings -import numpy as np -import tensorflow as tf -from tensorflow.keras.models import Sequential, load_model -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint -from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback -import tensorflow.keras.backend as K -import optuna -from optuna.integration import TFKerasPruningCallback -from rich.table import Table -from rich.console import Console -from rich import box -import matplotlib.pyplot as plt -import seaborn as sns -import threading - -matplotlib_lock = threading.Lock() -from sklearn.preprocessing import MinMaxScaler -from sklearn.utils.class_weight import compute_class_weight -from sklearn.model_selection import train_test_split -import time -import sys -import os -import functools -from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError -import threading -from multiprocessing import Pool -from itertools import product -import signal -import talib -import pandas_ta_classic as ta -import fastparquet - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from torch.optim import Adam -from torch.nn import HuberLoss - - -def _calculate_indicators_for_group(group_data_tuple): - symbol, group, col_mapping = group_data_tuple - df = group.copy() - - # Rename columns to match pandas_ta expectations - df = df.rename(columns=col_mapping) - - # ===== MOMENTUM INDICATORS (14) ===== - df.ta.rsi(length=14, append=True) - df.ta.macd(fast=12, slow=26, signal=9, append=True) - stoch = df.ta.stoch(length=14, k=3, d=3, append=True) - df.ta.willr(length=14, append=True) - df.ta.cmo(length=14, append=True) - df.ta.cci(length=20, append=True) - df.ta.roc(length=12, append=True) - df.ta.mfi(length=14, append=True) - df.ta.fisher(length=9, append=True) - df.ta.kst(append=True) - df.ta.mom(length=10, append=True) - df.ta.tsi(long=25, short=13, append=True) - df.ta.uo(fast=7, medium=14, slow=28, append=True) - df.ta.ao(append=True) - - # ===== TREND INDICATORS (18) ===== - df.ta.sma(length=20, append=True) - df.ta.sma(length=50, append=True) - df.ta.sma(length=200, append=True) - df.ta.ema(length=12, append=True) - df.ta.ema(length=26, append=True) - df.ta.ema(length=50, append=True) - df.ta.ema(length=200, append=True) - df.ta.adx(length=14, append=True) - df.ta.aroon(length=14, append=True) - df.ta.supertrend(length=10, multiplier=3, append=True) - df.ta.psar(af0=0.02, af=0.2, append=True) - df.ta.alma(length=9, append=True) - df.ta.dema(length=10, append=True) - df.ta.tema(length=10, append=True) - df.ta.hma(length=9, append=True) - df.ta.kama(length=10, append=True) - df.ta.linreg(length=25, append=True) - - # ===== VOLATILITY INDICATORS (14) ===== - df['True_Range'] = np.maximum( - np.maximum( - df['High'] - df['Low'], - np.abs(df['High'] - df['Close'].shift()) - ), - np.abs(df['Low'] - df['Close'].shift()) - ) - - bbands = df.ta.bbands(length=20, std=2, append=True) - df.ta.atr(length=14, append=True) - df.ta.natr(length=14, append=True) - df.ta.kc(length=20, scalar=2, append=True) - df.ta.stdev(length=20, append=True) - - df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252) - df['Donchian_High'] = df['High'].rolling(20).max() - df['Donchian_Low'] = df['Low'].rolling(20).min() - df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2 - - hl_range = df['High'] - df['Low'] - mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum() - df['Mass_Index'] = mass_index - - df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - - atr_sum = df['True_Range'].rolling(14).sum() - high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9)) - df['CHOP'] = 100 * high_low / np.log10(14) - - # ===== VOLUME INDICATORS (13) ===== - df.ta.obv(append=True) - df.ta.ad(append=True) - df.ta.adosc(fast=3, slow=10, append=True) - df.ta.cmf(length=20, append=True) - - df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum() - df['Volume_SMA_20'] = df['Volume'].rolling(20).mean() - - klinger_fast = df['Volume'].ewm(span=34).mean() - klinger_slow = df['Volume'].ewm(span=55).mean() - df['KO'] = klinger_fast - klinger_slow - - distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2 - box_height = df['High'] - df['Low'] - eom = distance_moved / (box_height / df['Volume']) - df['EOM'] = eom.rolling(14).mean() - - df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume'] - df['EFI'] = df['EFI'].ewm(span=13).mean() - - tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2) - vm_plus = np.abs(df['High'] - df['Low'].shift(1)) - vm_minus = np.abs(df['Low'] - df['High'].shift(1)) - df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum() - df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum() - - # ===== OSCILLATORS & ADDITIONAL ===== - df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['Median_Price'] = (df['High'] + df['Low']) / 2 - df['HL2'] = (df['High'] + df['Low']) / 2 - df['OC2'] = (df['Open'] + df['Close']) / 2 - df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4 - - df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['R1'] = 2 * df['Pivot'] - df['Low'] - df['S1'] = 2 * df['Pivot'] - df['High'] - df['R2'] = df['Pivot'] + (df['High'] - df['Low']) - df['S2'] = df['Pivot'] - (df['High'] - df['Low']) - - weights = np.arange(1, 21) - df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights)) - df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean() - df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum() - - df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20'] - - df['Price_vs_SMA20'] = df['Close'] - df['SMA_20'] - df['Price_vs_EMA12'] = df['Close'] - df['EMA_12'] - df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26'] - df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200'] - - df['RSI_Divergence'] = df['RSI_14'].diff() - df['Volume_Divergence'] = df['Volume'].diff() - - df['Momentum_Strength'] = df['RSI_14'].rolling(5).std() - df['Trend_Strength'] = df['ADX_14'] - - df['Close_Change'] = df['Close'].pct_change() - df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close'] - df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open'] - df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean() - - # Revert column names to uppercase - inv_col_mapping = {v: k for k, v in col_mapping.items()} - df = df.rename(columns=inv_col_mapping) - - # Add sentiment & news features (4 features) - df['Sentiment'] = 0.0 - df['News_Count'] = 0.0 - df['News_Sentiment'] = 0.0 - df['News_Impact'] = 0.0 - - # Add hybrid strategy indicators (4 features) - df['VIX'] = 15.0 - df['Vol_20MA'] = df['NET_TRDQTY'].rolling(window=20).mean().bfill().fillna(0.0) - df['Prev_High'] = df['HIGH_PRICE'].shift(1).bfill().fillna(0.0) - df['Breakout_Signal'] = ((df['CLOSE_PRICE'] > df['Prev_High']) & (df['NET_TRDQTY'] > df['Vol_20MA'])).astype(float) - - return df - -class StockData: - """Manages loading, preprocessing, and feature engineering of stock data.""" - def __init__(self, base_dir=None): - self.base_dir = Path(base_dir or Path.cwd()) - self.setup_paths() - self.setup_logging() - self.df = pd.DataFrame() # Main DataFrame for engineered features - self.profit_df = pd.DataFrame() # DataFrame used for profit maximizer backtesting - self.load_data(label_creator_type = 'simple') # Load and preprocess data on initialization - self.init_symbol_master_tracker() # Initialize unified tracking JSON - - def init_symbol_master_tracker(self): - """Initializes the symbol_master_tracker.json file if it doesn't exist.""" - from .utils import NIFTY_STOCKS, load_json_file, save_json_file - tracker_path = self.data_dir / 'symbol_master_tracker.json' - - existing_data = load_json_file(tracker_path, {}) - all_symbols = self.get_all_symbols_from_df() - - updated = False - for sym in all_symbols: - if sym not in existing_data: - existing_data[sym] = { - "nifty_indicator": sym in NIFTY_STOCKS, - "optuna_best_parameters": None, - "tuning_status": "PENDING", - "training_status": "PENDING" - } - updated = True - - if updated: - save_json_file(tracker_path, existing_data) - logging.info(f"Initialized/Updated symbol_master_tracker.json with {len(existing_data)} symbols.") - self.symbol_master_tracker_path = tracker_path - - def _create_forward_target(self, group, lookahead_days=1): - """ - Creates a forward-looking regression target: the next closing price after `lookahead_days`. - Optionally allows for volatility computation or future annotation. - Returns a DataFrame with 'TARGET_PRICES' column containing single future prices. - """ - # Ensure chronological order - group = group.sort_values(by='TRAN_DATE').copy() - - # Extract future single-step targets - close_prices = group['CLOSE_PRICE'].values - future_targets = [] - - for i in range(len(close_prices)): - if i + lookahead_days < len(close_prices): - future_targets.append(close_prices[i + lookahead_days]) - else: - future_targets.append(np.nan) - - group['TARGET_PRICES'] = future_targets - - # Drop rows where TARGET_PRICES is NaN - group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True) - - return group - - def _create_simple_target(self, group, lookahead_days=1): - """ - Creates a regression target for each row: the next closing price. - Returns a DataFrame with an added column 'TARGET_PRICES' containing a single future price. - """ - # Ensure chronological order - group = group.sort_values(by='TRAN_DATE').copy() - - # Create single-step future target - future_targets = [] - close_prices = group['CLOSE_PRICE'].values - - for i in range(len(close_prices)): - if i + lookahead_days < len(close_prices): - future_targets.append(close_prices[i + lookahead_days]) - else: - future_targets.append(np.nan) - - group['TARGET_PRICES'] = future_targets - - # Drop rows where TARGET_PRICES is NaN - group = group[~pd.isna(group['TARGET_PRICES'])].reset_index(drop=True) - - return group - - def setup_paths(self): - """Sets up directory paths for data, models, and logs.""" - self.data_dir = self.base_dir / 'processed' - self.recommender_dir = self.data_dir / 'recommender' - self.log_dir = self.base_dir / 'downloads' - self.watchlist_dir = self.data_dir / 'watchlists' - self.input_file = self.data_dir / 'nse_processed_data.csv' - self.extract_file = self.data_dir / 'nse_extract_data.parquet' - self.extract_csvfile = self.data_dir / 'nse_extract_data.csv' - self.recommender_dir.mkdir(parents=True, exist_ok=True) - self.data_dir.mkdir(parents=True, exist_ok=True) - self.log_dir.mkdir(parents=True, exist_ok=True) - - def setup_logging(self): - """Configures the logging system for console and file output.""" - log_file_path = self.log_dir / 'nse_data_recommender.log' - # Remove existing handlers to prevent duplicate logs - for handler in logging.root.handlers[:]: - logging.root.removeHandler(handler) - - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(log_file_path), - logging.StreamHandler(sys.stdout) - ] - ) - # Suppress TensorFlow warnings for cleaner logs - tf.get_logger().setLevel('ERROR') - logging.getLogger('tensorflow').setLevel(logging.ERROR) - - def get_all_symbols_from_df(self): - """Returns a list of all unique stock symbols in the loaded DataFrame.""" - if hasattr(self, 'df') and not self.df.empty: - return self.df['SYMBOL'].unique().tolist() - return [] - - @staticmethod - def run_correlation_analysis(df): - try: - logging.info("Running correlation analysis on engineered features...") - numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() - - # Drop rows with NaNs to ensure clean correlation - clean_df = df[numeric_cols].dropna() - - # Compute correlation matrix - corr_matrix = clean_df.corr() - - # Focus on correlation with CLOSE_PRICE - target_corr = corr_matrix['CLOSE_PRICE'].sort_values(ascending=False) - - print("\n📈 Correlation with CLOSE_PRICE:") - print(target_corr.to_string(float_format=lambda x: f"{x:.4f}")) - - # Optionally, show top 5 positively and negatively correlated features - print("\n⬆️ Top 5 positively correlated features:") - print(target_corr.drop('CLOSE_PRICE').head(5).to_string(float_format=lambda x: f"{x:.4f}")) - - print("\n⬇️ Top 5 negatively correlated features:") - print(target_corr.drop('CLOSE_PRICE').tail(5).to_string(float_format=lambda x: f"{x:.4f}")) - - """ - # Mutual Information Analysis - X = clean_df.drop(columns=['CLOSE_PRICE'], errors='ignore') - y = clean_df['CLOSE_PRICE'] - scaler = MinMaxScaler() - X_scaled = scaler.fit_transform(X) - - mi_scores = mutual_info_regression(X_scaled, y) - mi_series = pd.Series(mi_scores, index=X.columns).sort_values(ascending=False) - - print("\n📊 Mutual Information Scores:") - print(mi_series.to_string(float_format=lambda x: f"{x:.4f}")) - """ - - except Exception as e: - logging.error(f"Correlation analysis failed: {e}") - - def calculate_mlbench_indicators(self, df): - df = df.copy() - - # ===== MOMENTUM INDICATORS (14) ===== - df.ta.rsi(length=14, append=True) - df.ta.macd(fast=12, slow=26, signal=9, append=True) - stoch = df.ta.stoch(length=14, k=3, d=3, append=True) - df.ta.willr(length=14, append=True) - df.ta.cmo(length=14, append=True) - df.ta.cci(length=20, append=True) - df.ta.roc(length=12, append=True) - df.ta.mfi(length=14, append=True) - df.ta.fisher(length=9, append=True) - df.ta.kst(append=True) - df.ta.mom(length=10, append=True) - df.ta.tsi(long=25, short=13, append=True) - df.ta.uo(fast=7, medium=14, slow=28, append=True) - df.ta.ao(append=True) - - # ===== TREND INDICATORS (18) ===== - df.ta.sma(length=20, append=True) - df.ta.sma(length=50, append=True) - df.ta.sma(length=200, append=True) - df.ta.ema(length=12, append=True) - df.ta.ema(length=26, append=True) - df.ta.ema(length=50, append=True) - df.ta.ema(length=200, append=True) - df.ta.adx(length=14, append=True) - df.ta.aroon(length=14, append=True) - df.ta.supertrend(length=10, multiplier=3, append=True) - df.ta.psar(af0=0.02, af=0.2, append=True) - df.ta.alma(length=9, append=True) - df.ta.dema(length=10, append=True) - df.ta.tema(length=10, append=True) - df.ta.hma(length=9, append=True) - df.ta.kama(length=10, append=True) - df.ta.linreg(length=25, append=True) - - # ===== VOLATILITY INDICATORS (14) ===== - df['True_Range'] = np.maximum( - np.maximum( - df['High'] - df['Low'], - np.abs(df['High'] - df['Close'].shift()) - ), - np.abs(df['Low'] - df['Close'].shift()) - ) - - bbands = df.ta.bbands(length=20, std=2, append=True) - df.ta.atr(length=14, append=True) - df.ta.natr(length=14, append=True) - df.ta.kc(length=20, scalar=2, append=True) - df.ta.stdev(length=20, append=True) - - df['Historical_Volatility'] = df['Close'].pct_change().rolling(20).std() * np.sqrt(252) - df['Donchian_High'] = df['High'].rolling(20).max() - df['Donchian_Low'] = df['Low'].rolling(20).min() - df['Donchian_Mid'] = (df['Donchian_High'] + df['Donchian_Low']) / 2 - - hl_range = df['High'] - df['Low'] - mass_index = (hl_range.ewm(span=9).mean() / hl_range.ewm(span=9).mean().ewm(span=9).mean()).rolling(25).sum() - df['Mass_Index'] = mass_index - - df['RVI'] = df['Close'].rolling(14).std() / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - - atr_sum = df['True_Range'].rolling(14).sum() - high_low = np.log10(df['High'].rolling(14).max() / (df['Low'].rolling(14).min() + 1e-9)) - df['CHOP'] = 100 * high_low / np.log10(14) - - # ===== VOLUME INDICATORS (13) ===== - df.ta.obv(append=True) - df.ta.ad(append=True) - df.ta.adosc(fast=3, slow=10, append=True) - df.ta.cmf(length=20, append=True) - - df['VPT'] = (df['Close'].pct_change() * df['Volume']).cumsum() - df['Volume_SMA_20'] = df['Volume'].rolling(20).mean() - - klinger_fast = df['Volume'].ewm(span=34).mean() - klinger_slow = df['Volume'].ewm(span=55).mean() - df['KO'] = klinger_fast - klinger_slow - - distance_moved = (df['High'] + df['Low']) / 2 - (df['High'].shift(1) + df['Low'].shift(1)) / 2 - box_height = df['High'] - df['Low'] - eom = distance_moved / (box_height / df['Volume']) - df['EOM'] = eom.rolling(14).mean() - - df['EFI'] = (df['Close'] - df['Close'].shift(1)) * df['Volume'] - df['EFI'] = df['EFI'].ewm(span=13).mean() - - tr = np.sqrt((df['High'] - df['Low'])**2 + (df['High'] - df['Close'].shift(1))**2 + (df['Low'] - df['Close'].shift(1))**2) - vm_plus = np.abs(df['High'] - df['Low'].shift(1)) - vm_minus = np.abs(df['Low'] - df['High'].shift(1)) - df['VI_Plus'] = vm_plus.rolling(14).sum() / tr.rolling(14).sum() - df['VI_Minus'] = vm_minus.rolling(14).sum() / tr.rolling(14).sum() - - # ===== OSCILLATORS & ADDITIONAL ===== - df['Typical_Price'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['Median_Price'] = (df['High'] + df['Low']) / 2 - df['HL2'] = (df['High'] + df['Low']) / 2 - df['OC2'] = (df['Open'] + df['Close']) / 2 - df['HLC3'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['OHLC4'] = (df['Open'] + df['High'] + df['Low'] + df['Close']) / 4 - - df['Pivot'] = (df['High'] + df['Low'] + df['Close']) / 3 - df['R1'] = 2 * df['Pivot'] - df['Low'] - df['S1'] = 2 * df['Pivot'] - df['High'] - df['R2'] = df['Pivot'] + (df['High'] - df['Low']) - df['S2'] = df['Pivot'] - (df['High'] - df['Low']) - - weights = np.arange(1, 21) - df['WMA'] = df['Close'].rolling(20).apply(lambda x: np.sum(x * weights) / np.sum(weights)) - df['RMA'] = df['Close'].ewm(span=20, adjust=False).mean() - df['VWMA'] = (df['Close'] * df['Volume']).rolling(20).sum() / df['Volume'].rolling(20).sum() - - df['Price_Position'] = (df['Close'] - df['Low'].rolling(14).min()) / (df['High'].rolling(14).max() - df['Low'].rolling(14).min() + 1e-9) - df['Oscillator_Ratio'] = (df['Close'] - df['SMA_20']) / df['STDEV_20'] - - df['Price_vs_SMA20'] = df['Close'] - df['SMA_20'] - df['Price_vs_EMA12'] = df['Close'] - df['EMA_12'] - df['EMA12_vs_EMA26'] = df['EMA_12'] - df['EMA_26'] - df['SMA50_vs_SMA200'] = df['SMA_50'] - df['SMA_200'] - - df['RSI_Divergence'] = df['RSI_14'].diff() - df['Volume_Divergence'] = df['Volume'].diff() - - df['Momentum_Strength'] = df['RSI_14'].rolling(5).std() - df['Trend_Strength'] = df['ADX_14'] - - df['Close_Change'] = df['Close'].pct_change() - df['High_Low_Ratio'] = (df['High'] - df['Low']) / df['Close'] - df['Close_Open_Pct'] = (df['Close'] - df['Open']) / df['Open'] - df['Volume_MA_Ratio'] = df['Volume'] / df['Volume'].rolling(20).mean() - - return df - - def create_features(self): - logging.info("Calculating technical indicators (105 total) and other features...") - - col_mapping = { - 'OPEN_PRICE': 'Open', - 'HIGH_PRICE': 'High', - 'LOW_PRICE': 'Low', - 'CLOSE_PRICE': 'Close', - 'NET_TRDQTY': 'Volume' - } - - # Ensure correct column types - float_columns = ['OPEN_PRICE', 'HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'NET_TRDQTY', 'PREV_CL_PR'] - for col in float_columns: - if col in self.df.columns: - self.df[col] = self.df[col].astype('float64') - - grouped = self.df.groupby('SYMBOL', observed=False) - tasks = [(symbol, group, col_mapping) for symbol, group in grouped if len(group) >= 200] - - import multiprocessing - # Use available CPU cores minus 2 for efficiency and system responsiveness - num_cores = max(1, multiprocessing.cpu_count() - 2) - logging.info(f"Processing indicators in parallel using {num_cores} cores...") - - from tqdm import tqdm - processed_groups = [] - with multiprocessing.Pool(processes=num_cores) as pool: - for group_res in tqdm(pool.imap(_calculate_indicators_for_group, tasks), total=len(tasks), desc="Calculating indicators per symbol"): - processed_groups.append(group_res) - - self.df = pd.concat(processed_groups, ignore_index=True) - - # Add NIFTY50 Index features (2 features) - logging.info("Checking for NIFTY50 index data...") - if 'NIFTY50' in self.df['SYMBOL'].values: - logging.info("Calculating NIFTY50 index features...") - nifty_df = self.df[self.df['SYMBOL'] == 'NIFTY50'].copy() - nifty_df = nifty_df[['TRAN_DATE', 'CLOSE_PRICE']].rename(columns={'CLOSE_PRICE': 'NIFTY50_CLOSE'}) - self.df = self.df.merge(nifty_df, on='TRAN_DATE', how='left') - self.df['NIFTY50_RETURN'] = self.df.groupby('SYMBOL', observed=False)['NIFTY50_CLOSE'].pct_change() * 100 - self.df['NIFTY50_RETURN'] = self.df['NIFTY50_RETURN'].fillna(0.0) - else: - self.df['NIFTY50_CLOSE'] = 0.0 - self.df['NIFTY50_RETURN'] = 0.0 - - # Replace infinity and NaNs - self.df.replace([np.inf, -np.inf], np.nan, inplace=True) - # Safe fillna/bfill for numeric columns - numeric_cols = self.df.select_dtypes(include=[np.number]).columns - self.df[numeric_cols] = self.df[numeric_cols].bfill().fillna(0.0) - - def create_target(self): - # Add a column for the symbol name - self.df['SYMBOL'] = self.df['SYMBOL'].astype('category') - # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE - self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True) - self.df.reset_index(drop=True, inplace=True) - - # Calculate and populate RECOMMEND column using the chosen strategy - # Options: 'strategy' (Ultra Profit Maximizer), 'forward' (future price movement), 'simple' (basic up/down/flat) - target_creator_type = 'simple' # Options: 'forward', 'simple' - logging.info(f"Generating Type: {target_creator_type} target price for model training...") - - if target_creator_type == 'forward': - self.df = self.df.groupby('SYMBOL').apply( - self._create_forward_target, - lookahead_days=1 - ).reset_index(drop=True) - elif target_creator_type == 'simple': - self.df = self.df.groupby('SYMBOL').apply( - self._create_simple_target, - lookahead_days=1 - ).reset_index(drop=True) - - # Verify if TARGET_PRICES column is created - if 'TARGET_PRICES' not in self.df.columns: - logging.error("TARGET_PRICES column not found after target creation...") - - def save_extract_file(self): - # Save the processed DataFrame to a CSV for future use - # Ensure the DataFrame is sorted by SYMBOL and TRAN_DATE - # Drop any rows with NaN in any column to ensure clean data - self.df.dropna(inplace=True) - self.df.sort_values(by=['SYMBOL', 'TRAN_DATE'], inplace=True) - self.df.reset_index(drop=True, inplace=True) - self.df.to_parquet(self.extract_file, index=False, compression='snappy') - # self.df.to_csv(self.extract_csvfile, index=False) - logging.info(f"Processed data saved to {self.extract_file}") - - def load_data(self, label_creator_type = 'simple'): - """ - Loads stock data from CSV, performs date filtering, and calculates - various technical indicators and lag features. - """ - logging.info("Loading stock data... This may take a moment.") - try: - # Define data types for efficient memory usage and accurate loading - dtypes = { - 'SYMBOL': 'category', 'TRAN_DATE': str, 'OPEN_PRICE': 'float32', - 'HIGH_PRICE': 'float32', 'LOW_PRICE': 'float32', 'PREV_CL_PR': 'float32', - 'CLOSE_PRICE': 'float32', 'NET_TRDVAL': 'float32', 'NET_TRDQTY': 'float32', - 'TRADES': 'float32', 'HI_52_WK': 'float32', 'LO_52_WK': 'float32', - 'HI_DIFF': 'float32', 'LO_DIFF': 'float32' - } - - """ - # Try to read the CSV file with specified dtypes - try: - self.df = pd.read_csv( - self.input_file, - usecols=list(dtypes.keys()), - dtype=dtypes - ) - except Exception as e: - logging.warning(f"Initial read_csv failed: {e}. Retrying without parse_dates...") - - self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce') - self.df.dropna(subset=['TRAN_DATE'], inplace=True) - - self.first_date = self.df['TRAN_DATE'].min() - self.last_date = self.df['TRAN_DATE'].max() - - # Filter data for the last two years - scale_data = False - training_days = 365 * 2 # Years of data to consider for scaling - if scale_data: - self.scaled_date = self.last_date - timedelta(days=training_days) - logging.info(f"Processing transaction dates and filtering data from: {self.scaled_date.strftime('%Y-%m-%d')}...") - mask = self.df['TRAN_DATE'] >= self.scaled_date - latest_data = self.df[mask].copy() - self.df = latest_data - else: - logging.info(f"Processing transaction dates and filtering data from: {self.first_date.strftime('%Y-%m-%d')}...") - - self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True) - """ - - # If extract file already exists, load it to avoid recomputing - loaded_from_parquet = False - if self.extract_file.exists(): - logging.info(f"Extract file {self.extract_file} already exists. Loading existing data...") - try: - self.df = pd.read_parquet(self.extract_file) - # Verify required columns exist - required_cols = ['HIGH_PRICE', 'LOW_PRICE', 'CLOSE_PRICE', 'OPEN_PRICE', 'NET_TRDQTY'] - missing_cols = [col for col in required_cols if col not in self.df.columns] - if missing_cols: - logging.warning(f"Parquet file missing required columns: {missing_cols}. Falling back to CSV.") - self.df = pd.DataFrame() - else: - loaded_from_parquet = True - except Exception as e: - logging.warning(f"Initial read_parquet failed: {e}. Retrying to load csv...") - - if not loaded_from_parquet: - parquet_src = self.data_dir / 'nse_processed_data.parquet' - if parquet_src.exists(): - logging.info(f"Loading data from Parquet source: {parquet_src}") - try: - self.df = pd.read_parquet(parquet_src) - try: - self.save_extract_file() - except Exception as e: - logging.error(f"Failed to save updated extract file: {e}") - except Exception as e: - logging.warning(f"Initial read_parquet from source failed: {e}. Retrying with CSV...") - - if self.df.empty: - if self.input_file.exists(): - logging.info(f"Loading data from CSV: {self.input_file}") - try: - self.df = pd.read_csv(self.input_file) - try: - self.save_extract_file() - except Exception as e: - logging.error(f"Failed to save updated extract file: {e}") - except Exception as e: - logging.warning(f"Initial read_csv failed: {e}. Exiting...") - sys.exit(1) - else: - logging.error(f"Input file parquet and csv does not exist. Cannot load data exiting...") - sys.exit(1) - - self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE'], errors='coerce') - self.df.dropna(subset=['TRAN_DATE'], inplace=True) - # Sort on SYMBOL and TRAN_DATE and reset index to ensure contiguous arrays for each symbol group (prevents talib core dumps) - self.df = self.df.sort_values(by=['SYMBOL', 'TRAN_DATE']).reset_index(drop=True) - self.first_date = self.df['TRAN_DATE'].min() - self.last_date = self.df['TRAN_DATE'].max() - - logging.info("Data loading and preprocessing completed successfully...") - # Show correlation analysis results - # self.run_correlation_analysis(self.df) - # Log the final DataFrame statistics - logging.info(f"Data loaded with {len(self.df)} rows and {len(self.df.columns)} columns.") - # Show all the column from dataframe - logging.info(f"Data columns: {', '.join(self.df.columns)}") - logging.info(f"Data contains {self.df['SYMBOL'].nunique()} unique symbols.") - logging.info(f"Data last date: {self.last_date.strftime('%Y-%m-%d')}, Data first date: {self.first_date.strftime('%Y-%m-%d')}") - logging.info(f"Data contains {self.df['TRAN_DATE'].nunique()} unique transaction dates.") - # Show the first and lst few rows of the DataFrame for verification - #logging.info(f"First 5 rows of the DataFrame:\n{self.df.head()}") - #logging.info(f"Last 5 rows of the DataFrame:\n{self.df.tail()}") - - gc.collect() # Explicit garbage collection to free up memory - - except FileNotFoundError: - logging.error(f"Error: Input file not found at {self.input_file}. Please ensure 'nse_processed_data.csv' exists.") - sys.exit(1) # Exit if the critical input file is missing - except Exception as e: - logging.error(f"Failed to load and preprocess data: {e}") - sys.exit(1) # Exit on general data processing failure - - def select_features(self, corr_threshold=0.9999): - """ - Selects relevant features by removing highly correlated ones. - Mutual Information (MI) filtering is not explicitly applied here based on the original code, - but the function is retained for clarity of purpose. - """ - logging.info(f"Running feature selection using correlation. Number of features before selection: {len(self.df.columns)}") - - numeric_cols = self.df.select_dtypes(include=[np.number]).columns.tolist() - - clean_df = self.df[numeric_cols].dropna() - - # Drop highly correlated features based on absolute correlation - corr_matrix = clean_df.corr().abs() - # Select upper triangle of correlation matrix to avoid duplicates and self-correlation - upper_tri = corr_matrix.where( - pd.DataFrame(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool), - columns=corr_matrix.columns, index=corr_matrix.index) - ) - # Find features with correlation greater than threshold - to_drop_corr = [col for col in upper_tri.columns if any(upper_tri[col] > corr_threshold)] - reduced_df = clean_df.drop(columns=to_drop_corr) - - selected_features = reduced_df.columns.tolist() - - logging.info(f"Selected {len(selected_features)} features after correlation filtering.") - return selected_features, to_drop_corr - -# --- SymbolRecommender Class (for model training and tuning) --- diff --git a/nsereco/models.py b/nsereco/models.py deleted file mode 100644 index 16325ae1c70ee72c50cfa4f22d487dfcd1a8c47b..0000000000000000000000000000000000000000 --- a/nsereco/models.py +++ /dev/null @@ -1,274 +0,0 @@ -import pandas as pd -import logging -from pathlib import Path -from datetime import datetime, timedelta -import json -import gc -import warnings -import numpy as np -import tensorflow as tf -from tensorflow.keras.models import Sequential, load_model -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint -from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback -import tensorflow.keras.backend as K -import optuna -from optuna.integration import TFKerasPruningCallback -from rich.table import Table -from rich.console import Console -from rich import box -import matplotlib.pyplot as plt -import seaborn as sns -import threading - -matplotlib_lock = threading.Lock() -from sklearn.preprocessing import MinMaxScaler -from sklearn.utils.class_weight import compute_class_weight -from sklearn.model_selection import train_test_split -import time -import sys -import os -import functools -from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError -import threading -from multiprocessing import Pool -from itertools import product -import signal -import talib -import fastparquet - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from torch.optim import Adam -from torch.nn import HuberLoss - - -class FunctionalAttentionModel(nn.Module): - def __init__(self, input_dim, params): - super(FunctionalAttentionModel, self).__init__() - self.conv1d = nn.Conv1d(in_channels=input_dim, out_channels=64, kernel_size=3, padding=2) # causal padding manually - self.bn_c1 = nn.BatchNorm1d(64) - self.dropout_c1 = nn.Dropout(0.2) - - self.bilstm1 = nn.LSTM(input_size=64, hidden_size=params["lstm_units_1"], batch_first=True, - bidirectional=True) - self.bn_lstm1 = nn.BatchNorm1d(2 * params["lstm_units_1"]) - self.dropout_lstm1 = nn.Dropout(params["dropout_1"]) - - self.attention = nn.MultiheadAttention(embed_dim=2 * params["lstm_units_1"], - num_heads=1, - batch_first=True) - - self.bilstm2 = nn.LSTM(input_size=4 * params["lstm_units_1"], hidden_size=params["lstm_units_1"], - batch_first=True, bidirectional=True) - self.bn_lstm2 = nn.BatchNorm1d(2 * params["lstm_units_1"]) - self.dropout_lstm2 = nn.Dropout(params["dropout_2"]) - - self.dense1 = nn.Linear(2 * params["lstm_units_1"], params["dense_units_1"]) - self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"]) - self.dropout_d1 = nn.Dropout(params["dropout_3"]) - - self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"]) - self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"]) - self.dropout_d2 = nn.Dropout(params["dropout_4"]) - - self.output = nn.Linear(params["dense_units_2"], 1) - - def forward(self, x): - # x: (batch, seq_len, input_dim) - x = x.transpose(1, 2) # (batch, input_dim, seq_len) for Conv1d - x = self.conv1d(x)[..., :-2] # causal padding: remove last 2 timesteps - x = self.bn_c1(x) - x = self.dropout_c1(x) - x = x.transpose(1, 2) # back to (batch, seq_len, channels) - - lstm1_out, _ = self.bilstm1(x) - lstm1_out_bn = self.bn_lstm1(lstm1_out.transpose(1, 2)).transpose(1, 2) - lstm1_out_drop = self.dropout_lstm1(lstm1_out_bn) - - attn_output, _ = self.attention(lstm1_out_drop, lstm1_out_drop, lstm1_out_drop) - concat = torch.cat([lstm1_out_drop, attn_output], dim=-1) - - lstm2_out, _ = self.bilstm2(concat) - lstm2_out_bn = self.bn_lstm2(lstm2_out.transpose(1, 2)).transpose(1, 2) - lstm2_out = self.dropout_lstm2(lstm2_out_bn) - - # Take last timestep output - last_output = lstm2_out[:, -1, :] - - x = self.dense1(last_output) - x = self.bn_d1(x) - x = F.relu(x) - x = self.dropout_d1(x) - - x = self.dense2(x) - x = self.bn_d2(x) - x = F.relu(x) - x = self.dropout_d2(x) - - return self.output(x) - - -class LSTMModel(nn.Module): - def __init__(self, input_dim, params): - super(LSTMModel, self).__init__() - self.lstm_units = params["lstm_units_1"] - - # Bidirectional LSTM Layer 1 - self.bilstm1 = nn.LSTM(input_size=input_dim, - hidden_size=self.lstm_units, - batch_first=True, - bidirectional=True) - self.bn_lstm1 = nn.BatchNorm1d(2 * self.lstm_units) - self.dropout_lstm1 = nn.Dropout(params["dropout_1"]) - - # Bidirectional LSTM Layer 2 - self.bilstm2 = nn.LSTM(input_size=2 * self.lstm_units, - hidden_size=self.lstm_units, - batch_first=True, - bidirectional=True) - self.bn_lstm2 = nn.BatchNorm1d(2 * self.lstm_units) - self.dropout_lstm2 = nn.Dropout(params["dropout_2"]) - - # Dense layers - self.dense1 = nn.Linear(2 * self.lstm_units, params["dense_units_1"]) - self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"]) - self.dropout_d1 = nn.Dropout(params["dropout_3"]) - - self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"]) - self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"]) - self.dropout_d2 = nn.Dropout(params["dropout_4"]) - - # Output layer - self.output = nn.Linear(params["dense_units_2"], 1) - - def forward(self, x): - # x: (batch, seq_len, input_dim) - lstm1_out, _ = self.bilstm1(x) - lstm1_out_bn = self.bn_lstm1(lstm1_out.transpose(1, 2)).transpose(1, 2) - lstm1_out_drop = self.dropout_lstm1(lstm1_out_bn) - - lstm2_out, _ = self.bilstm2(lstm1_out_drop) - lstm2_out_bn = self.bn_lstm2(lstm2_out.transpose(1, 2)).transpose(1, 2) - lstm2_out_drop = self.dropout_lstm2(lstm2_out_bn) - - # Take last timestep - last_output = lstm2_out_drop[:, -1, :] - - x = self.dense1(last_output) - x = self.bn_d1(x) - x = F.relu(x) - x = self.dropout_d1(x) - - x = self.dense2(x) - x = self.bn_d2(x) - x = F.relu(x) - x = self.dropout_d2(x) - - return self.output(x) - -class GRUAttentionModel(nn.Module): - def __init__(self, input_dim, params): - super(GRUAttentionModel, self).__init__() - self.conv1d = nn.Conv1d(in_channels=input_dim, out_channels=64, kernel_size=3, padding=2) # causal padding - self.bn_c1 = nn.BatchNorm1d(64) - self.dropout_c1 = nn.Dropout(0.2) - - self.bi_gru1 = nn.GRU(input_size=64, hidden_size=params["lstm_units_1"], - batch_first=True, bidirectional=True) - self.bn_gru1 = nn.BatchNorm1d(2 * params["lstm_units_1"]) - self.dropout_gru1 = nn.Dropout(params["dropout_1"]) - - self.attention = nn.MultiheadAttention(embed_dim=2 * params["lstm_units_1"], - num_heads=1, - batch_first=True) - - self.bi_gru2 = nn.GRU(input_size=4 * params["lstm_units_1"], hidden_size=params["lstm_units_1"], - batch_first=True, bidirectional=True) - self.bn_gru2 = nn.BatchNorm1d(2 * params["lstm_units_1"]) - self.dropout_gru2 = nn.Dropout(params["dropout_2"]) - - self.dense1 = nn.Linear(2 * params["lstm_units_1"], params["dense_units_1"]) - self.bn_d1 = nn.BatchNorm1d(params["dense_units_1"]) - self.dropout_d1 = nn.Dropout(params["dropout_3"]) - - self.dense2 = nn.Linear(params["dense_units_1"], params["dense_units_2"]) - self.bn_d2 = nn.BatchNorm1d(params["dense_units_2"]) - self.dropout_d2 = nn.Dropout(params["dropout_4"]) - - self.output = nn.Linear(params["dense_units_2"], 1) - - def forward(self, x): - # x: (batch, seq_len, input_dim) - x = x.transpose(1, 2) # (batch, input_dim, seq_len) for Conv1d - x = self.conv1d(x)[..., :-2] # causal padding: remove last 2 timesteps - x = self.bn_c1(x) - x = self.dropout_c1(x) - x = x.transpose(1, 2) # back to (batch, seq_len, channels) - - gru1_out, _ = self.bi_gru1(x) - gru1_out_bn = self.bn_gru1(gru1_out.transpose(1, 2)).transpose(1, 2) - gru1_out_drop = self.dropout_gru1(gru1_out_bn) - - attn_output, _ = self.attention(gru1_out_drop, gru1_out_drop, gru1_out_drop) - concat = torch.cat([gru1_out_drop, attn_output], dim=-1) - - gru2_out, _ = self.bi_gru2(concat) - gru2_out_bn = self.bn_gru2(gru2_out.transpose(1, 2)).transpose(1, 2) - gru2_out_drop = self.dropout_gru2(gru2_out_bn) - - last_output = gru2_out_drop[:, -1, :] # final timestep - - x = self.dense1(last_output) - x = self.bn_d1(x) - x = F.relu(x) - x = self.dropout_d1(x) - - x = self.dense2(x) - x = self.bn_d2(x) - x = F.relu(x) - x = self.dropout_d2(x) - - return self.output(x) - - - -class HalveOnImprovement(Callback): - def __init__(self, monitor='val_loss', improve_count=3, reduction_factor=0.75, patience=5, verbose=1, min_lr=1e-6): - super().__init__() - self.monitor = monitor - self.patience = patience - self.verbose = verbose - self.min_lr = min_lr - self.best = float('inf') - self.epoch_counter = 0 - self.wait = 0 # Tracks if any improvement occurred - self.improve_count = improve_count - self.reduction_factor = reduction_factor - - def on_epoch_end(self, epoch, logs=None): - current = logs.get(self.monitor) - if current is None: - return - - self.epoch_counter += 1 - - if current < self.best: - self.best = current - self.wait += 1 # Mark that improvement occurred - - if self.epoch_counter >= self.patience: - if self.wait >= self.improve_count: - old_lr = float(K.get_value(self.model.optimizer.lr)) - new_lr = max(old_lr * self.reduction_factor, self.min_lr) - K.set_value(self.model.optimizer.lr, new_lr) - print(f"\n[HalveOnImprovement] Epoch {epoch}: LR reduced from {old_lr:.6f} to {new_lr:.6f} after {self.epoch_counter} epochs with {self.wait} improvements.") - # Reset counters - self.epoch_counter = 0 - self.wait = 0 - -# --- Main Application Logic (Menu Driven) --- diff --git a/nsereco/recommender.py b/nsereco/recommender.py deleted file mode 100644 index cba52f4db8b5c41c3ffb89a2e723a7667d610aff..0000000000000000000000000000000000000000 --- a/nsereco/recommender.py +++ /dev/null @@ -1,2000 +0,0 @@ -import pandas as pd -import logging -from pathlib import Path -from datetime import datetime, timedelta -import json -import gc -import warnings -from . import utils -from .utils import load_json_file, save_json_file, NIFTY_STOCKS -from .data import StockData -import numpy as np -import tensorflow as tf -try: - tf.config.set_visible_devices([], 'GPU') -except Exception: - pass -from tensorflow.keras.models import Sequential, load_model -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint -from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback -import tensorflow.keras.backend as K -import optuna -from optuna.integration import TFKerasPruningCallback -from rich.table import Table -from rich.console import Console -from rich import box -import matplotlib -matplotlib.use('Agg') -import matplotlib.pyplot as plt -import seaborn as sns -import threading - -matplotlib_lock = threading.Lock() -from sklearn.preprocessing import MinMaxScaler -from sklearn.utils.class_weight import compute_class_weight -from sklearn.model_selection import train_test_split -import time -import sys -import os -import glob -import functools -from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError -import threading -from multiprocessing import Pool -from itertools import product -import signal -import talib -import fastparquet - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from torch.optim import Adam as TorchAdam -from torch.nn import HuberLoss - - -from .models import FunctionalAttentionModel, LSTMModel, GRUAttentionModel, HalveOnImprovement -class PyTorchPredictionLogger: - def __init__(self, model, x_val, target_scaler, actual_price, symbol, trial_number, total_trials, recommender_dir): - self.model = model - self.x_val = x_val - self.target_scaler = target_scaler - self.actual_price = actual_price - self.symbol = symbol - self.trial_number = trial_number - self.total_trials = total_trials - self.recommender_dir = recommender_dir - self.min_error = float('inf') - self.min_error_epoch = -1 - self.best_pred_price = None - self.weights_filepath = None - self.best_key = "NO" - - def update(self, epoch, device): - self.model.eval() - with torch.no_grad(): - y_pred = self.model(self.x_val.to(device)).cpu().numpy() - pred_price = self.target_scaler.inverse_transform(y_pred)[-1][0] - pred_error = ((pred_price - self.actual_price) / self.actual_price * 100) - - if abs(pred_error) < self.min_error: - self.min_error = abs(pred_error) - self.min_error_epoch = epoch + 1 - self.best_pred_price = pred_price - self.best_key = "YES" - if self.total_trials == 1: - self.weights_filepath = self.recommender_dir / f"trial_{self.trial_number}_of_{self.total_trials}_{self.symbol}.pt" - torch.save(self.model.state_dict(), self.weights_filepath) - - if self.total_trials == 1: - print(f" Predicted price: {pred_price:.2f} Error: {pred_error:.4f}% Min Error: {self.min_error:.4f}% at epoch: {self.min_error_epoch}") - -class PyTorchMAELogger: - def __init__(self, model, x_val_tensor, y_val_tensor, target_scaler, symbol, trial_number, total_trials, recommender_dir): - self.model = model - self.x_val_tensor = x_val_tensor # shape: (n_samples, seq_len, features) - self.y_val_tensor = y_val_tensor # shape: (n_samples, 1) - self.target_scaler = target_scaler - self.symbol = symbol - self.trial_number = trial_number - self.total_trials = total_trials - self.recommender_dir = recommender_dir - - self.minmae_error = float('inf') - self.minmae_error_epoch = -1 - self.best_pred_mae = None - self.best_mae_key = "NO" - self.weights_filepath = None - - def update(self, epoch, device): - self.model.eval() - with torch.no_grad(): - y_pred_scaled = self.model(self.x_val_tensor.to(device)).cpu().numpy() - pred_price = self.target_scaler.inverse_transform(y_pred_scaled) - - # Inverse transform actual targets correctly - actual_price = self.target_scaler.inverse_transform(self.y_val_tensor.cpu().numpy()) - - mae = np.mean(np.abs(pred_price - actual_price)) - pred_error = np.abs(pred_price - actual_price) / (actual_price + 1e-8) * 100 - mae_error = np.mean(pred_error) - - if mae_error < self.minmae_error: - self.minmae_error = mae_error - self.minmae_error_epoch = epoch + 1 - self.best_pred_mae = mae_error - self.best_mae_key = "YES" - if self.total_trials == 1: - self.weights_filepath = self.recommender_dir / f"trial_{self.trial_number}_of_{self.total_trials}_{self.symbol}_mae.pt" - torch.save(self.model.state_dict(), self.weights_filepath) - - if self.total_trials == 1: - print(f" Predicted MAE: {mae:.2f} Error: {mae_error:.4f}% Min Error: {self.minmae_error:.4f}% at epoch: {self.minmae_error_epoch}") - - -class SymbolRecommender: - """ - Manages deep learning model training, hyperparameter tuning, and profit maximization dashboards. - Handles loading/saving of training/tuning progress for resume functionality across different operations. - """ - DL_FEATURES = [] # Placeholder, will be populated by StockData.select_features at runtime - dropped_features = [] # Features dropped during correlation/manual analysis, will be populated at runtime - - # Default hyperparameters for the LSTM model, used as a baseline - DEFAULT_HYPERPARAMETERS = { - "epochs": 50, "learning_rate": 0.005, "lstm_units_1": 64, "dropout_1": 0.3, - "lstm_units_2": 64, "dropout_2": 0.3, "dense_units_1": 64, "dropout_3": 0.2, - "dense_units_2": 32, "dropout_4": 0.2, "sequence_length": 30, - "avg_accuracy": -1.0, "avg_loss": -1.0 - } - - HYPERPARAMETERS_COMBO = { - "epochs": [50], - "learning_rate": [0.001], # Different learning rates to test - "lstm_units_1": [64], - "dropout_1": [0.4], - "lstm_units_2": [64], - "dropout_2": [0.4], - "dense_units_1": [64], - "dropout_3": [0.4], - "dense_units_2": [64], - "dropout_4": [0.4], - "sequence_length": [60] # Different sequence lengths to test - } - - def __init__(self, recommender_dir="./processed/recommender", - training_tracker_path="./processed/nse_dashboard_recommender.json", # Tracks general model training status - tuning_parameters_path="./processed/nsereco_parameters.json", # Tracks hyperparameter tuning progress - nifty_profit_tracker_path="./processed/nse_dashboard_nifty_profit.json", # Tracks Nifty 50 profit maximizer results - all_profit_tracker_path="./processed/nse_dashboard_all_profit.json"): # Tracks All stocks profit maximizer results - - self.recommender_dir = Path(recommender_dir) - self.training_tracker_path = Path(training_tracker_path) - self.tuning_parameters_path = Path(tuning_parameters_path) - self.nifty_profit_tracker_path = Path(nifty_profit_tracker_path) - self.all_profit_tracker_path = Path(all_profit_tracker_path) - - self.recommender_dir.mkdir(parents=True, exist_ok=True) - - # Initialize internal data structures for tracking progress - self.training_tracker_data = {} - self.tuning_parameters_data = {} - self.nifty_profit_tracker_data = {} - self.all_profit_tracker_data = {} - - self.progress_lock = threading.Lock() # To ensure thread-safe access to shared data structures in multi-threaded environments - - self._load_all_trackers() # Load existing states from disk when the recommender is initialized - - # Initialize best_hyperparameters from loaded tuning data or fall back to defaults - self.best_hyperparameters = self.tuning_parameters_data.get("best_params_info", {}).get("best_params", self.DEFAULT_HYPERPARAMETERS.copy()) - - # Ensure all possible hyperparameter combinations are generated if not present in the loaded data - self._generate_all_param_combinations_if_needed() - # Save initial state of parameters if they were newly generated or just loaded/updated - self._save_parameters(self.best_hyperparameters, self.tuning_parameters_data.get("all_param_combinations", {}), initial_save=True) - - logging.debug(f"Recommender initialized. Training tracker: {self.training_tracker_path}, Tuning params: {self.tuning_parameters_path}") - - def _load_all_trackers(self): - """Loads all relevant JSON tracker files (training, tuning, profit maximizers) into memory.""" - self.training_tracker_data = load_json_file(self.training_tracker_path, {}) - self.tuning_parameters_data = load_json_file(self.tuning_parameters_path, {}) - self.nifty_profit_tracker_data = load_json_file(self.nifty_profit_tracker_path, {}) - self.all_profit_tracker_data = load_json_file(self.all_profit_tracker_path, {}) - - def _save_training_tracker(self): - """Saves the training tracker data to its JSON file in a thread-safe manner.""" - with self.progress_lock: - save_json_file(self.training_tracker_path, self.training_tracker_data) - - def _save_parameters(self, best_params, all_combinations, tuning_symbols=None, initial_save=False): - """Saves the hyperparameter tuning data (best params and all combinations) to its JSON file in a thread-safe manner.""" - data_to_save = { - "best_params_info": { - "best_params": best_params, - "last_tuned_date": datetime.now().strftime("%Y-%m-%d"), - "tuning_symbols_used": tuning_symbols if tuning_symbols else [] - }, - "all_param_combinations": all_combinations - } - with self.progress_lock: - save_json_file(self.tuning_parameters_path, data_to_save) - if not initial_save: - logging.debug(f"Successfully saved hyperparameters to {self.tuning_parameters_path}") - - def _save_profit_tracker(self, nifty_only=True): - """Saves the relevant profit maximizer tracker data (Nifty 50 or All stocks) in a thread-safe manner.""" - with self.progress_lock: - if nifty_only: - save_json_file(self.nifty_profit_tracker_path, self.nifty_profit_tracker_data) - else: - save_json_file(self.all_profit_tracker_path, self.all_profit_tracker_data) - - def _generate_all_param_combinations_if_needed(self): - """ - Generates all possible hyperparameter combinations if they haven't been generated yet, - and initializes their tuning status to "PENDING". This ensures a comprehensive search space. - """ - if not self.tuning_parameters_data.get("all_param_combinations"): - logging.info("Generating all possible hyperparameter combinations...") - - param_ranges = self.HYPERPARAMETERS_COMBO - keys = list(param_ranges.keys()) - value_lists = list(param_ranges.values()) - - all_combinations_list = list(product(*value_lists)) # Generates Cartesian product of all parameter values - total_combinations = len(all_combinations_list) - - self.tuning_parameters_data["all_param_combinations"] = {} - for i, combo in enumerate(all_combinations_list): - params_dict = dict(zip(keys, combo)) - # Explicitly convert to float for consistency and to avoid potential JSON issues with numpy types - params_dict["learning_rate"] = float(params_dict["learning_rate"]) - params_dict["dropout_1"] = float(params_dict["dropout_1"]) - params_dict["dropout_2"] = float(params_dict["dropout_2"]) - params_dict["dropout_3"] = float(params_dict["dropout_3"]) - params_dict["dropout_4"] = float(params_dict["dropout_4"]) - - combo_key = f"hyper_{i+1}_of_{total_combinations}" # Unique key for each combination - self.tuning_parameters_data["all_param_combinations"][combo_key] = { - "used_params": { - **params_dict, - "avg_accuracy": -1.0, # Initialize accuracy for this combination - "avg_loss": -1.0 # Initialize loss for this combination - }, - "last_tuned_date": "N/A", # Tracks when this combination was last tuned - "tuning_symbols_used": [], # List of symbols used for tuning this combination - "tuning_status": "PENDING" # Initial status for the combination (PENDING, IN_PROGRESS, COMPLETED, FAILED) - } - logging.info(f"Generated {total_combinations} initial hyperparameter combinations.") - - def _normalize_series(self, data_df, normalize_targets=True): - """ - Normalizes input features and optionally single future price targets using separate MinMaxScalers. - Returns normalized feature DataFrame, normalized target array (if applicable), and both scalers. - """ - try: - # Identify numeric features from DL_FEATURES - features_to_scale = [f for f in self.DL_FEATURES if f in data_df.columns and pd.api.types.is_numeric_dtype(data_df[f])] - - if not features_to_scale: - logging.warning("No suitable features found for scaling. Returning original DataFrame.") - return data_df.copy(), None, MinMaxScaler(), None - - # Normalize input features - features_scaler = MinMaxScaler() - normalized_data = features_scaler.fit_transform(data_df[features_to_scale]) - normalized_df = pd.DataFrame(normalized_data, columns=features_to_scale, index=data_df.index) - - # Normalize TARGET_PRICES using a separate scaler - y_scaled = None - target_scaler = None - if normalize_targets and 'TARGET_PRICES' in data_df.columns: - try: - # Extract scalar target values - target_array = np.array([ - t[0] if isinstance(t, (list, np.ndarray)) else t - for t in data_df['TARGET_PRICES'] - ], dtype=np.float32).reshape(-1, 1) - - target_scaler = MinMaxScaler() - y_scaled = target_scaler.fit_transform(target_array) - except Exception as te: - logging.warning(f"Target normalization failed: {te}") - y_scaled = target_array # fallback to unscaled targets - - return normalized_df, y_scaled, features_scaler, target_scaler - - except Exception as e: - logging.error(f"Normalization failed: {e}") - return data_df.copy(), None, MinMaxScaler(), None - - def _create_sequences(self, normalized_features_data, original_labels_data, seq_length=30): - """ - Creates sequences of historical data (X) and corresponding single future price (y) for LSTM regression. - Each y is a scalar value representing the next predicted price. - """ - x, y = [], [] - try: - # Ensure input is a NumPy array - feature_array = np.array(normalized_features_data, dtype=np.float32) - target_array = np.array([ - t[0] if isinstance(t, (list, np.ndarray)) else t - for t in original_labels_data - ], dtype=np.float32).reshape(-1, 1) # shape: (samples, 1) - - for i in range(len(feature_array) - seq_length): - x_seq = feature_array[i:i + seq_length] # shape: (seq_length, features) - y_seq = target_array[i + seq_length] # shape: (1,) - - if np.isnan(y_seq).any(): - continue - - x.append(x_seq) - y.append(y_seq[0]) # extract scalar - - x_arr = np.array(x, dtype=np.float32) # shape: (samples, seq_length, features) - y_arr = np.array(y, dtype=np.float32).reshape(-1, 1) # shape: (samples, 1) - - return x_arr, y_arr - - except Exception as e: - logging.error(f"Error in sequence creation: {e}") - return np.array(x, dtype=np.float32), np.array(y, dtype=np.float32).reshape(-1, 1) - - def _predict_and_recommend(self, model, scaler, x_test): - """ - Uses the trained regression model to predict the next price - based on the most recent sequence. Returns the predicted price and summary stats. - """ - if x_test.shape[0] == 0: - logging.warning("Empty input sequence. Cannot generate prediction.") - return "FLAT", 0.0, None - - # Predict future price - if isinstance(model, torch.nn.Module): - model.eval() - with torch.no_grad(): - device = next(model.parameters()).device - x_tensor = torch.tensor(x_test, dtype=torch.float32).to(device) - predictions = model(x_tensor).cpu().numpy() - else: - predictions = model.predict(x_test, verbose=1) # shape: (samples, 1) - - if scaler is not None: - predictions = scaler.inverse_transform(predictions) # shape: (samples, 1) - last_prediction = float(predictions[-1][0]) # extract scalar - - # Estimate return vs start of sequence - if scaler is not None: - start_price = float(scaler.inverse_transform([[x_test[-1][-1][0]]])[0][0]) if x_test.ndim == 3 else 0.0 - else: - start_price = float(x_test[-1][-1][0]) if x_test.ndim == 3 else 0.0 - predicted_return_pct = ((last_prediction - start_price) / start_price) * 100 if start_price != 0 else 0.0 - - # Classify trend direction - if predicted_return_pct > 1.0: - trend = "UP" - elif predicted_return_pct < -1.0: - trend = "DOWN" - else: - trend = "FLAT" - - return trend, round(predicted_return_pct, 2), round(last_prediction, 2) - - def make_recommendation(self, model, new_data_df, features_scaler, target_scaler, hyperparameters=None): - """ - Generates a future price prediction for the latest sequence of stock data. - - Args: - model (tf.keras.Model): Trained LSTM regression model. - new_data_df (pd.DataFrame): Latest historical data. - features_scaler (MinMaxScaler): Scaler fitted on input features. - target_scaler (MinMaxScaler): Scaler fitted on target price values. - hyperparameters (dict): Includes 'sequence_length'. - - Returns: - tuple: (trend_label, predicted_return_pct, predicted_price) - """ - hyperparameters = hyperparameters if hyperparameters is not None else self.best_hyperparameters - sequence_length = hyperparameters.get("sequence_length", 30) - - if len(new_data_df) < sequence_length: - logging.warning("Insufficient data to create a prediction sequence. Returning default.") - return "FLAT", 0.0, None - - try: - # Normalize input features - features_to_scale = [f for f in self.DL_FEATURES if f in new_data_df.columns and pd.api.types.is_numeric_dtype(new_data_df[f])] - if not features_to_scale: - logging.warning("No valid features found for scaling. Returning default.") - return "FLAT", 0.0, None - - normalized_data = features_scaler.transform(new_data_df[features_to_scale]) - latest_sequence = normalized_data[-sequence_length:] - X_predict = np.reshape(latest_sequence, (1, sequence_length, len(features_to_scale))) - - # Predict future price (scaled) - if isinstance(model, torch.nn.Module): - model.eval() - with torch.no_grad(): - device = next(model.parameters()).device - X_predict_tensor = torch.tensor(X_predict, dtype=torch.float32).to(device) - y_pred_scaled = model(X_predict_tensor).cpu().numpy() - else: - y_pred_scaled = model.predict(X_predict, verbose=0) # shape: (1, 1) - - raw_pred = float(y_pred_scaled[0][0]) - if target_scaler is not None: - predicted_price = target_scaler.inverse_transform([[raw_pred]])[0][0] - else: - predicted_price = raw_pred - predicted_price = round(predicted_price, 4) - - # Estimate return vs last known price - last_close_price = new_data_df.iloc[-1]["CLOSE_PRICE"] if "CLOSE_PRICE" in new_data_df.columns else None - if last_close_price is None or last_close_price == 0: - predicted_return_pct = 0.0 - else: - predicted_return_pct = ((predicted_price - last_close_price) / last_close_price) * 100 - - # Classify trend - if predicted_return_pct > 0.5: - trend = "UP" - elif predicted_return_pct < -0.5: - trend = "DOWN" - else: - trend = "FLAT" - - return trend, round(predicted_return_pct, 2), predicted_price - - except Exception as e: - logging.error(f"Error during prediction: {e}") - return "FLAT", 0.0, None - - def train_model(self, symbol, historical_df, stock_data_manager_instance, hyperparameters=None, param_key=None, tune=False): - """ - Trains an LSTM model for a given stock symbol. - Handles loading/saving models and updating training status in the tracker JSON. - Includes resume logic for interrupted training sessions and is used by both - bulk training and hyperparameter tuning functionalities. - """ - if utils.interrupted: # Check for global interruption signal to allow graceful exit - logging.info(f"[{symbol}] Interruption detected. Skipping training for this symbol.") - return False, None, None, None, "N/A", None, None - - symbol_start_time = time.time() - hyperparameters = hyperparameters if hyperparameters is not None else self.best_hyperparameters - hyper_key = param_key if param_key else "best_params" # Key to identify the hyperparameter set being used - - # Extract hyperparameters, falling back to defaults if not provided - epochs = hyperparameters.get("epochs", self.DEFAULT_HYPERPARAMETERS["epochs"]) - learning_rate = hyperparameters.get("learning_rate", self.DEFAULT_HYPERPARAMETERS["learning_rate"]) - lstm_units_1 = hyperparameters.get("lstm_units_1", self.DEFAULT_HYPERPARAMETERS["lstm_units_1"]) - dropout_1 = hyperparameters.get("dropout_1", self.DEFAULT_HYPERPARAMETERS["dropout_1"]) - lstm_units_2 = hyperparameters.get("lstm_units_2", self.DEFAULT_HYPERPARAMETERS["lstm_units_2"]) - dropout_2 = hyperparameters.get("dropout_2", self.DEFAULT_HYPERPARAMETERS["dropout_2"]) - dense_units_1 = hyperparameters.get("dense_units_1", self.DEFAULT_HYPERPARAMETERS["dense_units_1"]) - dropout_3 = hyperparameters.get("dropout_3", self.DEFAULT_HYPERPARAMETERS["dropout_3"]) - dense_units_2 = hyperparameters.get("dense_units_2", self.DEFAULT_HYPERPARAMETERS["dense_units_2"]) - dropout_4 = hyperparameters.get("dropout_4", self.DEFAULT_HYPERPARAMETERS["dropout_4"]) - seq_length = hyperparameters.get("sequence_length", self.DEFAULT_HYPERPARAMETERS["sequence_length"]) - - # Ensure CLOSE_PRICE is always included - base_features = [f for f in self.DL_FEATURES if f != 'CLOSE_PRICE'] - train_features = ['CLOSE_PRICE'] + base_features - - # Define model save path for PyTorch - model_path_pt = self.recommender_dir / f"nsereco_model_{symbol}.pt" - - # Get current training info for the symbol from the training tracker - info = self.training_tracker_data.get(symbol, {}) - model_exists = model_path_pt.exists() - # Tracker is considered complete if it has all expected keys and 'training' status is True - tracker_complete = all(key in info for key in ["date_trained", "training", "final_loss", "epochs_ran", "accuracy", "recommendation", "confidence_score"]) and info.get("training", False) - - # Use a copy to avoid modifying the original stock_data_manager_instance.df - df_for_target = historical_df.copy() - - # Ensure 'SYMBOL' is in the correct type - if 'SYMBOL' in df_for_target.columns: - df_for_target['SYMBOL'] = df_for_target['SYMBOL'].astype('str') - else: - # If 'SYMBOL' column is missing (shouldn't happen with current data loading), - # assign the current symbol to all rows for consistency. - df_for_target['SYMBOL'] = symbol - - target_df_for_training = df_for_target.copy() # Work on a copy to avoid modifying original data - - if target_df_for_training.empty or 'TARGET_PRICES' not in target_df_for_training.columns: - logging.warning(f"[{symbol}] Failed to generate TARGET PRICES for training or target_df is empty. Skipping training for this symbol.") - return False, None, None, None, "N/A", None, None - - # Scenario 1: Model is already trained and tracker data is complete (only applicable if not tuning) - if not tune and model_exists and tracker_complete: - logging.debug(f"[{symbol}] Skipped: Model already trained and tracker data complete on {info.get('date_trained', 'N/A')}.") - return True, info.get("final_loss"), info.get("epochs_ran"), info.get("accuracy"), info.get("recommendation"), info.get("confidence_score"), 0.0 - - # Scenario 2: Model exists but tracker data is incomplete or indicates previous failure (attempt recovery) - if not tune and model_exists and not tracker_complete: - logging.info(f"[{symbol}] Model exists but tracker data incomplete/failed. Attempting to load model and predict to recover status.") - try: - # Load the existing model - model = LSTMModel(input_dim=len(train_features), params=hyperparameters) - model.load_state_dict(torch.load(model_path_pt)) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.to(device) - - num_features = len(train_features) - if len(target_df_for_training) < seq_length + 1: - logging.debug(f"[{symbol}] Not enough historical data ({len(target_df_for_training)} rows) for prediction. Requires at least {seq_length + 1}.") - return False, None, None, None, "N/A", None, None - - # Prepare data for prediction using the newly labeled DataFrame - normalized_df, original_labels_data, features_scaler, target_scaler = self._normalize_series(target_df_for_training) - for feature in train_features: - if feature not in normalized_df.columns: - normalized_df[feature] = 0.0 # Fill missing features with default value - features_data = normalized_df[train_features].values - # original_labels_data = target_df_for_training['TARGET_PRICES'].values # Use labels from the labeled DataFrame - x, y_true = self._create_sequences(features_data, original_labels_data, seq_length=seq_length) - - if len(x) == 0: - logging.warning(f"[{symbol}] No sequences created for prediction after data building. Cannot recover status.") - return False, None, None, None, "N/A", None, None - - # Make prediction and evaluate model performance - recommendation, confidence_score, prediction = self._predict_and_recommend(model, target_scaler, x) - x_tensor = torch.tensor(x, dtype=torch.float32).to(device) - y_true_tensor = torch.tensor(y_true, dtype=torch.float32).to(device) - model.eval() - with torch.no_grad(): - pred = model(x_tensor) - loss = torch.nn.MSELoss()(pred, y_true_tensor).item() - accuracy = torch.nn.L1Loss()(pred, y_true_tensor).item() - - today_str = datetime.now().strftime("%Y-%m-%d") - with self.progress_lock: # Thread-safe update of tracker data - self.training_tracker_data[symbol] = { - "date_trained": today_str, - "training": True, # Mark as successfully recovered - "final_loss": round(float(loss), 4), - "epochs_ran": info.get("epochs_ran", "N/A"), # Preserve original epochs if available - "accuracy": round(float(accuracy), 4), - "recommendation": recommendation, - "confidence_score": round(float(confidence_score), 4) if confidence_score is not None else None - } - self._save_training_tracker() # Persist the updated tracker - time_to_complete_symbol = time.time() - symbol_start_time - logging.info(f"[{symbol}] Predicted using existing model. Reco: {recommendation}, Conf: {confidence_score:.4f}, Acc: {accuracy:.4f} Loss: {loss:.4f} Prediction: {prediction} Time: {time_to_complete_symbol:.2f}s (Status Recovered)") - return True, loss, info.get("epochs_ran", "N/A"), accuracy, recommendation, confidence_score, time_to_complete_symbol - except Exception as e: - logging.error(f"[{symbol}] Failed to load or predict with existing model during recovery attempt ({e}). Proceeding with re-training.", exc_info=True) - # Fall through to actual training if recovery fails - - # --- Actual Model Training Logic (Scenario 3: No model, explicit re-training, or tuning) --- - if tune: - logging.info(f"[{symbol}] [{hyper_key}] Starting model training for hyperparameter tuning.") - else: - logging.info(f"[{symbol}] Starting training (model missing, no tracker entry, or explicit re-training).") - - num_features = len(train_features) - # Use labeled_df_for_training for sequence creation - if len(target_df_for_training) < seq_length + 1: - logging.debug(f"[{symbol}] Skipped: Not enough historical data ({len(target_df_for_training)} rows). Requires at least {seq_length + 1} for sequence creation.") - return False, None, None, None, "N/A", None, None - - # Step 1: Normalize features and target prices - normalized_df, y_scaled, features_scaler, target_scaler = self._normalize_series( - target_df_for_training, normalize_targets=True) - - # Step 2: Ensure all train_features are present - for feature in train_features: - if feature not in normalized_df.columns: - normalized_df[feature] = 0.0 # Fill missing features with zeros - - # Step 3: Extract normalized feature matrix - features_data = normalized_df[train_features].values - - # Step 4: Use scaled target vectors for sequence creation - # Ensure y_scaled is a NumPy array with shape (num_samples, 5) - if y_scaled is None or len(y_scaled) != len(features_data): - logging.warning("Scaled target data is missing or misaligned. Falling back to unscaled targets.") - original_labels_data = target_df_for_training['TARGET_PRICES'].values - else: - original_labels_data = y_scaled # Use scaled target vectors - - # Step 5: Create sequences for LSTM - x, y = self._create_sequences(features_data, original_labels_data, seq_length=seq_length) - - if len(x) == 0: - logging.warning(f"[{symbol}] Skipped: No sequences created after data building. Check data consistency or length.") - return False, None, None, None, "N/A", None, None - - # Build the LSTM model architecture for 1-day price prediction - model = LSTMModel(input_dim=x.shape[2], params=hyperparameters) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.to(device) - - optimizer = TorchAdam(model.parameters(), lr=learning_rate) - loss_fn = torch.nn.MSELoss() # Match the 'mean_squared_error' from Keras - mae_fn = torch.nn.L1Loss() - best_weight_filepath = self.recommender_dir / f"model_best_weight_{symbol}.pt" - - # Initialize training metrics to None or default values - final_loss = None - epochs_ran = 0 - final_mae = None - recommendation = "N/A" - confidence_score = None - - try: - # Train the model using the prepared data and callbacks - # Split data explicitly - # Determine split index - split_index = int(len(x) * 0.8) - - # Chronological split - x_train, x_test = x[:split_index], x[split_index:] - y_train, y_test = y[:split_index], y[split_index:] - - x_train_tensor = torch.tensor(x_train, dtype=torch.float32) - y_train_tensor = torch.tensor(y_train, dtype=torch.float32) - x_test_tensor = torch.tensor(x_test, dtype=torch.float32) - y_test_tensor = torch.tensor(y_test, dtype=torch.float32) - - train_loader = DataLoader( - TensorDataset(x_train_tensor, y_train_tensor), - batch_size=64, - shuffle=True, - drop_last=False - ) - val_loader = DataLoader(TensorDataset(x_test_tensor, y_test_tensor), batch_size=64) - - best_val_loss = float('inf') - early_stop_counter = 0 - patience = 5 - - for epoch in range(epochs): - model.train() - train_losses = [] - for xb, yb in train_loader: - xb, yb = xb.to(device), yb.to(device) - optimizer.zero_grad() - pred = model(xb) - loss = loss_fn(pred, yb) - loss.backward() - optimizer.step() - train_losses.append(loss.item()) - - model.eval() - val_losses = [] - val_maes = [] - with torch.no_grad(): - for xb, yb in val_loader: - xb, yb = xb.to(device), yb.to(device) - pred = model(xb) - val_loss = loss_fn(pred, yb) - val_mae = mae_fn(pred, yb) - val_losses.append(val_loss.item()) - val_maes.append(val_mae.item()) - - avg_val_loss = np.mean(val_losses) - avg_val_mae = np.mean(val_maes) - final_loss = avg_val_loss - final_mae = avg_val_mae - epochs_ran = epoch + 1 - - if avg_val_loss < best_val_loss: - best_val_loss = avg_val_loss - early_stop_counter = 0 - torch.save(model.state_dict(), best_weight_filepath) - else: - early_stop_counter += 1 - if early_stop_counter >= patience: - logging.info(f"[{symbol}] Early stopping at epoch {epoch+1}") - break - - # Make a prediction with the newly trained model - try: - model.load_state_dict(torch.load(best_weight_filepath)) - logging.info(f"Best weights loaded from file...") - except: - logging.info(f"Error loading best weights from file...") - # Use the test set for prediction to get a realistic recommendation - recommendation, predicted_percent, predicted_price = self._predict_and_recommend(model, target_scaler, x_test) - - time_to_complete_symbol = time.time() - symbol_start_time # Calculate total time taken for training - # Only save the model and update the main training tracker if not in tuning mode. - # Tuning results are managed separately by `tune_model`. - if not tune: - with self.progress_lock: # Ensure thread-safe update of training tracker data - self.training_tracker_data[symbol] = { - "date_trained": datetime.now().strftime("%Y-%m-%d"), # Date of training - "training": True, # Mark as successfully trained - "final_loss": round(float(final_loss), 4), - "epochs_ran": int(epochs_ran), - "final_mae": round(float(final_mae), 4), - "recommendation": recommendation, - "predicted_price": round(float(predicted_price), 2) if predicted_price is not None else None - } - self._save_training_tracker() # Persist the updated tracker data to JSON - try: - torch.save(model.state_dict(), model_path_pt) - except Exception as e: - logging.error(f"[{symbol}] Failed to save model to {model_path_pt}: {e}", exc_info=True) - logging.debug(f"[{symbol}] Model saved to {model_path_pt}") - - # Generate and save backtest plot - try: - # Generate predictions for the full dataset to plot backtest - model.eval() - x_tensor = torch.tensor(x, dtype=torch.float32).to(device) - with torch.no_grad(): - y_pred_scaled = model(x_tensor).cpu().numpy() - - y_pred_unscaled = target_scaler.inverse_transform(y_pred_scaled).reshape(-1) - y_actual_unscaled = target_scaler.inverse_transform(y).reshape(-1) - - # Get latest target price and prediction error - actual_price = float(y_actual_unscaled[-1]) - pred_price = float(y_pred_unscaled[-1]) - pred_error = ((pred_price - actual_price) / actual_price * 100) - - with matplotlib_lock: - # Temporarily restore original warnings.warn to prevent langchain's filtered_warn crash with matplotlib - orig_warn = warnings.warn - if hasattr(sys, 'original_warn'): - warnings.warn = sys.original_warn - try: - sns.set_style("darkgrid", {"axes.facecolor": "black", "grid.color": ".5"}) - plt.figure(figsize=(19.2, 10.8), dpi=100) - plt.plot(y_actual_unscaled, label='Actual TARGET_PRICES', color='#00FF41', linestyle='-', marker='o', ms=4, markevery=max(1, len(y_actual_unscaled)//50)) - plt.plot(y_pred_unscaled, label='Predicted TARGET_PRICES', color='#FFD700', linestyle='--', marker='x', ms=4, markevery=max(1, len(y_pred_unscaled)//50)) - - plt.title(f"{symbol} Target Prices Backtest", color='white', fontsize=16) - plt.xlabel('Time Steps', color='white', fontsize=12) - plt.ylabel('TARGET_PRICES', color='white', fontsize=12) - plt.legend(facecolor='black', edgecolor='white', labelcolor='white') - plt.xticks(color='white') - plt.yticks(color='white') - for spine in plt.gca().spines.values(): - spine.set_color('white') - - # Overlay hyperparameters (bottom-right) and results (top-left) - params_text = ( - "Hyperparameters:\n" + \ - "\n".join([f" {k}: {v:.6f}" if isinstance(v, float) else f" {k}: {v}" for k, v in hyperparameters.items()]) - ) - results_text = ( - "Test Results:\n" - f" Final Loss: {final_loss:.4f}\n" - f" Epochs Ran: {epochs_ran}\n" - f" Actual Price: {actual_price:.2f}\n" - f" Predicted Price: {pred_price:.2f}\n" - f" Pred Error: {pred_error:.4f}%\n" - ) - ax = plt.gca() - props = dict(boxstyle='round,pad=0.5', facecolor='#111111', alpha=0.8, edgecolor='#555555') - ax.text(0.02, 0.95, results_text, transform=ax.transAxes, fontsize=11, color='#00FF41', - verticalalignment='top', bbox=props, family='monospace') - ax.text(0.98, 0.05, params_text, transform=ax.transAxes, fontsize=11, color='#FFD700', - verticalalignment='bottom', horizontalalignment='right', bbox=props, family='monospace') - - vn = 1 - backtest_filepath = self.recommender_dir / f"nsereco_backtest_{symbol}_v{vn}.png" - while backtest_filepath.exists(): - vn += 1 - backtest_filepath = self.recommender_dir / f"nsereco_backtest_{symbol}_v{vn}.png" - - plt.savefig(backtest_filepath, facecolor='black') - plt.close() - logging.info(f"[{symbol}] Saved backtest plot to {backtest_filepath}") - finally: - warnings.warn = orig_warn - except Exception as e: - logging.error(f"[{symbol}] Failed to generate/save backtest plot: {e}", exc_info=True) - - # Return training results - logging.info(f"[{symbol}] Tuned | Reco: {recommendation} Pred Price: {predicted_price:.2f} mae: {final_mae:.4f} Loss: {final_loss:.4f} Epochs: {epochs_ran} Predicted % Gain: {predicted_percent} Time: {time_to_complete_symbol:.2f}s") - return True, final_loss, epochs_ran, final_mae, recommendation, predicted_price, time_to_complete_symbol - - except Exception as e: - logging.error(f"\n[{symbol}] Training (model.fit or prediction) failed: {e}", exc_info=True) - if not tune: # Update tracker with error information if not in tuning mode - with self.progress_lock: - self.training_tracker_data[symbol] = { - "date_trained": datetime.now().strftime("%Y-%m-%d"), - "training": False, # Mark as failed training - "error": str(e) - } - self._save_training_tracker() - # Return failure status and None for metrics - return False, None, None, None, "N/A", None, None - - def tune_model(self, stock_data_manager, num_symbols_for_tuning=10, num_tuning_cycles=1, max_workers=16): - """ - Orchestrates the hyperparameter tuning process for the LSTM model. - It samples a subset of NIFTY 50 stocks, evaluates different hyperparameter combinations, - and identifies the best-performing set. It also handles resuming tuning and logging progress. - """ - logging.info(f"Starting model tuning with {num_symbols_for_tuning} symbols for {num_tuning_cycles} cycles...") - - # Create a local copy of dataframe for tuning - tuning_df = stock_data_manager.df.copy() - - # Remove dropped_features from the DataFrame - if self.dropped_features: - tuning_df.drop(columns=self.dropped_features, errors='ignore', inplace=True) - # Drop index if it exists - tuning_df.reset_index(drop=True, inplace=True) - # Ensure the DataFrame is not empty after dropping features - if tuning_df.empty: - logging.error("Tuning DataFrame is empty after dropping features. Cannot proceed with tuning...") - return - else: - # Log the number of rows and columns labels after feature adjustments - logging.info(f"Tuning DataFrame after feature adjustments: {tuning_df.shape[0]} rows, {tuning_df.shape[1]} columns.") - # Log the columns remaining after dropping features - logging.info(f"Remaining columns after dropping features: {tuning_df.columns.tolist()}") - # Ensure 'SYMBOL' column is present for grouping - if 'SYMBOL' not in tuning_df.columns: - logging.error("Tuning DataFrame is missing 'SYMBOL' column. Cannot proceed with tuning...") - return - - all_symbols = NIFTY_STOCKS # Tuning is specifically performed on NIFTY 50 stocks as per requirement - if len(all_symbols) < num_symbols_for_tuning: - logging.warning(f"Not enough NIFTY 50 symbols ({len(all_symbols)}) for tuning. Using all available NIFTY 50 symbols...") - num_symbols_for_tuning = len(all_symbols) - - # Load latest best hyperparameters and all combinations from the tuning tracker - best_avg_accuracy = self.best_hyperparameters["avg_accuracy"] - best_avg_loss = self.best_hyperparameters["avg_loss"] - best_params = self.best_hyperparameters.copy() # Copy to avoid direct modification - - MAX_WORKERS = max(1, max_workers) # Ensure at least 1 worker thread - logging.info(f"Using {MAX_WORKERS} parallel threads for hyperparameter tuning...") - - self._load_all_trackers() # Ensure latest state of all trackers is loaded - # Ensure combinations are generated if the file was empty or missing. - self._generate_all_param_combinations_if_needed() - all_param_combinations = self.tuning_parameters_data.get("all_param_combinations", {}) - - # Prioritize combinations based on their tuning status for efficient resuming: - # PENDING (not started) > FAILED (to retry) > IN_PROGRESS (interrupted) - params_to_tune_keys = sorted([k for k, v in all_param_combinations.items() if v.get("tuning_status", "PENDING") in ["PENDING", "FAILED", "IN_PROGRESS"]], - key=lambda k: ( - all_param_combinations[k].get("tuning_status", "PENDING") == "IN_PROGRESS", - all_param_combinations[k].get("tuning_status", "PENDING") == "FAILED", - all_param_combinations[k].get("last_tuned_date", "9999-12-31") - )) - - # If no pending/failed/in-progress combinations are found, select the oldest COMPLETED ones for re-tuning - if not params_to_tune_keys: - logging.info("No pending, failed or in-progress hyperparameter combinations. Selecting oldest completed for re-tuning...") - params_to_tune_keys = sorted([k for k, v in all_param_combinations.items() if v.get("tuning_status") == "COMPLETED"], - key=lambda k: all_param_combinations[k].get("last_tuned_date", "9999-12-31")) - - if not params_to_tune_keys: - logging.warning("No hyperparameter combinations available for tuning. Exiting tuning...") - return - - # Iterate through defined tuning cycles - for cycle in range(num_tuning_cycles): - if utils.interrupted: # Check for global interruption - logging.info("Interruption detected. Stopping tuning cycles...") - break # Break from outer tuning cycles loop - - logging.info(f"\n--- Starting Tuning Cycle {cycle + 1}/{num_tuning_cycles} ---") - - # Select up to 16 combinations for this specific cycle from the prioritized list - current_cycle_params_keys = params_to_tune_keys[cycle * 16: (cycle + 1) * 16] - if not current_cycle_params_keys: - logging.info("No more hyperparameter combinations to tune in this cycle...") - break # No more combinations to process - - # Initialize lists to store accuracies and losses for each parameter set within this cycle - cycle_accuracies_by_param = {key: [] for key in current_cycle_params_keys} - cycle_losses_by_param = {key: [] for key in current_cycle_params_keys} - - total_symbols_in_cycle = len(current_cycle_params_keys) * num_symbols_for_tuning - trained_count_cycle = 0 - skipped_count_cycle = 0 - start_time_cycle = time.time() - - # Mark selected combinations as IN_PROGRESS before starting threads - for param_key in current_cycle_params_keys: - all_param_combinations[param_key]["tuning_status"] = "IN_PROGRESS" - self._save_parameters(best_params, all_param_combinations, initial_save=False) # Save immediately to persist status - - # Use ThreadPoolExecutor for parallel training of models for different combinations/symbols - with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: - futures = {} - for param_key in current_cycle_params_keys: - # Randomly select a subset of symbols for tuning each specific hyperparameter combination - #tuning_symbols_for_combo = np.random.choice(all_symbols, num_symbols_for_tuning, replace=False).tolist() - tuning_symbols_for_combo = ['SBIN'] - current_params = all_param_combinations[param_key]["used_params"] - logging.info(f"--- Tuning hyper number: {param_key} ---") - logging.info(f"[Hyperparameters: {json.dumps(current_params)}]") - logging.info(f"Symbols randomly selected for tuning: {tuning_symbols_for_combo}") - - for sym in tuning_symbols_for_combo: - tuning_df_for_sym = tuning_df[tuning_df['SYMBOL'] == sym].copy() # Filter data for the current symbol - # View final correlation after final feature generation - # StockData.run_correlation_analysis(tuning_df_for_sym) - futures[executor.submit( - self.train_model, # Call the train_model method - sym, - tuning_df_for_sym, # Pass symbol's data - stock_data_manager, # Pass stock_data_manager_instance - current_params, # Pass the hyperparameter set to be tested - param_key, # Pass the key for tracking - tune=True # Indicate that this is a tuning run - )] = (sym, param_key) # Store mapping for result tracking - - # Process results as futures complete (non-blocking) - for future in as_completed(futures): - if utils.interrupted: - logging.info("Interruption detected during tuning. Breaking from current cycle.") - break # Break from as_completed loop - - sym, param_key = futures[future] # Retrieve symbol and parameter key - try: - # Get the results from the completed future (with a timeout) - trained_status, final_loss, epochs_ran, accuracy, recommendation, confidence, time_taken = future.result(timeout=1200) # Increased timeout to 20 minutes (1200 seconds) - - if trained_status and accuracy is not None and final_loss is not None: - cycle_accuracies_by_param[param_key].append(accuracy) # Collect accuracy - cycle_losses_by_param[param_key].append(final_loss) # Collect loss - trained_count_cycle += 1 - logging.info(f"[{sym}] [{param_key}] Processed | Tuned: {trained_count_cycle} Skipped: {skipped_count_cycle} Time: {time_taken:.2f}s final_loss: {final_loss:.4f} accuracy: {accuracy:.4f} recommendation: {recommendation} confidence: {confidence:.4f}") - else: - skipped_count_cycle += 1 - logging.info(f"[{sym}] [{param_key}] Processed | Tuned: {trained_count_cycle} Skipped: {skipped_count_cycle} Time: {time_taken:.2f}s (Failed to train)") - except TimeoutError: - logging.error(f"[TIMEOUT] [{param_key}] Tuning for {sym} timed out.") - skipped_count_cycle += 1 - except Exception as e: - skipped_count_cycle += 1 - logging.error(f"[ERROR] [{param_key}] Tuning failed for {sym} with error: {e}", exc_info=True) - - # After all symbols for this cycle's combinations have been processed, - # update the average accuracy and loss for each parameter set - for param_key in current_cycle_params_keys: - accuracies = cycle_accuracies_by_param[param_key] - losses = cycle_losses_by_param[param_key] - - if accuracies: # Only update if there were successful training runs - avg_accuracy_for_param = np.mean(accuracies) - avg_loss_for_param = np.mean(losses) - - # Update the parameter combination's metrics and status - all_param_combinations[param_key]["used_params"]["avg_accuracy"] = round(float(avg_accuracy_for_param), 4) - all_param_combinations[param_key]["used_params"]["avg_loss"] = round(float(avg_loss_for_param), 4) - all_param_combinations[param_key]["last_tuned_date"] = datetime.now().strftime("%Y-%m-%d") - all_param_combinations[param_key]["tuning_symbols_used"] = tuning_symbols_for_combo # Store symbols used for this tune - all_param_combinations[param_key]["tuning_status"] = "COMPLETED" # Mark as completed - logging.info(f"[{param_key}] Final cycle average (Avg Acc: {avg_accuracy_for_param:.4f}, Avg Loss: {avg_loss_for_param:.4f})") - - # Check if this parameter set is the new best overall - if avg_accuracy_for_param > best_avg_accuracy or \ - (avg_accuracy_for_param == best_avg_accuracy and avg_loss_for_param < best_avg_loss): - best_avg_accuracy = avg_accuracy_for_param - best_avg_loss = avg_loss_for_param - best_params = all_param_combinations[param_key]["used_params"].copy() - logging.info(f"New best parameters found (Avg Acc: {best_avg_accuracy:.4f}, Avg Loss: {best_avg_loss:.4f}): {best_params}") - else: - all_param_combinations[param_key]["tuning_status"] = "FAILED" # Mark as failed if no successful runs for this combo - logging.warning(f"No successful training runs for parameter set {param_key} in Cycle {cycle+1}. Marked as FAILED.") - - # Save parameters after each cycle for persistence (critical for interruption and resume) - self._save_parameters(best_params, all_param_combinations, initial_save=False) - logging.info(f"Saved tuning progress after Cycle {cycle + 1}.") - - if utils.interrupted: - logging.info("Interruption detected. Exiting tuning process.") - break # Break from outer tuning cycles loop - - logging.info(f"Final Best Hyperparameters (Avg Acc: {best_avg_accuracy:.4f}, Avg Loss: {best_avg_loss:.4f}): {best_params}") - self._save_parameters(best_params, all_param_combinations, initial_save=False) # Final save of all parameters - self.best_hyperparameters = best_params # Update the instance's best_hyperparameters for future use by other functions - return best_params - - def run_model_training_all_stocks(self, stock_data_manager, max_workers=1, training_timeout_seconds=1800): - """ - Orchestrates bulk model training for all stocks using best hyperparameters. - Processes symbols in parallel and updates tracker with training status and metrics. - """ - logging.info("Starting model training for all stocks...") - - self._load_all_trackers() - - model_df = stock_data_manager.df.copy() - - if self.dropped_features: - model_df.drop(columns=self.dropped_features, errors='ignore', inplace=True) - model_df.reset_index(drop=True, inplace=True) - - if model_df.empty: - logging.error("DataFrame is empty after dropping features. Aborting training.") - return - - if 'SYMBOL' not in model_df.columns: - logging.error("'SYMBOL' column missing. Cannot group data.") - return - - logging.info(f"DataFrame ready: {model_df.shape[0]} rows, {model_df.shape[1]} columns.") - logging.info(f"Remaining columns: {model_df.columns.tolist()}") - - all_symbols = model_df['SYMBOL'].unique().tolist() - if not all_symbols: - logging.error("No symbols found in data.") - return - - symbols_to_train = [ - sym for sym in all_symbols - if sym not in self.training_tracker_data or not self.training_tracker_data[sym].get("training", False) - ] - - if not symbols_to_train: - logging.info("All models are already trained. Nothing to do.") - return - - # Do a random shuffling of symbols to distribute load - np.random.shuffle(symbols_to_train) - total_symbols = len(symbols_to_train) - trained_count = 0 - skipped_count = 0 - start_time_all = time.time() - - logging.info(f"Launching {max_workers} threads to train {total_symbols} symbols...") - - def get_historical_df_for_symbol(symbol): - return model_df[model_df['SYMBOL'] == symbol].copy() - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_to_symbol = { - executor.submit( - self.train_model, - sym, - get_historical_df_for_symbol(sym), - stock_data_manager, - self.best_hyperparameters, - None, - False - ): sym - for sym in symbols_to_train - } - - for future in as_completed(future_to_symbol): - if utils.interrupted: - logging.info("Interruption detected. Halting training.") - break - - sym = future_to_symbol[future] - try: - result = future.result(timeout=training_timeout_seconds) - if not isinstance(result, tuple) or len(result) < 7: - raise ValueError("Invalid result format from train_model") - - trained_status, final_loss, epochs_ran, final_mae, recommendation, confidence, time_taken = result - - with self.progress_lock: - self.training_tracker_data[sym] = { - "date_trained": datetime.now().strftime("%Y-%m-%d"), - "training": trained_status, - "final_loss": round(final_loss, 4), - "final_mae": round(final_mae, 4), - "epochs_ran": epochs_ran, - "recommendation": recommendation, - "confidence": round(confidence, 4), - "time_taken_sec": round(time_taken, 2) - } - self._save_training_tracker() - - if trained_status: - trained_count += 1 - else: - skipped_count += 1 - - except TimeoutError: - logging.error(f"[TIMEOUT] {sym} exceeded {training_timeout_seconds}s. Skipping.") - skipped_count += 1 - with self.progress_lock: - self.training_tracker_data[sym] = { - "date_trained": datetime.now().strftime("%Y-%m-%d"), - "training": False, - "error": f"Timeout after {training_timeout_seconds}s" - } - self._save_training_tracker() - - except Exception as e: - skipped_count += 1 - logging.error(f"[ERROR] {sym} failed: {e}", exc_info=True) - with self.progress_lock: - self.training_tracker_data[sym] = { - "date_trained": datetime.now().strftime("%Y-%m-%d"), - "training": False, - "error": str(e) - } - self._save_training_tracker() - - # Progress logging - total_processed = trained_count + skipped_count - elapsed_total = time.time() - start_time_all - pct_complete = (total_processed / total_symbols) * 100 if total_symbols else 0 - - if trained_count > 0 and elapsed_total > 0: - rate_per_sec = trained_count / elapsed_total - eta_seconds = (total_symbols - total_processed) / rate_per_sec if rate_per_sec > 0 else float('inf') - eta_display = str(timedelta(seconds=int(eta_seconds))) - else: - eta_display = "N/A" - - logging.info(f"[{sym}] Trained: {trained_count}, Skipped: {skipped_count}, Progress: {total_processed}/{total_symbols}, {pct_complete:.2f}% complete, ETA: {eta_display}") - - total_duration = time.time() - start_time_all - logging.info(f"\nTraining complete in {total_duration:.2f} seconds.") - logging.info(f"Symbols processed: {total_symbols}, Trained: {trained_count}, Skipped: {skipped_count}") - - if trained_count > 0: - logging.info(f"Avg Training Rate: {(trained_count / total_duration * 60):.2f} models/min") - else: - logging.info("No models were successfully trained.") - - def run_optuna_tuning(self, stock_data_manager, num_trials=30, num_symbols=10, max_workers=1, tuning_symbols=None): - logging.info(f"Starting Optuna hyperparameter tuning with {max_workers} threads...") - - # Ensure CLOSE_PRICE is always included - base_features = [f for f in self.DL_FEATURES if f != 'CLOSE_PRICE'] - - # Generate all combinations of features including CLOSE_PRICE - self.feature_combos = ['CLOSE_PRICE'] + base_features - - def price_match_error(y_true, y_pred): - """ - Custom loss: scaled absolute error using only the last value in the batch. - """ - epsilon = K.epsilon() - - # Extract last value from batch - y_true_last = y_true[-1] - y_pred_last = y_pred[-1] - - # Compute normalized absolute error - error = K.abs(y_pred_last - y_true_last) / (K.abs(y_true_last) + epsilon) - error = K.clip(error, 0.0, 1.0) - - return error - - def _save_tuning_history(trial_number, total_trials, params, best_loss, final_loss, final_mae, epochs_ran, symbol, - target_price=0, pred_price=0, pred_error=0, best_pred=0, best_error=0, best_epoch=0, weights_filepath='', best_key="NO", status="COMPLETED"): - """ - Saves tuning snapshot to nsereco_tuner.json in structured format. - Includes per-symbol metrics if provided. - """ - symbol = symbol[0] if isinstance(symbol, list) else symbol - tuner_path = Path(self.recommender_dir) / f"nsereco_tuner_{symbol}.json" - - trial_key = f"trial_{trial_number}_of_{total_trials}" - timestamp = datetime.now().strftime("%Y-%m-%d") - pred_error=abs(round(pred_error, 4)) - best_error=abs(round(float(best_error), 4)) - best_pred=round(float(best_pred), 2) - - # Convert to standard Python float/int to prevent JSON serialization errors with float32/int64 - best_loss = float(best_loss) - final_loss = float(final_loss) - final_mae = float(final_mae) - target_price = float(target_price) - pred_price = float(pred_price) - pred_error = float(pred_error) - best_pred = float(best_pred) - best_error = float(best_error) - best_epoch = int(best_epoch) - epochs_ran = int(epochs_ran) - - entry = { - "used_params": { - **params, - "final_mae": round(final_mae, 4), - "best_loss": round(best_loss, 4), - "final_loss": round(final_loss, 4), - "epochs_ran": epochs_ran - }, - "target_price": target_price, - "predicted_price": pred_price, - "predicted_price_error": abs(round(pred_error, 4)), - "best_predicted_price": round(best_pred, 2), - "best_predicted_price_error": abs(round(best_error, 4)), - "best_epoch": best_epoch, - "last_trial_date": timestamp, - "trial_symbols_used": symbol, - "weights_filepath": str(weights_filepath) if weights_filepath else "", - "trial_status": status, - "best_key": "NO" - } - - # Load existing data if file exists - data = {} - if tuner_path.exists(): - try: - with open(tuner_path, "r") as f: - content = f.read().strip() - if content: - data = json.loads(content) - except Exception as e: - logging.warning(f"Failed to load tuner file: {e}. Reinitializing.") - - # Ensure the main structure exists - if "all_param_combinations" not in data: - data["all_param_combinations"] = {} - - # Update the main data structure - data["all_param_combinations"][trial_key] = entry - - # Write back to file - with open(tuner_path, "w") as f: - json.dump(data, f, indent=2) - - #logging.info(f"Saved tuning history for {trial_key} to {tuner_path}") - - def create_functional_attention_model(input_shape, params): - # Use the functional API for explicit layer connections - inputs = Input(shape=input_shape) - - # 1D Convolutional Layer - conv1d = Conv1D(filters=64, kernel_size=3, activation='relu', padding='causal')(inputs) - batch_norm_c1 = BatchNormalization()(conv1d) - dropout_c1 = Dropout(0.2)(batch_norm_c1) - - # Bidirectional LSTM Layer 1 (returns sequences) - bilstm1_out = Bidirectional(LSTM(params["lstm_units_1"], return_sequences=True, activation='tanh', recurrent_activation='sigmoid', unroll=True))(dropout_c1) - batch_norm_lstm1 = BatchNormalization()(bilstm1_out) - dropout_lstm1 = Dropout(params["dropout_1"])(batch_norm_lstm1) - - # Attention layer requires a list of inputs (query, value) - # For self-attention, the query and value are from the same source - attention_output = Attention()([dropout_lstm1, dropout_lstm1]) - - # Concatenate the attention output with the original LSTM output - concat_layer = Concatenate()([dropout_lstm1, attention_output]) - - # Bidirectional LSTM Layer 2 (returns a single output) - bilstm2_out = Bidirectional(LSTM(params["lstm_units_1"], return_sequences=False, activation='tanh', recurrent_activation='sigmoid', unroll=True))(concat_layer) - batch_norm_lstm2 = BatchNormalization()(bilstm2_out) - dropout_lstm2 = Dropout(params["dropout_2"])(batch_norm_lstm2) - - # Dense layers with regularization - dense1 = Dense(params["dense_units_1"], activation='relu')(dropout_lstm2) - batch_norm_d1 = BatchNormalization()(dense1) - dropout_d1 = Dropout(params["dropout_3"])(batch_norm_d1) - - dense2 = Dense(params["dense_units_2"], activation='relu')(dropout_d1) - batch_norm_d2 = BatchNormalization()(dense2) - dropout_d2 = Dropout(params["dropout_4"])(batch_norm_d2) - - # Output layer - outputs = Dense(1, activation='linear')(dropout_d2) - - # Create the model using the functional API - model = Model(inputs=inputs, outputs=outputs) - - # Compile the model - model.compile(optimizer=Adam(learning_rate=params["learning_rate"]), loss='huber', metrics=['mae', 'mse']) - - return model - - # Example of how to call the function - # model = create_functional_attention_model(input_shape, params) - - def create_LSTM_model(input_shape, params): - """ - Creates a hybrid CNN-Attention-Bidirectional LSTM model using the Keras Functional API. - - Args: - input_shape (tuple): The shape of the input data (sequence_length, num_features). - params (dict): A dictionary containing hyperparameters for the model layers. - - Returns: - A compiled Keras model. - """ - # Input layer - inputs = Input(shape=input_shape) - - # 1D Convolutional Layer - #conv1d = Conv1D(filters=64, kernel_size=3, activation='relu', padding='causal')(inputs) - #batch_norm_c1 = BatchNormalization()(conv1d) - #dropout_c1 = Dropout(0.2)(batch_norm_c1) - - # Bidirectional LSTM Layer 1 (returns sequences for the next LSTM and attention) - bilstm1_out = Bidirectional(LSTM(params["lstm_units_1"], return_sequences=True, activation='tanh', recurrent_activation='sigmoid', unroll=True))(inputs) - batch_norm_lstm1 = BatchNormalization()(bilstm1_out) - dropout_lstm1 = Dropout(params["dropout_1"])(batch_norm_lstm1) - - # Attention Mechanism (requires [query, value] inputs) - #attention_output = Attention()([dropout_lstm1, dropout_lstm1]) - - # Concatenate attention output with the original LSTM output - #concat_layer = Concatenate()([dropout_lstm1, attention_output]) - - # Bidirectional LSTM Layer 2 (returns a single output) - bilstm2_out = Bidirectional(LSTM(params["lstm_units_1"], return_sequences=False, activation='tanh', recurrent_activation='sigmoid', unroll=True))(dropout_lstm1) - batch_norm_lstm2 = BatchNormalization()(bilstm2_out) - dropout_lstm2 = Dropout(params["dropout_2"])(batch_norm_lstm2) - - # Dense layers with regularization - dense1 = Dense(params["dense_units_1"], activation='relu')(dropout_lstm2) - batch_norm_d1 = BatchNormalization()(dense1) - dropout_d1 = Dropout(params["dropout_3"])(batch_norm_d1) - - dense2 = Dense(params["dense_units_2"], activation='relu')(dropout_d1) - batch_norm_d2 = BatchNormalization()(dense2) - dropout_d2 = Dropout(params["dropout_4"])(batch_norm_d2) - - # Output layer - outputs = Dense(1, activation='linear')(dropout_d2) - - # Create the model using the functional API - model = Model(inputs=inputs, outputs=outputs) - - # Compile the model - model.compile(optimizer=Adam(learning_rate=params["learning_rate"]), loss='huber', metrics=['mae', 'mse']) - - return model - - def create_GRU_model(input_shape, params): - """ - Creates a hybrid CNN-Attention-Bidirectional GRU model using the Keras Functional API. - - Args: - input_shape (tuple): The shape of the input data (sequence_length, num_features). - params (dict): A dictionary containing hyperparameters for the model layers. - - Returns: - A compiled Keras model. - """ - # Input layer - inputs = Input(shape=input_shape) - - # 1D Convolutional Layer - conv1d = Conv1D(filters=64, kernel_size=3, activation='relu', padding='causal')(inputs) - batch_norm_c1 = BatchNormalization()(conv1d) - dropout_c1 = Dropout(0.2)(batch_norm_c1) - - # Bidirectional GRU Layer 1 (returns sequences for the next GRU and attention) - # GRUs do not have 'recurrent_activation' parameters like LSTMs, so that is removed. - bi_gru1_out = Bidirectional(GRU(params["lstm_units_1"], return_sequences=True, activation='tanh', unroll=True))(dropout_c1) - batch_norm_gru1 = BatchNormalization()(bi_gru1_out) - dropout_gru1 = Dropout(params["dropout_1"])(batch_norm_gru1) - - # Attention Mechanism (requires [query, value] inputs) - attention_output = Attention()([dropout_gru1, dropout_gru1]) - - # Concatenate attention output with the original GRU output - concat_layer = Concatenate()([dropout_gru1, attention_output]) - - # Bidirectional GRU Layer 2 (returns a single output) - bi_gru2_out = Bidirectional(GRU(params["lstm_units_1"], return_sequences=False, activation='tanh', unroll=True))(concat_layer) - batch_norm_gru2 = BatchNormalization()(bi_gru2_out) - dropout_gru2 = Dropout(params["dropout_2"])(batch_norm_gru2) - - # Dense layers with regularization - dense1 = Dense(params["dense_units_1"], activation='relu')(dropout_gru2) - batch_norm_d1 = BatchNormalization()(dense1) - dropout_d1 = Dropout(params["dropout_3"])(batch_norm_d1) - - dense2 = Dense(params["dense_units_2"], activation='relu')(dropout_d1) - batch_norm_d2 = BatchNormalization()(dense2) - dropout_d2 = Dropout(params["dropout_4"])(batch_norm_d2) - - # Output layer - outputs = Dense(1, activation='linear')(dropout_d2) - - # Create the model using the functional API - model = Model(inputs=inputs, outputs=outputs) - - # Compile the model - model.compile(optimizer=Adam(learning_rate=params["learning_rate"]), loss='huber', metrics=['mae', 'mse']) - - return model - - def train_symbol(symbol, params, trial_number, total_trials, best_weights="", best_epoch=None): - - if utils.interrupted: - logging.info("Interruption detected. Aborting training for current symbol.") - return None - - training = bool(best_weights) - recommender_dir = self.recommender_dir - model_filepath = recommender_dir / f"nsereco_model_{symbol}.pt" - - try: - if total_trials == 1: - best_weight_save_path = model_filepath - else: - best_weight_save_path = recommender_dir / f"trial_{trial_number}_of_{total_trials}_{symbol}_best_loss.pt" - df = stock_data_manager.df[stock_data_manager.df['SYMBOL'] == symbol].dropna().copy() - if df.empty or 'TARGET_PRICES' not in df.columns: - return None - - actual_vector = df.iloc[-1]["TARGET_PRICES"] - actual_price = actual_vector if isinstance(actual_vector, (float, int)) else actual_vector[0] - close_price = df.iloc[-1]["CLOSE_PRICE"] if "CLOSE_PRICE" in df.columns else None - - numeric_features = [f for f in self.feature_combos if f in df.columns] - feature_data = df[numeric_features].values - target_array = np.array([t[0] if isinstance(t, (list, np.ndarray)) else t for t in df['TARGET_PRICES']], dtype=np.float32).reshape(-1, 1) - - x_raw, y_raw = [], [] - seq_len = params["sequence_length"] - for i in range(len(feature_data) - seq_len): - x_seq = feature_data[i:i + seq_len] - y_seq = target_array[i + seq_len] - if np.isnan(y_seq).any(): - continue - x_raw.append(x_seq) - y_raw.append(y_seq[0]) - - if not x_raw or not y_raw: - logging.warning(f"No valid training samples for sequence_length={seq_len}") - return None - - x_raw = np.array(x_raw, dtype=np.float32) - y_raw = np.array(y_raw, dtype=np.float32).reshape(-1, 1) - - split = int(len(x_raw) * 0.8) - x_train_raw, x_val_raw = x_raw[:split], x_raw[split:] - y_train_raw, y_val_raw = y_raw[:split], y_raw[split:] - - feature_scaler = MinMaxScaler() - x_train_scaled = feature_scaler.fit_transform(x_train_raw.reshape(-1, x_train_raw.shape[-1])).reshape(x_train_raw.shape) - x_val_scaled = feature_scaler.transform(x_val_raw.reshape(-1, x_val_raw.shape[-1])).reshape(x_val_raw.shape) - - target_scaler = MinMaxScaler() - y_train_scaled = target_scaler.fit_transform(y_train_raw) - y_val_scaled = target_scaler.transform(y_val_raw) - - x_train_tensor = torch.tensor(x_train_scaled, dtype=torch.float32) - y_train_tensor = torch.tensor(y_train_scaled, dtype=torch.float32) - x_val_tensor = torch.tensor(x_val_scaled, dtype=torch.float32) - y_val_tensor = torch.tensor(y_val_scaled, dtype=torch.float32) - - train_loader = DataLoader( - TensorDataset(x_train_tensor, y_train_tensor), - batch_size=64, - shuffle=True, - drop_last=(len(x_train_tensor) > 64) - ) - val_loader = DataLoader(TensorDataset(x_val_tensor, y_val_tensor), batch_size=64) - - input_shape = (params["sequence_length"], len(numeric_features)) - model = LSTMModel(input_dim=input_shape[1], params=params) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.to(device) - - optimizer = TorchAdam(model.parameters(), lr=params["learning_rate"]) - loss_fn = HuberLoss() - - # Restore best weights if available and evaluate immediately - if best_weights and os.path.exists(best_weights): - try: - model.load_state_dict(torch.load(best_weights)) - logging.info(f"[{symbol}] Restored best weights from {best_weights} and skipping training loop.") - except Exception as e: - logging.warning(f"[{symbol}] Failed to load best weights: {e}") - - val_losses = [] - model.eval() - with torch.no_grad(): - for xb, yb in val_loader: - xb, yb = xb.to(device), yb.to(device) - pred = model(xb) - loss = loss_fn(pred, yb) - val_losses.append(loss.item()) - avg_val_loss = np.mean(val_losses) - best_val_loss = avg_val_loss - best_epoch_val = best_epoch if best_epoch is not None else 1 - epoch = best_epoch_val - 1 - - logger = PyTorchPredictionLogger(model, x_val_tensor, target_scaler, actual_price, symbol, trial_number, total_trials, recommender_dir) - MAElogger = PyTorchMAELogger(model, x_val_tensor, y_val_tensor, target_scaler, symbol, trial_number, total_trials, recommender_dir) - logger.update(epoch, device) - MAElogger.update(epoch, device) - - # Save best weights to final model path - torch.save(model.state_dict(), best_weight_save_path) - else: - logger = PyTorchPredictionLogger(model, x_val_tensor, target_scaler, actual_price, symbol, trial_number, total_trials, recommender_dir) - MAElogger = PyTorchMAELogger(model, x_val_tensor, y_val_tensor, target_scaler, symbol, trial_number, total_trials, recommender_dir) - - best_val_loss = float("inf") - best_epoch_val = -1 - early_stop_counter = 0 - history = {"train_loss": [], "val_loss": []} - - for epoch in range(params["epochs"]): - model.train() - train_losses = [] - for xb, yb in train_loader: - xb, yb = xb.to(device), yb.to(device) - optimizer.zero_grad() - pred = model(xb) - loss = loss_fn(pred, yb) - loss.backward() - optimizer.step() - train_losses.append(loss.item()) - - model.eval() - val_losses = [] - with torch.no_grad(): - for xb, yb in val_loader: - xb, yb = xb.to(device), yb.to(device) - pred = model(xb) - loss = loss_fn(pred, yb) - val_losses.append(loss.item()) - - avg_train_loss = np.mean(train_losses) - avg_val_loss = np.mean(val_losses) - history["train_loss"].append(avg_train_loss) - history["val_loss"].append(avg_val_loss) - - if total_trials == 1: - logging.info(f"[{symbol}] Epoch {epoch+1}: Train Loss={avg_train_loss:.4f}, Val Loss={avg_val_loss:.4f}") - - if avg_val_loss < best_val_loss: - best_val_loss = avg_val_loss - best_epoch_val = epoch + 1 - early_stop_counter = 0 - torch.save(model.state_dict(), best_weight_save_path) - else: - early_stop_counter += 1 - if early_stop_counter >= 20: - if total_trials == 1: - logging.info(f"[{symbol}] Early stopping at epoch {epoch+1}") - break - - logger.update(epoch, device) - MAElogger.update(epoch, device) - - model.load_state_dict(torch.load(best_weight_save_path)) - model.eval() - with torch.no_grad(): - val_pred = model(x_val_tensor.to(device)).cpu().numpy() - raw_pred = float(val_pred[-1][0]) - pred_price = target_scaler.inverse_transform([[raw_pred]])[0][0] - pred_error = ((pred_price - actual_price) / actual_price * 100) - pred_price = round(pred_price, 2) - actual_price = round(actual_price, 2) - - if total_trials == 1: - logging.info(f"[{symbol}] Final val_loss: {avg_val_loss:.4f} Best val_loss: {best_val_loss:.4f} Epochs: {epoch+1}") - logging.info(f"[{symbol}] CLOSE_PRICE (reference): {round(close_price, 2)}") - logging.info(f"[{symbol}] Actual TARGET_PRICE: {actual_price}") - logging.info(f"[{symbol}] Predicted TARGET_PRICE: {pred_price} Error: {pred_error:.4f}%") - logging.info(f"[{symbol}] Best prediction: {logger.best_pred_price:.2f} Min error: {logger.min_error:.4f}% at epoch: {logger.min_error_epoch}") - - if training: - try: - torch.save(model.state_dict(), model_filepath) - logging.info(f"[{symbol}] Saved trained model to {model_filepath}") - except Exception as e: - logging.warning(f"[{symbol}] Failed to save trained model: {e}") - - x_scaled_raw = feature_scaler.fit_transform(x_raw.reshape(-1, x_raw.shape[-1])).reshape(x_raw.shape) - x_raw_tensor = torch.tensor(x_scaled_raw, dtype=torch.float32).to(device) - with torch.no_grad(): - y_full_pred_scaled = model(x_raw_tensor).cpu().numpy() - y_full_pred = target_scaler.inverse_transform(y_full_pred_scaled).reshape(-1) - - - with matplotlib_lock: - # Temporarily restore original warnings.warn to prevent langchain's filtered_warn crash with matplotlib - orig_warn = warnings.warn - if hasattr(sys, 'original_warn'): - warnings.warn = sys.original_warn - try: - sns.set_style("darkgrid", {"axes.facecolor": "black", "grid.color": ".5"}) - plt.figure(figsize=(19.2, 10.8), dpi=100) - plt.plot(y_raw, label='Actual TARGET_PRICES', color='#00FF41', linestyle='-', marker='o', ms=4, markevery=max(1, len(y_raw)//50)) - plt.plot(y_full_pred, label='Predicted TARGET_PRICES', color='#FFD700', linestyle='--', marker='x', ms=4, markevery=max(1, len(y_full_pred)//50)) - - plt.title(f"{symbol} Target Prices Backtest", color='white', fontsize=16) - plt.xlabel('Time Steps', color='white', fontsize=12) - plt.ylabel('TARGET_PRICES', color='white', fontsize=12) - plt.legend(facecolor='black', edgecolor='white', labelcolor='white') - plt.xticks(color='white') - plt.yticks(color='white') - for spine in plt.gca().spines.values(): - spine.set_color('white') - - # Overlay best hyperparameters (bottom-right) and best test results (top-left) - best_params_text = ( - "Best Hyperparameters:\n" + \ - "\n".join([f" {k}: {v:.6f}" if isinstance(v, float) else f" {k}: {v}" for k, v in params.items()]) - ) - best_results_text = ( - "Best Test Results:\n" - f" Best Loss: {best_val_loss:.4f}\n" - f" Epochs Ran: {epoch+1}\n" - f" Actual Price: {actual_price:.2f}\n" - f" Predicted Price: {pred_price:.2f}\n" - f" Pred Error: {pred_error:.4f}%\n" - f" Best Pred: {logger.best_pred_price:.2f}\n" - f" Min Error: {logger.min_error:.4f}%\n" - f" Best Epoch: {logger.min_error_epoch}" - ) - ax = plt.gca() - props = dict(boxstyle='round,pad=0.5', facecolor='#111111', alpha=0.8, edgecolor='#555555') - ax.text(0.02, 0.95, best_results_text, transform=ax.transAxes, fontsize=11, color='#00FF41', - verticalalignment='top', bbox=props, family='monospace') - ax.text(0.98, 0.05, best_params_text, transform=ax.transAxes, fontsize=11, color='#FFD700', - verticalalignment='bottom', horizontalalignment='right', bbox=props, family='monospace') - - vn = 1 - backtest_filepath = recommender_dir / f"nsereco_backtest_{symbol}_v{vn}.png" - while backtest_filepath.exists(): - vn += 1 - backtest_filepath = recommender_dir / f"nsereco_backtest_{symbol}_v{vn}.png" - - plt.savefig(backtest_filepath, facecolor='black') - plt.close() - logging.info(f"[{symbol}] Saved backtest plot to {backtest_filepath}") - finally: - warnings.warn = orig_warn - - return { - "symbol": symbol, - "final_loss": avg_val_loss, - "final_mae": np.mean(np.abs(y_val_scaled - val_pred)), - "best_loss": best_val_loss, - "epochs_ran": epoch + 1, - "actual_price": actual_price, - "pred_price": pred_price, - "pred_error": pred_error, - "best_pred": logger.best_pred_price, - "best_error": logger.min_error, - "best_epoch": logger.min_error_epoch, - "weights_filepath": str(best_weight_save_path) if best_weight_save_path else str(model_filepath), - "best_key": logger.best_key - } - - except Exception as e: - logging.warning(f"Training failed for {symbol}: {e}") - return None - finally: - if total_trials == 1: - try: - if logger.weights_filepath and os.path.exists(logger.weights_filepath): - os.remove(logger.weights_filepath) - if MAElogger.weights_filepath and os.path.exists(MAElogger.weights_filepath): - os.remove(MAElogger.weights_filepath) - except Exception as e: - pass - - # Use only NIFTY 50 stocks for tuning as per requirement - symbols = [ - 'ADANIENT', 'ADANIPORTS', 'APOLLOHOSP', 'ASIANPAINT', 'AXISBANK', 'BAJAJ-AUTO', - 'BAJFINANCE', 'BAJAJFINSV', 'BHARTIARTL', 'BPCL', 'BRITANNIA', 'CIPLA', 'COALINDIA', - 'DIVISLAB', 'DRREDDY', 'EICHERMOT', 'GRASIM', 'HCLTECH', 'HDFCBANK', 'HDFCLIFE', - 'HEROMOTOCO', 'HINDALCO', 'HINDUNILVR', 'ICICIBANK', 'INDUSINDBK', 'INFY', 'IOC', - 'ITC', 'JSWSTEEL', 'KOTAKBANK', 'LT', 'LTIM', 'M&M', 'MARUTI', 'NESTLEIND', - 'NTPC', 'ONGC', 'POWERGRID', 'RELIANCE', 'SBILIFE', 'SBIN', 'SUNPHARMA', 'TCS', - 'TATACONSUM', 'TATAMOTORS', 'TATASTEEL', 'TECHM', 'TITAN', 'ULTRACEMCO', 'WIPRO' - ] - - #symbols1 = [ADANIENT,ADANIPORTS,APOLLOHOSP,ASIANPAINT,AXISBANK,BAJAJ-AUTO,BAJFINANCE,BAJAJFINSV,BHARTIARTL,BPCL] - #symbols2 = [BRITANNIA,CIPLA,COALINDIA,DIVISLAB,DRREDDY,EICHERMOT,GRASIM,HCLTECH,HDFCBANK,HDFCLIFE] - #symbols3 = [HEROMOTOCO,HINDALCO,HINDUNILVR,ICICIBANK,INDUSINDBK,INFY,IOC,ITC,JSWSTEEL,KOTAKBANK] - #symbols4 = [LT,LTIM,M&M,MARUTI,NESTLEIND,NTPC,ONGC,POWERGRID,RELIANCE,SBILIFE] - #symbols5 = [SBIN,SUNPHARMA,TCS,TATACONSUM,TATAMOTORS,TATASTEEL,TECHM,TITAN,ULTRACEMCO,WIPRO] - - # Allow CLI override for symbol list - argv_symbols = [] - if len(sys.argv) > 1: - argv_symbols = [sym.strip().upper() for sym in sys.argv[1].split(',') if sym.strip()] - logging.info(f"Command line symbols for tuning: {argv_symbols}") - - if tuning_symbols: - symbols = tuning_symbols - elif argv_symbols: - symbols = argv_symbols - else: - symbols = symbols[:num_symbols] - - from nsereco.task_manager import task_manager - if task_manager.total_count <= 1: - task_manager.update_progress(0, len(symbols)) - - def tune_single_symbol(idx, sym): - if utils.interrupted: - logging.info(f"[{sym}] Interruption detected. Skipping...") - return - - # Skip tuning if already completed - tuner_path = Path(self.recommender_dir) / f"nsereco_tuner_{sym}.json" - if tuner_path.exists(): - try: - with open(tuner_path, "r") as f: - content = f.read().strip() - if content: - existing_data = json.loads(content) - param_combos = existing_data.get("all_param_combinations", {}) - completed_trials = sum(1 for entry in param_combos.values() if entry.get("trial_status") == "COMPLETED") - if completed_trials >= num_trials: - logging.info(f"[{sym}] Skipping trial as already completed {num_trials} trials...") - if task_manager.total_count <= 1 or len(symbols) > 1: - task_manager.increment_progress() - return - except Exception as e: - logging.warning(f"Failed to parse tuner file for {sym}: {e}. Proceeding...") - - # Define Optuna objective function - min_val_loss = float('inf') - min_trial_number = -1 - - def objective(trial): - if utils.interrupted: - raise optuna.TrialPruned() # Gracefully handle interruption - - nonlocal min_val_loss, min_trial_number - - # Define hyperparameter search space - params = { - "epochs": trial.suggest_int("epochs", 60, 100), - "learning_rate": trial.suggest_float("learning_rate", 1e-5, 1e-2, log=True), - "lstm_units_1": trial.suggest_int("lstm_units_1", 32, 128, step=16), - "dropout_1": trial.suggest_float("dropout_1", 0.1, 0.5), - "lstm_units_2": trial.suggest_int("lstm_units_2", 16, 96, step=16), - "dropout_2": trial.suggest_float("dropout_2", 0.1, 0.5), - "dense_units_1": trial.suggest_int("dense_units_1", 16, 128, step=16), - "dropout_3": trial.suggest_float("dropout_3", 0.1, 0.5), - "dense_units_2": trial.suggest_int("dense_units_2", 16, 96, step=16), - "dropout_4": trial.suggest_float("dropout_4", 0.1, 0.5), - "sequence_length": trial.suggest_int("sequence_length", 16, 64, step=8) - } - - # Evaluate trial parameters - result = train_symbol(sym, params, trial_number=trial.number, total_trials=num_trials) - - if result: - _save_tuning_history( - trial_number=trial.number, - total_trials=num_trials, - params=params, - best_loss=result["best_loss"], - final_loss=result["final_loss"], - final_mae=result["final_mae"], - epochs_ran=result["epochs_ran"], - symbol=[result["symbol"]], - target_price=result["actual_price"], - pred_price=result["pred_price"], - pred_error=result["pred_error"], - best_pred=result["best_pred"], - best_error=result["best_error"], - best_epoch=result["best_epoch"], - weights_filepath=result["weights_filepath"], - best_key="NO" - ) - - if result["best_loss"] < min_val_loss: - min_val_loss = result["best_loss"] - min_trial_number = trial.number - - logging.info(f"[{sym}] Trial {trial.number} completed. Best Val Loss: {result['best_loss']:.4f}, Epochs: {result['epochs_ran']}") - return result["best_loss"] - else: - logging.warning(f"[{sym}] Trial {trial.number} failed...") - return 1.0 - - # 1. Define the pruner (stops trials whose performance is below the median of previous trials) - pruner = optuna.pruners.MedianPruner( - n_startup_trials=5, # Number of trials before pruning starts - n_warmup_steps=10 # Number of epochs to run before considering pruning - ) - - # 2. Create the study with the pruner - study = optuna.create_study( - direction="minimize", # Minimize the 'val_loss' or 'val_mape' - pruner=pruner - ) - - # 3. Run the optimization with the allocated number of threads per symbol - study.optimize(objective, n_trials=num_trials, n_jobs=jobs_per_symbol) - - weights_filepath = "" - best_params = {} - # Mark best trial in tuner file - if min_trial_number >= 0 and tuner_path.exists(): - try: - with open(tuner_path, "r") as f: - content = f.read().strip() - if content: - data = json.loads(content) - trial_key = f"trial_{min_trial_number}_of_{num_trials}" - if "all_param_combinations" in data and trial_key in data["all_param_combinations"]: - data["all_param_combinations"][trial_key]["best_key"] = "YES" - best_params = data["all_param_combinations"][trial_key]["used_params"] - weights_filepath = data["all_param_combinations"][trial_key].get("weights_filepath", "") - with open(tuner_path, "w") as fw: - json.dump(data, fw, indent=2) - logging.info(f"[{sym}] Marked trial {min_trial_number} as best_key=YES in tuner file.") - except Exception as e: - logging.warning(f"Failed to update best_key in tuner file for {sym}: {e}") - - # Save best hyperparameters - if min_trial_number >= 0 and tuner_path.exists(): - best_params = best_params.copy() - for k in ["final_mae", "best_loss", "final_loss", "epochs_ran"]: - if k in best_params: - del best_params[k] - best_params["avg_loss"] = min_val_loss - best_params["avg_accuracy"] = -1.0 - else: - try: - best_params = study.best_params.copy() - best_params["avg_loss"] = study.best_value - best_params["avg_accuracy"] = -1.0 - except ValueError: - logging.warning("No successful trials completed in this Optuna run. Retaining existing best hyperparameters.") - best_params = self.best_hyperparameters.copy() - - with self.progress_lock: - self.best_hyperparameters = best_params - - # Save into symbol_master_tracker.json (thread-safe lock) - with self.progress_lock: - tracker_path = Path(self.recommender_dir).parent / 'symbol_master_tracker.json' - if tracker_path.exists(): - try: - with open(tracker_path, 'r') as f: - tracker_data = json.load(f) - if sym in tracker_data: - tracker_data[sym]["optuna_best_parameters"] = best_params - tracker_data[sym]["tuning_status"] = "COMPLETED" - with open(tracker_path, 'w') as f: - json.dump(tracker_data, f, indent=4) - logging.info(f"[{sym}] Updated symbol_master_tracker.json with best_params and status COMPLETED.") - except Exception as e: - logging.warning(f"Failed to update symbol_master_tracker.json for {sym}: {e}") - - logging.info(f"Optuna tuning complete for {sym}.") - logging.info(f"Best Parameters for {sym}:\n{json.dumps(best_params, indent=2)}") - - # Train final model with best parameters - logging.info(f"[{sym}] Training final model with best hyperparameters from Optuna tuning...") - try: - best_epoch = None - if min_trial_number >= 0 and tuner_path.exists(): - try: - with open(tuner_path, "r") as f: - content = f.read().strip() - if content: - data = json.loads(content) - trial_key = f"trial_{min_trial_number}_of_{num_trials}" - if "all_param_combinations" in data and trial_key in data["all_param_combinations"]: - best_epoch = data["all_param_combinations"][trial_key].get("best_epoch") - except Exception as e: - pass - - result = train_symbol(sym, best_params, trial_number=0, total_trials=1, best_weights=weights_filepath, best_epoch=best_epoch) - if result: - _save_tuning_history( - trial_number=0, - total_trials=1, - params=best_params, - best_loss=result["best_loss"], - final_loss=result["final_loss"], - final_mae=result["final_mae"], - epochs_ran=result["epochs_ran"], - symbol=[result["symbol"]], - target_price=result["actual_price"], - pred_price=result["pred_price"], - pred_error=result["pred_error"], - best_pred=result["best_pred"], - best_error=result["best_error"], - best_epoch=result["best_epoch"], - weights_filepath=result["weights_filepath"], - best_key=result["best_key"], - status="TRAINED" - ) - logging.info(f"[{sym}] Final model training complete with best hyperparameters...") - finally: - # Clean up intermediate trial weight files - try: - trial_files = glob.glob(str(Path(self.recommender_dir) / f"trial_*_{sym}*.pt")) - for tf_file in trial_files: - if os.path.exists(tf_file): - os.remove(tf_file) - logging.info(f"[{sym}] Cleaned up {len(trial_files)} intermediate trial weight files.") - except Exception as e: - logging.warning(f"[{sym}] Failed to clean up intermediate trial weights: {e}") - - if task_manager.total_count <= 1 or len(symbols) > 1: - task_manager.increment_progress() - - from concurrent.futures import ThreadPoolExecutor - pool_workers = min(max_workers, len(symbols)) - jobs_per_symbol = max(1, max_workers // pool_workers) - logging.info(f"Running Optuna tuning for {len(symbols)} symbols in parallel with {pool_workers} workers (each using {jobs_per_symbol} threads)...") - with ThreadPoolExecutor(max_workers=pool_workers) as executor: - futures = [executor.submit(tune_single_symbol, idx, sym) for idx, sym in enumerate(symbols)] - for future in futures: - try: - future.result() - except Exception as e: - logging.error(f"Error during parallel symbol tuning: {e}") - - def analyze_tuner_data(self, symbol=None): - """ - Loads tuner history and displays trials in a Rich table, - sorted by final loss ascending (best first). - """ - console = Console() - if not symbol: - symbol = input("Enter symbol to analyze (e.g. SBIN): ").strip().upper() - else: - symbol = symbol.strip().upper() - if not symbol: - console.print("[bold red]No symbol entered.[/bold red]") - return - - tuner_path = Path(self.recommender_dir) / f"nsereco_tuner_{symbol}.json" - - if not tuner_path.exists(): - console.print(f"[bold red]Tuner file for {symbol} not found at {tuner_path}.[/bold red]") - return - - try: - with open(tuner_path, "r") as f: - data = json.load(f) - except Exception as e: - console.print(f"[bold red]Failed to load tuner file:[/bold red] {e}") - return - - all_trials = data.get("all_param_combinations", {}) - filtered = { - k: v for k, v in all_trials.items() - if v.get("trial_status") in ["COMPLETED", "TRAINED"] - } - - if not filtered: - console.print("[yellow]No completed trials found in tuner data.[/yellow]") - return - - # Sort by ascending final_loss - sorted_trials = sorted( - filtered.items(), - key=lambda x: x[1]["used_params"].get("final_loss", float('inf')), - reverse=False - ) - - # Create Rich table - table = Table(title=f"Tuner Trials for {symbol} (Sorted by Loss)", show_lines=True, box=box.SIMPLE_HEAVY) - table.add_column("Parameter", style="bold cyan", no_wrap=True) - - for trial_key, _ in sorted_trials: - table.add_column(trial_key, style="bold green", no_wrap=True) - - param_keys = [ - "epochs", "epochs_ran", "learning_rate", "lstm_units_1", "dropout_1", - "lstm_units_2", "dropout_2", "dense_units_1", "dropout_3", - "dense_units_2", "dropout_4", "sequence_length", - "final_mae", "best_loss", "final_loss" - ] - - for param in param_keys: - row = [param] - for _, trial_data in sorted_trials: - val = trial_data["used_params"].get(param, "") - row.append(f"{val:.4f}" if isinstance(val, float) else str(val)) - table.add_row(*row) - - console.print(table) - diff --git a/nsereco/routes.py b/nsereco/routes.py deleted file mode 100644 index 1804325b2b3184f280bfe6c732aac23d421bf367..0000000000000000000000000000000000000000 --- a/nsereco/routes.py +++ /dev/null @@ -1,260 +0,0 @@ -from flask import Blueprint, render_template, jsonify, request -import logging -import gc - -from .data import StockData -from .recommender import SymbolRecommender -from .task_manager import task_manager - -bp = Blueprint('main', __name__) - -# Global instances (initialized lazily or at startup) -_stock_data_manager = None -_recommender = None - -def get_managers(): - global _stock_data_manager, _recommender - if _stock_data_manager is None: - logging.info("Initializing StockData...") - _stock_data_manager = StockData() - - # Check if features need to be created/loaded - if not SymbolRecommender.DL_FEATURES: - selected_features, dropped_features = _stock_data_manager.select_features() - SymbolRecommender.DL_FEATURES = selected_features - SymbolRecommender.dropped_features = dropped_features - - expected_features = SymbolRecommender.DL_FEATURES - if 'NIFTY50' not in _stock_data_manager.df['SYMBOL'].values: - expected_features = [f for f in expected_features if not f.startswith('NIFTY50')] - - missing_features = [f for f in expected_features if f not in _stock_data_manager.df.columns] - if missing_features or 'TARGET_PRICES' not in _stock_data_manager.df.columns: - logging.info("Features or target prices missing from loaded data. Generating features and targets...") - _stock_data_manager.create_features() - _stock_data_manager.create_target() - try: - _stock_data_manager.save_extract_file() - except Exception as e: - logging.error(f"Failed to save updated extract file: {e}") - - # Filter DL_FEATURES to only include columns that were successfully generated/loaded - SymbolRecommender.DL_FEATURES = [f for f in SymbolRecommender.DL_FEATURES if f in _stock_data_manager.df.columns] - - logging.info("Initializing SymbolRecommender...") - _recommender = SymbolRecommender( - recommender_dir=_stock_data_manager.recommender_dir, - training_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_recommender.json', - tuning_parameters_path=_stock_data_manager.data_dir / 'nsereco_parameters.json', - nifty_profit_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_nifty_profit.json', - all_profit_tracker_path=_stock_data_manager.data_dir / 'nse_dashboard_all_profit.json' - ) - return _stock_data_manager, _recommender - - -@bp.route('/') -def index(): - return render_template('index.html') - -@bp.route('/api/status') -def status(): - return jsonify(task_manager.get_status()) - -@bp.route('/api/initialize', methods=['POST']) -def initialize(): - success, msg = task_manager.start_task("System Initialization", get_managers) - return jsonify({"success": success, "message": msg}) - -@bp.route('/api/stop', methods=['POST']) -def stop(): - success, msg = task_manager.stop_task() - return jsonify({"success": success, "message": msg}) - -@bp.route('/api/kill', methods=['POST']) -def kill(): - # Hard kill the backend process - import threading - import os - def shutdown(): - import time - import logging - import os - import signal - time.sleep(1.5) - logging.warning("Graceful shutdown message due to UI kill switch") - os.kill(os.getpid(), signal.SIGINT) - threading.Thread(target=shutdown, daemon=True).start() - return jsonify({"success": True, "message": "Application shutting down."}) - -import os -from flask import send_from_directory - -@bp.route('/api/graphs') -def list_graphs(): - sm, _ = get_managers() - if not sm or not sm.recommender_dir: - return jsonify([]) - recommender_dir = sm.recommender_dir - graphs = [] - if os.path.exists(recommender_dir): - graphs = [f for f in os.listdir(recommender_dir) if f.startswith('nsereco_backtest_') and f.endswith('.png')] - return jsonify(sorted(graphs)) - -@bp.route('/api/graphs/') -def serve_graph(filename): - sm, _ = get_managers() - recommender_dir = sm.recommender_dir - return send_from_directory(recommender_dir, filename) - -@bp.route('/api/tune_selected', methods=['POST']) -def tune_selected(): - data = request.json - num_stocks = int(data.get('num_stocks', 20)) - worker_threads = int(data.get('worker_threads', 8)) - - def task(): - from nsereco.agent_workflow import run_agent_workflow - run_agent_workflow('tune', num_stocks, worker_threads) - import gc - gc.collect() - - success, msg = task_manager.start_task(f"Agentic Hyperparameter tuning ({num_stocks} stocks, {worker_threads} threads)", task) - return jsonify({"success": success, "message": msg}) - -@bp.route('/api/train_all', methods=['POST']) -def train_all(): - data = request.json or {} - num_stocks = int(data.get('num_stocks', 20)) - worker_threads = int(data.get('worker_threads', 8)) - - def task(): - from nsereco.agent_workflow import run_agent_workflow - run_agent_workflow('train', num_stocks, worker_threads) - import gc - gc.collect() - - success, msg = task_manager.start_task(f"Agentic Model training ({num_stocks} stocks, {worker_threads} threads)", task) - return jsonify({"success": success, "message": msg}) - -@bp.route('/api/optuna_finder', methods=['POST']) -def optuna_finder(): - data = request.json - num_trials = int(data.get('num_trials', 30)) - num_symbols = int(data.get('num_symbols', 1)) - - def task(): - sm, rec = get_managers() - rec.run_optuna_tuning(sm, num_trials=num_trials, num_symbols=num_symbols) - gc.collect() - - success, msg = task_manager.start_task("Optuna hyperparameters finder", task) - return jsonify({"success": success, "message": msg}) - -@bp.route('/api/analyze', methods=['POST']) -def analyze(): - data = request.json - symbol = data.get('symbol', 'SBIN') - - def task(): - sm, rec = get_managers() - rec.analyze_tuner_data(symbol=symbol) - - success, msg = task_manager.start_task(f"Tuner data analysis for {symbol}", task) - return jsonify({"success": success, "message": msg}) - - -@bp.route('/api/metrics') -def get_metrics(): - import json - import re - import os - from pathlib import Path - - # 1. Total symbols - global _stock_data_manager - total_symbols_list = [] - if _stock_data_manager is not None and _stock_data_manager.df is not None: - total_symbols_list = sorted(_stock_data_manager.df['SYMBOL'].dropna().unique().astype(str).tolist()) - else: - try: - data_dir = Path("/source/python/code/nsedata/analysis/processed") - symbol_list_path = data_dir / 'nse_dashboard_symbollist.json' - if symbol_list_path.exists(): - with open(symbol_list_path, 'r') as f: - symbol_dict = json.load(f) - total_symbols_list = sorted(list(symbol_dict.keys())) - except Exception: - pass - - # 2. Tuned symbols - tuned_symbols = set() - for d in ["recommender", "recommender_torch"]: - path = Path("/source/python/code/nsedata/analysis/processed") / d - if path.exists(): - for f in os.listdir(path): - if f.startswith('nsereco_tuner_') and f.endswith('.json'): - sym = f[len('nsereco_tuner_'):-len('.json')] - tuned_symbols.add(sym) - tuned_symbols_list = sorted(list(tuned_symbols)) - - # 3. Trained symbols - trained_symbols_list = [] - global _recommender - if _recommender is not None: - trained_symbols_list = [sym for sym, val in _recommender.training_tracker_data.items() if val.get("training", False)] - else: - try: - recommender_json_path = Path("/source/python/code/nsedata/analysis/processed/nse_dashboard_recommender.json") - if recommender_json_path.exists(): - with open(recommender_json_path, 'r') as f: - rec_data = json.load(f) or {} - trained_symbols_list = [sym for sym, val in rec_data.items() if isinstance(val, dict) and val.get("training", False)] - except Exception: - pass - trained_symbols_list = sorted(trained_symbols_list) - - # 4. Backtest graphs symbols - backtest_symbols = set() - for d in ["recommender", "recommender_torch"]: - path = Path("/source/python/code/nsedata/analysis/processed") / d - if path.exists(): - for f in os.listdir(path): - if f.startswith('nsereco_backtest_') and f.endswith('.png'): - name = f[len('nsereco_backtest_'):-len('.png')] - match = re.match(r'^([A-Za-z0-9\-&]+)(?:_v\d+)?$', name) - if match: - backtest_symbols.add(match.group(1)) - else: - backtest_symbols.add(name.split('_')[0]) - backtest_symbols_list = sorted(list(backtest_symbols)) - - # 5. Tuning / Training in progress - status_data = task_manager.get_status() - is_running = status_data["is_running"] - status_str = status_data["status"].lower() - tuning_in_progress = is_running and ("tune" in status_str or "optuna" in status_str or "finder" in status_str) - training_in_progress = is_running and ("train" in status_str) - - return jsonify({ - "total_symbols": { - "count": len(total_symbols_list), - "symbols": total_symbols_list - }, - "tuned_symbols": { - "count": len(tuned_symbols_list), - "symbols": tuned_symbols_list - }, - "trained_symbols": { - "count": len(trained_symbols_list), - "symbols": trained_symbols_list - }, - "backtest_graphs": { - "count": len(backtest_symbols_list), - "symbols": backtest_symbols_list - }, - "tuning_in_progress": tuning_in_progress, - "training_in_progress": training_in_progress, - "completed_count": status_data["completed_count"], - "total_count": status_data["total_count"] - }) - diff --git a/nsereco/static/css/style.css b/nsereco/static/css/style.css deleted file mode 100644 index fa9b8d458ee0c572001f1a1325b99f73829f8994..0000000000000000000000000000000000000000 --- a/nsereco/static/css/style.css +++ /dev/null @@ -1,377 +0,0 @@ -:root { - --bg-dark: #0f172a; - --card-bg: rgba(30, 41, 59, 0.7); - --primary-color: #3b82f6; - --primary-hover: #2563eb; - --text-main: #f8fafc; - --text-muted: #94a3b8; - --success: #10b981; - --warning: #f59e0b; - --danger: #ef4444; -} - -* { - box-sizing: border-box; - margin: 0; - padding: 0; -} - -body { - font-family: 'Inter', sans-serif; - background-color: var(--bg-dark); - color: var(--text-main); - line-height: 1.6; - min-height: 100vh; - background: linear-gradient(135deg, #0f172a 0%, #1e1b4b 100%); -} - -.app-container { - max-width: 1400px; - margin: 0 auto; - padding: 2rem; - display: flex; - flex-direction: column; - gap: 2rem; - height: 100vh; -} - -.app-header { - display: flex; - justify-content: space-between; - align-items: center; - border-bottom: 1px solid rgba(255, 255, 255, 0.1); - padding-bottom: 1rem; -} - -.app-header h1 { - font-weight: 600; - letter-spacing: -0.5px; - background: -webkit-linear-gradient(#60a5fa, #a78bfa); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; -} - -.status-indicator { - display: flex; - align-items: center; - gap: 0.5rem; - font-weight: 600; - background: rgba(0, 0, 0, 0.3); - padding: 0.5rem 1rem; - border-radius: 20px; -} - -.status-dot { - width: 10px; - height: 10px; - border-radius: 50%; - background-color: var(--success); - box-shadow: 0 0 8px var(--success); - transition: background-color 0.3s ease; -} - -.status-dot.running { - background-color: var(--warning); - box-shadow: 0 0 8px var(--warning); - animation: pulse 1.5s infinite; -} - -@keyframes pulse { - 0% { transform: scale(0.95); opacity: 0.7; } - 50% { transform: scale(1.1); opacity: 1; } - 100% { transform: scale(0.95); opacity: 0.7; } -} - -.dashboard-grid { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); - gap: 1.5rem; -} - -.glassmorphism { - background: var(--card-bg); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); - border: 1px solid rgba(255, 255, 255, 0.08); - border-radius: 12px; - padding: 1.5rem; - box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2); - transition: transform 0.2s ease, box-shadow 0.2s ease; -} - -.glassmorphism:hover { - transform: translateY(-2px); - box-shadow: 0 15px 35px rgba(0, 0, 0, 0.3); -} - -.card h2 { - font-size: 1.25rem; - margin-bottom: 0.75rem; - color: #e2e8f0; -} - -.card p { - color: var(--text-muted); - font-size: 0.9rem; - margin-bottom: 1.5rem; -} - -.form-group { - margin-bottom: 1rem; -} - -.form-group label { - display: block; - font-size: 0.85rem; - color: var(--text-muted); - margin-bottom: 0.25rem; -} - -.form-group input { - width: 100%; - background: rgba(0, 0, 0, 0.2); - border: 1px solid rgba(255, 255, 255, 0.1); - color: var(--text-main); - padding: 0.5rem 0.75rem; - border-radius: 6px; - font-family: inherit; -} - -.form-group input:focus { - outline: none; - border-color: var(--primary-color); -} - -.btn { - width: 100%; - padding: 0.75rem; - border: none; - border-radius: 6px; - font-family: inherit; - font-weight: 600; - cursor: pointer; - transition: all 0.2s ease; -} - -.btn.primary { - background-color: var(--primary-color); - color: white; -} - -.btn.primary:hover { - background-color: var(--primary-hover); -} - -.btn:disabled { - opacity: 0.5; - cursor: not-allowed; -} - -.console-section { - flex-grow: 1; - display: flex; - flex-direction: column; - min-height: 300px; - background: rgba(0, 0, 0, 0.5); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 12px; - overflow: hidden; -} - -.console-header { - background: rgba(255, 255, 255, 0.05); - padding: 0.75rem 1.5rem; - border-bottom: 1px solid rgba(255, 255, 255, 0.1); -} - -.console-header h3 { - font-size: 1rem; - font-weight: 600; -} - -.console-window { - padding: 1rem 1.5rem; - overflow-y: auto; - font-family: 'Consolas', 'Courier New', monospace; - font-size: 0.85rem; - color: #a3e635; /* Terminal green */ - line-height: 1.5; - flex-grow: 1; -} - -.console-line { - margin-bottom: 2px; - word-wrap: break-word; -} - -.console-actions { - display: flex; - gap: 0.5rem; -} - -.btn.secondary { - background-color: #64748b; - color: white; -} - -.btn.secondary:hover { - background-color: #475569; -} - -.btn.warning { - background-color: var(--warning); - color: white; -} - -.btn.warning:hover { - background-color: #d97706; -} - -.btn.danger { - background-color: var(--danger); - color: white; -} - -.btn.danger:hover { - background-color: #b91c1c; -} - -/* Modal CSS */ -.modal { - display: none; - position: fixed; - z-index: 1000; - left: 0; - top: 0; - width: 100%; - height: 100%; - overflow: auto; - background-color: rgba(0,0,0,0.6); - backdrop-filter: blur(5px); -} - -.modal-content { - background-color: var(--bg-dark); - margin: 5% auto; - padding: 20px; - border: 1px solid rgba(255, 255, 255, 0.1); - width: 80%; - max-width: 900px; - border-radius: 12px; - position: relative; -} - -.close-btn { - color: var(--text-muted); - float: right; - font-size: 28px; - font-weight: bold; - cursor: pointer; -} - -.close-btn:hover, -.close-btn:focus { - color: white; - text-decoration: none; -} - -select#graph-select { - width: 100%; - background: rgba(0, 0, 0, 0.2); - border: 1px solid rgba(255, 255, 255, 0.1); - color: var(--text-main); - padding: 0.5rem 0.75rem; - border-radius: 6px; - font-family: inherit; - margin-bottom: 1rem; -} - -/* Metrics Summary Grid and Cards */ -.metrics-grid { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); - gap: 1rem; - margin-bottom: 1rem; -} - -.metric-card { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - padding: 1rem; - text-align: center; -} - -.metric-label { - font-size: 0.8rem; - color: var(--text-muted); - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.5px; - margin-bottom: 0.5rem; -} - -.metric-value { - font-size: 1.8rem; - font-weight: 600; - color: #3b82f6; - background: -webkit-linear-gradient(#60a5fa, #3b82f6); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; -} - -.metric-value.clickable { - cursor: pointer; - text-decoration: underline dotted; - text-underline-offset: 4px; - transition: transform 0.2s ease, opacity 0.2s ease; -} - -.metric-value.clickable:hover { - transform: scale(1.05); - opacity: 0.85; -} - -.metric-value.active-status { - color: var(--warning); - background: -webkit-linear-gradient(#fcd34d, #f59e0b); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - animation: statusPulse 2s infinite; -} - -@keyframes statusPulse { - 0% { opacity: 0.7; } - 50% { opacity: 1; } - 100% { opacity: 0.7; } -} - -/* Symbols Grid in Modal */ -.symbols-list-grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(100px, 1fr)); - gap: 0.75rem; - margin-top: 1.5rem; - max-height: 400px; - overflow-y: auto; - padding-right: 0.5rem; -} - -.symbol-badge { - background: rgba(255, 255, 255, 0.05); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 6px; - padding: 0.35rem 0.5rem; - text-align: center; - font-family: monospace; - font-size: 0.85rem; - color: var(--text-main); - transition: background 0.2s ease, border-color 0.2s ease; -} - -.symbol-badge:hover { - background: rgba(59, 130, 246, 0.2); - border-color: var(--primary-color); -} - diff --git a/nsereco/static/js/app.js b/nsereco/static/js/app.js deleted file mode 100644 index 67bd715aa59ce0af5465e3e91df60cf251d34830..0000000000000000000000000000000000000000 --- a/nsereco/static/js/app.js +++ /dev/null @@ -1,261 +0,0 @@ -document.addEventListener('DOMContentLoaded', () => { - const statusText = document.getElementById('system-status'); - const statusDot = document.querySelector('.status-dot'); - const consoleOutput = document.getElementById('console-output'); - - let isTaskRunning = false; - let autoScroll = true; - - // Detect if user scrolls up to disable autoscroll - consoleOutput.addEventListener('scroll', () => { - const isAtBottom = consoleOutput.scrollHeight - consoleOutput.scrollTop <= consoleOutput.clientHeight + 10; - autoScroll = isAtBottom; - }); - - function setButtonsDisabled(disabled) { - document.querySelectorAll('.btn:not(#btn-stop):not(#btn-view-graphs):not(#btn-kill)').forEach(btn => { - btn.disabled = disabled; - }); - document.getElementById('btn-stop').disabled = !disabled; - } - - function appendLogs(logs) { - if (!logs || logs.length === 0) return; - - // We only append new logs in a real app, but for simplicity we'll just rewrite it if it's small, - // or since task_manager clears it on start, we just set innerHTML. - consoleOutput.innerHTML = logs.map(line => `
${line}
`).join(''); - - if (autoScroll) { - consoleOutput.scrollTop = consoleOutput.scrollHeight; - } - } - - async function checkStatus() { - try { - const res = await fetch('/api/status'); - const data = await res.json(); - - isTaskRunning = data.is_running; - statusText.textContent = data.status; - - if (isTaskRunning) { - statusDot.classList.add('running'); - setButtonsDisabled(true); - } else { - statusDot.classList.remove('running'); - setButtonsDisabled(false); - } - - appendLogs(data.logs); - } catch (err) { - console.error('Error fetching status:', err); - statusText.textContent = 'Connection Error'; - statusDot.classList.remove('running'); - } - } - - // Poll status every 2 seconds - setInterval(checkStatus, 2000); - checkStatus(); - - async function postAction(endpoint, payload = {}) { - if (isTaskRunning) return; - - try { - const res = await fetch(endpoint, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(payload) - }); - const data = await res.json(); - if (!data.success) { - alert(data.message); - } else { - checkStatus(); - } - } catch (err) { - alert('Failed to start task: ' + err.message); - } - } - - document.getElementById('btn-init').addEventListener('click', () => { - postAction('/api/initialize'); - }); - - document.getElementById('btn-tune').addEventListener('click', () => { - const numStocks = parseInt(document.getElementById('tune-stocks').value, 10); - const numThreads = parseInt(document.getElementById('tune-threads').value, 10); - postAction('/api/tune_selected', { num_stocks: numStocks, worker_threads: numThreads }); - }); - - document.getElementById('btn-train').addEventListener('click', () => { - const numStocks = parseInt(document.getElementById('train-stocks').value, 10); - const numThreads = parseInt(document.getElementById('train-threads').value, 10); - postAction('/api/train_all', { num_stocks: numStocks, worker_threads: numThreads }); - }); - - - document.getElementById('btn-stop').addEventListener('click', async () => { - try { - const res = await fetch('/api/stop', { method: 'POST' }); - const data = await res.json(); - alert(data.message); - } catch (err) { - console.error(err); - } - }); - - document.getElementById('btn-kill').addEventListener('click', () => { - if (confirm("Are you sure you want to completely shut down the application?")) { - fetch('/api/kill', { method: 'POST' }).catch(err => console.log('Shutdown connection drop:', err)); - alert("Application shutdown initiated."); - document.body.innerHTML = "

Application Terminated

You can close this window.

"; - } - }); - - // Modal Logic - const modal = document.getElementById('graph-modal'); - const closeBtn = document.querySelector('.close-btn'); - const selectGraph = document.getElementById('graph-select'); - const graphImg = document.getElementById('graph-img'); - - document.getElementById('btn-view-graphs').addEventListener('click', async () => { - modal.style.display = 'block'; - try { - const res = await fetch('/api/graphs'); - const graphs = await res.json(); - selectGraph.innerHTML = ''; - if (graphs.length === 0) { - selectGraph.innerHTML = ''; - } else { - graphs.forEach(g => { - const option = document.createElement('option'); - option.value = g; - option.textContent = g; - selectGraph.appendChild(option); - }); - } - graphImg.style.display = 'none'; - } catch (err) { - console.error(err); - } - }); - - closeBtn.addEventListener('click', () => { - modal.style.display = 'none'; - }); - - window.addEventListener('click', (e) => { - if (e.target === modal) { - modal.style.display = 'none'; - } - }); - - selectGraph.addEventListener('change', (e) => { - const filename = e.target.value; - if (filename) { - // Add a timestamp query param to bypass browser cache - graphImg.src = `/api/graphs/${filename}?t=${new Date().getTime()}`; - graphImg.style.display = 'block'; - } else { - graphImg.style.display = 'none'; - } - }); - - // Metrics and Modal management - let currentMetrics = { - total_symbols: { count: 0, symbols: [] }, - tuned_symbols: { count: 0, symbols: [] }, - trained_symbols: { count: 0, symbols: [] }, - backtest_graphs: { count: 0, symbols: [] } - }; - - async function fetchMetrics() { - try { - const res = await fetch('/api/metrics'); - const data = await res.json(); - currentMetrics = data; - - // Update UI elements - document.getElementById('metric-total-symbols').textContent = data.total_symbols.count; - document.getElementById('metric-tuned-symbols').textContent = data.tuned_symbols.count; - document.getElementById('metric-trained-symbols').textContent = data.trained_symbols.count; - document.getElementById('metric-backtest-graphs').textContent = data.backtest_graphs.count; - - // Tuning status - const tuningStatusEl = document.getElementById('metric-tuning-status'); - if (data.tuning_in_progress) { - tuningStatusEl.textContent = `Running (${data.completed_count}/${data.total_count})`; - tuningStatusEl.classList.add('active-status'); - } else { - tuningStatusEl.textContent = 'Idle'; - tuningStatusEl.classList.remove('active-status'); - } - - // Training status - const trainingStatusEl = document.getElementById('metric-training-status'); - if (data.training_in_progress) { - trainingStatusEl.textContent = `Running (${data.completed_count}/${data.total_count})`; - trainingStatusEl.classList.add('active-status'); - } else { - trainingStatusEl.textContent = 'Idle'; - trainingStatusEl.classList.remove('active-status'); - } - } catch (err) { - console.error('Error fetching metrics:', err); - } - } - - // Call fetchMetrics inside status loop - const originalCheckStatus = checkStatus; - checkStatus = async function() { - await originalCheckStatus(); - await fetchMetrics(); - }; - - // Generic Symbols Modal - const symbolsModal = document.getElementById('symbols-modal'); - const closeSymbolsBtn = document.getElementById('close-symbols-btn'); - const symbolsTitle = document.getElementById('symbols-modal-title'); - const symbolsContainer = document.getElementById('symbols-list-container'); - - function showSymbolsList(title, symbols) { - symbolsTitle.textContent = title; - symbolsContainer.innerHTML = ''; - if (!symbols || symbols.length === 0) { - symbolsContainer.innerHTML = '
No symbols available
'; - } else { - symbols.forEach(sym => { - const badge = document.createElement('div'); - badge.className = 'symbol-badge'; - badge.textContent = sym; - symbolsContainer.appendChild(badge); - }); - } - symbolsModal.style.display = 'block'; - } - - document.getElementById('metric-total-symbols').addEventListener('click', () => { - showSymbolsList('Total Symbols List', currentMetrics.total_symbols.symbols); - }); - document.getElementById('metric-tuned-symbols').addEventListener('click', () => { - showSymbolsList('Tuned Symbols List', currentMetrics.tuned_symbols.symbols); - }); - document.getElementById('metric-trained-symbols').addEventListener('click', () => { - showSymbolsList('Trained Symbols List', currentMetrics.trained_symbols.symbols); - }); - document.getElementById('metric-backtest-graphs').addEventListener('click', () => { - showSymbolsList('Backtest Graphs Available', currentMetrics.backtest_graphs.symbols); - }); - - closeSymbolsBtn.addEventListener('click', () => { - symbolsModal.style.display = 'none'; - }); - - window.addEventListener('click', (e) => { - if (e.target === symbolsModal) { - symbolsModal.style.display = 'none'; - } - }); -}); diff --git a/nsereco/task_manager.py b/nsereco/task_manager.py deleted file mode 100644 index 93897c0a141e23bb786b57ff89db4e6b0ae9c005..0000000000000000000000000000000000000000 --- a/nsereco/task_manager.py +++ /dev/null @@ -1,96 +0,0 @@ -import threading -import logging -from collections import deque -import traceback - -class TaskManager: - def __init__(self): - self.current_task = None - self.status = "Idle" - self.completed_count = 0 - self.total_count = 0 - self.logs = deque(maxlen=500) - self.lock = threading.Lock() - self.setup_custom_logger() - - def update_progress(self, completed, total, status=None): - with self.lock: - self.completed_count = completed - self.total_count = total - if status is not None: - self.status = status - - def increment_progress(self): - with self.lock: - self.completed_count += 1 - - def setup_custom_logger(self): - class DequeHandler(logging.Handler): - def __init__(self, deque_obj): - super().__init__() - self.deque = deque_obj - - def emit(self, record): - try: - msg = self.format(record) - self.deque.append(msg) - except Exception: - self.handleError(record) - - self.handler = DequeHandler(self.logs) - self.handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) - logging.getLogger().addHandler(self.handler) - # Ensure base logging level is INFO - logging.getLogger().setLevel(logging.INFO) - - def start_task(self, name, func, *args, **kwargs): - with self.lock: - if self.current_task and self.current_task.is_alive(): - return False, "A task is already running." - - self.status = f"Running: {name}" - self.completed_count = 0 - self.total_count = 0 - self.logs.clear() - logging.info(f"--- Started Task: {name} ---") - - import nsereco.utils as utils - utils.interrupted = False - - def task_wrapper(): - try: - func(*args, **kwargs) - logging.info(f"--- Task {name} Completed Successfully ---") - except Exception as e: - logging.error(f"Task Failed: {e}") - logging.error(traceback.format_exc()) - finally: - with self.lock: - self.status = "Idle" - self.completed_count = 0 - self.total_count = 0 - - self.current_task = threading.Thread(target=task_wrapper, daemon=True) - self.current_task.start() - return True, "Task started successfully." - - def stop_task(self): - with self.lock: - if self.current_task and self.current_task.is_alive(): - import nsereco.utils as utils - utils.interrupted = True - logging.info("--- Cancel Requested. Waiting for graceful shutdown... ---") - return True, "Stop signal sent. Task is shutting down." - return False, "No task is currently running." - - def get_status(self): - with self.lock: - return { - "status": self.status, - "is_running": self.current_task is not None and self.current_task.is_alive(), - "completed_count": self.completed_count, - "total_count": self.total_count, - "logs": list(self.logs) - } - -task_manager = TaskManager() diff --git a/nsereco/templates/index.html b/nsereco/templates/index.html deleted file mode 100644 index c25f4d8f3c124ba5c5b5ebb0267d0348e965d6c9..0000000000000000000000000000000000000000 --- a/nsereco/templates/index.html +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - NSEReco Dashboard - - - - - -
-
-

NSEReco Dashboard

-
- - Checking status... -
-
- - -
-
-
Total Symbols
-
-
-
-
-
Tuned Symbols
-
-
-
-
-
Trained Symbols
-
-
-
-
-
Backtest Graphs
-
-
-
-
-
Tuning Status
-
Idle
-
-
-
Training Status
-
Idle
-
-
- -
- - - -
- -
- -
-

System Setup

-

Initialize the data manager, load datasets, and engineer 50+ features. This must be done before running models.

- -
- - -
-

1. Optuna tuning

-

Check list of stock for pending tuning, prioritize nifty stocks, trigger number of symbols / threads for tuning via the Agent Workflow.

-
- - -
-
- - -
- -
- - -
-

2. Model training

-

Check for tuning completed but pending training, get best hyperparameters, prioritize nifty stocks, trigger training via the Agent Workflow.

-
- - -
-
- - -
- -
-
- -
-
-

Live Console Output

-
-
- -
-
-
- - - - - - - - - - diff --git a/nsereco/utils.py b/nsereco/utils.py deleted file mode 100644 index 91916271a474b22fb83768aed59e424e8a076cfa..0000000000000000000000000000000000000000 --- a/nsereco/utils.py +++ /dev/null @@ -1,152 +0,0 @@ -import pandas as pd -import logging -from pathlib import Path -from datetime import datetime, timedelta -import json -import gc -import warnings -import numpy as np -import tensorflow as tf -from tensorflow.keras.models import Sequential, load_model -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint -from tensorflow.keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, BatchNormalization, Conv1D, GRU, Attention, Concatenate -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import ReduceLROnPlateau, Callback -import tensorflow.keras.backend as K -import optuna -from optuna.integration import TFKerasPruningCallback -from rich.table import Table -from rich.console import Console -from rich import box -import matplotlib.pyplot as plt -import seaborn as sns -import threading - -matplotlib_lock = threading.Lock() -from sklearn.preprocessing import MinMaxScaler -from sklearn.utils.class_weight import compute_class_weight -from sklearn.model_selection import train_test_split -import time -import sys -import os -import functools -from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError -import threading -from multiprocessing import Pool -from itertools import product -import signal -import talib -import fastparquet - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from torch.optim import Adam -from torch.nn import HuberLoss - - -def show_splash_screen(): - """Prints a beautiful ANSI-colored ASCII splash screen banner at startup.""" - is_tty = sys.stdout.isatty() - GREEN = "\033[92m" if is_tty else "" - CYAN = "\033[96m" if is_tty else "" - RESET = "\033[0m" if is_tty else "" - BOLD = "\033[1m" if is_tty else "" - - splash = f"""{GREEN}{BOLD} -================================================================================ - _ _ _____ _____ ______ _ -| \\ | |/ ___|| ___|| ___ \\ | | -| \\| |\\ `--. | |__ | |_/ /___ ___ ___ _ __ ___ _ __ ___ ___ _ __ __| | -| . ` | `--. \\| __| | // _ \\/ __|/ _ \\| '_ ` _ \\| '_ ` _ \\ / _ \\ '_ \\ / _` | -| |\\ |/\\__/ /| |___ | |\\ \\ __/ (__| (_) | | | | | | | | | | | __/ | | | (_| | -\\_| \\_/\\____/ \\____/ \\_| \\_\\___|\\___|\\___/|_| |_| |_|_| |_| |_|\\___|_| |_|\\__,_| - - {CYAN}NSE Stock Recommender & Hyperparameter Tuner (v3.1){GREEN} -==============================================================================={RESET} - """ - print(splash) - -warnings.filterwarnings('ignore') -warnings.simplefilter('ignore', FutureWarning) -# Removed TF_CPP_MIN_LOG_LEVEL overwrite to preserve level 3 from main app - -# Check if GPU available -# print("Checking for GPU availability...") -# print("TensorFlow version:", tf.__version__) -physical_devices = tf.config.list_physical_devices('GPU') - -if physical_devices: - try: - for device in physical_devices: - tf.config.experimental.set_memory_growth(device, True) - # print(f"GPU devices found: {[device.name for device in physical_devices]}") - except Exception as e: - # print(f"Error setting up GPU memory growth: {e}") - pass -else: - # print("No GPU devices found. Running on CPU...") - pass - -# A static list of Nifty 50 stocks for filtering -NIFTY_STOCKS = ['ADANIENT', 'ADANIPORTS', 'APOLLOHOSP', 'ASIANPAINT', 'AXISBANK', 'BAJAJ-AUTO', - 'BAJFINANCE', 'BAJAJFINSV', 'BHARTIARTL', 'BPCL', 'BRITANNIA', 'CIPLA', 'COALINDIA', - 'DIVISLAB', 'DRREDDY', 'EICHERMOT', 'GRASIM', 'HCLTECH', 'HDFCBANK', 'HDFCLIFE', - 'HEROMOTOCO', 'HINDALCO', 'HINDUNILVR', 'ICICIBANK', 'INDUSINDBK', 'INFY', 'IOC', - 'ITC', 'JSWSTEEL', 'KOTAKBANK', 'LT', 'LTIM', 'M&M', 'MARUTI', 'NESTLEIND', - 'NTPC', 'ONGC', 'POWERGRID', 'RELIANCE', 'SBILIFE', 'SBIN', 'SUNPHARMA', 'TCS', - 'TATACONSUM', 'TATAMOTORS', 'TATASTEEL', 'TECHM', 'TITAN', 'ULTRACEMCO', 'WIPRO'] - -# Global interruption flag -interrupted = False -def signal_handler(signum, frame): - """ - Handles system signals (like Ctrl+C) to enable graceful shutdown. - Sets the global 'interrupted' flag to True. - """ - global interrupted - if interrupted: - logging.warning("\nForce quitting...") - os._exit(1) - - logging.info(f"\nCaught signal {signum}. Initiating graceful shutdown. Please wait... (Press Ctrl+C again to force quit)") - interrupted = True - sys.exit(0) - -# Register the signal handler for graceful exit on Ctrl+C or termination signals -signal.signal(signal.SIGINT, signal_handler) -signal.signal(signal.SIGTERM, signal_handler) - -# --- Utility Functions for JSON Handling --- -def load_json_file(file_path, default_data={}): - """Loads JSON data from a file, handles errors and returns default if file is empty or missing.""" - try: - if file_path.exists(): - with open(file_path, 'r') as f: - content = f.read().strip() - if content: - return json.loads(content) - else: - logging.warning(f"File {file_path} is empty. Returning default data.") - return default_data - else: - logging.info(f"File {file_path} not found. Returning default data.") - return default_data - except json.JSONDecodeError as e: - logging.error(f"Failed to load JSON data from {file_path} (JSON decode error: {e}). Returning default data.") - return default_data - except Exception as e: - logging.error(f"Failed to load JSON data from {file_path} (unexpected error: {e}). Returning default data.") - return default_data - -def save_json_file(file_path, data): - """Saves data to a JSON file.""" - try: - with open(file_path, 'w') as f: - json.dump(data, f, indent=4) - except Exception as e: - logging.error(f"Failed to save JSON data to {file_path}: {e}") - -# --- StockData Class (for data loading and preprocessing) --- diff --git a/nsesymhist.py b/nsesymhist.py deleted file mode 100644 index 533f7cdf94e746fc0ecd83cbc699794c3e856498..0000000000000000000000000000000000000000 --- a/nsesymhist.py +++ /dev/null @@ -1,670 +0,0 @@ -""" -NSE Symbol Data and Historical Volume Analyzer (nsesymhist.py) - -This console utility processes historical stock data from the National Stock Exchange (NSE) -to identify extreme price movers and volume surges over multiple timeframes (1D, 1W, 1M, 3M). - -Features: -- Standalone CLI utility keeping all original analysis functions intact. -- Interactive ANSI color-enabled splash screen at startup. -- Configurable base directory, volume cutoff thresholds, and log levels. -- Bug fix for the missing 'self.extract_file' path. -- Saves generated watchlists as JSON in the processed/watchlists/ directory. - -Author: Antigravity AI -Date: July 2026 -""" - -import pandas as pd -import numpy as np -import logging -from pathlib import Path -from datetime import datetime, timedelta -import json -import warnings -import time -import os -import sys -import argparse - -# Suppress warnings from libraries like pandas and matplotlib -warnings.filterwarnings('ignore') - - -def show_splash_screen(): - """Prints a beautiful ANSI-colored ASCII splash screen banner at startup.""" - is_tty = sys.stdout.isatty() - GREEN = "\033[92m" if is_tty else "" - CYAN = "\033[96m" if is_tty else "" - RESET = "\033[0m" if is_tty else "" - BOLD = "\033[1m" if is_tty else "" - - splash = f"""{GREEN}{BOLD} -================================================================================ - _ _ _____ _____ _____ _ _ ___ ___ _ _ _____ _____ _____ -| \\ | |/ ___|| ___|/ ___| | | || \\/ || | | |_ _/ ___|_ _| -| \\| |\\ `--. | |__ \\ `--. | |_| || . . || |_| | | | \\ `--. | | -| . ` | `--. \\| __| `--. \\| _ || |\\/| || _ | | | `--. \\| | -| |\\ |/\\__/ /| |___ /\\__/ /| | | || | | || | | |_| |_/\\__/ / | | -\\_| \\_/\\____/ \\____/ \\____/ \\_| |_/\\_| |_/\\_| |_/\\___/\\____/ \\_/ - - {CYAN}NSE Symbol Data & Historical Volume Analyzer (v1.0){GREEN} -================================================================================{RESET} - """ - print(splash) - - -class NSEAnalysisUtility: - """ - A console-based utility for performing Max Min Analysis and Volume Analysis - on NSE stock data and saving the results to JSON files. - """ - def __init__(self, base_dir=None, volume_cutoff_threshold=10000): - self.base_dir = Path(base_dir or Path.cwd()) - self.df = pd.DataFrame() # Initialize an empty DataFrame - self.etf_symbols = set() # Initialize empty set for ETF symbols - self.volume_cutoff_threshold = volume_cutoff_threshold # Default volume cutoff threshold - self.setup_paths() - self.load_data() - self.etf_symbols = self.load_etf_symbols() # Load ETF symbols after data is loaded - - def setup_paths(self): - """ - Sets up directory and file paths for processed data and watchlists. - Ensures necessary directories exist. - """ - self.data_dir = self.base_dir / 'processed' - self.watchlist_dir = self.data_dir / 'watchlists' - self.log_dir = self.base_dir / 'downloads' # For general logs if needed, though this utility logs to console - - self.input_file = self.data_dir / 'nse_processed_data.parquet' - self.etf_list_file = self.data_dir / 'nse_dashboard_etflist.json' - self.symboldata_list_file = self.watchlist_dir / 'nse_dashboard_symboldata.json' - self.extract_file = self.data_dir / 'nse_extract_data.parquet' # Bug Fix: defined extract_file - - # Watchlist files for Max Min Analysis output (Historical) - self.hist_max_price_1d_file = self.watchlist_dir / 'nse_dashboard_hist_maxprice1d50.json' - self.hist_min_price_1d_file = self.watchlist_dir / 'nse_dashboard_hist_minprice1d50.json' - self.hist_max_price_1w_file = self.watchlist_dir / 'nse_dashboard_hist_maxprice1w50.json' - self.hist_min_price_1w_file = self.watchlist_dir / 'nse_dashboard_hist_minprice1w50.json' - self.hist_max_price_1m_file = self.watchlist_dir / 'nse_dashboard_hist_maxprice1m50.json' - self.hist_min_price_1m_file = self.watchlist_dir / 'nse_dashboard_hist_minprice1m50.json' - self.hist_max_price_3m_file = self.watchlist_dir / 'nse_dashboard_hist_maxprice3m50.json' - self.hist_min_price_3m_file = self.watchlist_dir / 'nse_dashboard_hist_minprice3m50.json' - - # Watchlist files for Volume Analysis output (Historical) - self.hist_max_vol_1d_file = self.watchlist_dir / 'nse_dashboard_hist_maxvol1d50.json' - self.hist_max_vol_1w_file = self.watchlist_dir / 'nse_dashboard_hist_maxvol1w50.json' - self.hist_max_vol_1m_file = self.watchlist_dir / 'nse_dashboard_hist_maxvol1m50.json' - self.hist_max_vol_3m_file = self.watchlist_dir / 'nse_dashboard_hist_maxvol3m50.json' - - # Create directories - self.watchlist_dir.mkdir(parents=True, exist_ok=True) - self.log_dir.mkdir(parents=True, exist_ok=True) - self.data_dir.mkdir(parents=True, exist_ok=True) - - def load_etf_symbols(self): - """Loads ETF symbols from a JSON file.""" - try: - if self.etf_list_file.exists(): - with open(self.etf_list_file, 'r') as f: - etfs = json.load(f) - logging.info(f"Loaded {len(etfs)} ETF symbols from {self.etf_list_file}") - return set(etfs) # Use a set for efficient lookup - else: - logging.warning(f"ETF list file not found: {self.etf_list_file}. No ETFs will be excluded from analysis.") - return set() - except Exception as e: - logging.error(f"Error loading ETF symbols from {self.etf_list_file}: {e}") - return set() - - def load_data(self): - """ - Loads and processes stock data from the input CSV file. - Optimizes data types and filters for the last 2 years of data. - Falls back to parquet if CSV is not found. - """ - logging.info("Loading stock data...") - start_time = time.time() - try: - if self.input_file.exists(): - # Read only required columns from Parquet - self.df = pd.read_parquet( - self.input_file, - columns=['SYMBOL', 'SECURITY', 'TRAN_DATE', - 'PREV_CL_PR', 'CLOSE_PRICE', 'NET_TRDQTY', - 'HI_52_WK', 'LO_52_WK', 'HI_DIFF', 'LO_DIFF'] - ) - if not pd.api.types.is_datetime64_any_dtype(self.df['TRAN_DATE']): - self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE']) - elif self.extract_file.exists(): - logging.info(f"Input file not found. Loading from extract parquet file: {self.extract_file}") - self.df = pd.read_parquet(self.extract_file) - else: - parquet_input = self.data_dir / 'nse_processed_data.parquet' - if parquet_input.exists(): - logging.info(f"Input file not found. Loading from main parquet file: {parquet_input}") - self.df = pd.read_parquet( - parquet_input, - columns=['SYMBOL', 'SECURITY', 'TRAN_DATE', - 'PREV_CL_PR', 'CLOSE_PRICE', 'NET_TRDQTY', - 'HI_52_WK', 'LO_52_WK', 'HI_DIFF', 'LO_DIFF'] - ) - self.df['TRAN_DATE'] = pd.to_datetime(self.df['TRAN_DATE']) - else: - logging.error(f"Input file not found (Parquet) in {self.data_dir}. Please ensure data is processed.") - self.df = pd.DataFrame() - return - - self.last_date = self.df['TRAN_DATE'].max() - self.two_years_ago = self.last_date - timedelta(days=730) - - mask = self.df['TRAN_DATE'] >= self.two_years_ago - self.df = self.df[mask].copy() - - # Save extract to parquet for faster future loading if needed - self.df.to_parquet(self.extract_file, index=False) - - logging.info(f"Data loaded successfully (filtered to >= {self.two_years_ago.strftime('%Y-%m-%d')}).") - logging.info(f"DataFrame shape: {self.df.shape}. Elapsed time: {time.time() - start_time:.2f} seconds.") - - except Exception as e: - logging.error(f"Failed to load data: {e}") - self.df = pd.DataFrame() # Ensure df is empty on failure - - def get_data_for_date(self, date): - """ - Retrieves stock data for a specific date and calculates various - historical changes (week, month, 3-month, 2-year) and volume changes. - """ - logging.info(f"Preparing data for date: {date.strftime('%Y-%m-%d')}") - start_time = time.time() - try: - if self.df.empty: - logging.warning("DataFrame is empty. Cannot get data for date.") - return pd.DataFrame() - - # Convert to datetime if needed - if isinstance(date, datetime): - date_obj = date.date() - else: - date_obj = date - - # Get data for the specific date - date_data = self.df[self.df['TRAN_DATE'].dt.date == date_obj].copy() - - if date_data.empty: - logging.warning(f"No data for {date_obj.strftime('%Y-%m-%d')}. Skipping calculations for this date.") - return pd.DataFrame() - - # Initialize new columns - date_data['PRICE_HISTORY'] = None - date_data['VOLUME_HISTORY'] = None - date_data['AVG_VOLUME'] = None - date_data['DAILY_CHANGE'] = 0.0 # Initialize DAILY_CHANGE - date_data['WEEK_CHANGE'] = 0.0 - date_data['MONTH_CHANGE'] = 0.0 - date_data['THREE_MONTH_CHANGE'] = 0.0 - date_data['TWO_YEAR_CHANGE'] = 0.0 - date_data['VOLUME_CHANGES'] = None # To store the last 5 volume changes - - # Get all unique symbols from the data for the selected date - symbols_to_process = date_data['SYMBOL'].unique().tolist() - total_symbols = len(symbols_to_process) - processed_symbols = 0 - - for symbol in symbols_to_process: - processed_symbols += 1 - # Progress update - if processed_symbols % 100 == 0 or processed_symbols == total_symbols: - logging.info(f" Calculating changes for {symbol} ({processed_symbols}/{total_symbols} symbols processed)") - - symbol_data = self.df[self.df['SYMBOL'] == symbol].sort_values(by='TRAN_DATE', ascending=True).copy() - if symbol_data.empty: - continue - - curr_date_row = date_data[date_data['SYMBOL'] == symbol] - if curr_date_row.empty: - continue - curr_date = curr_date_row['TRAN_DATE'].iloc[0] - curr_date_idx = curr_date_row.index[0] # Get index for direct assignment - - # Calculate DAILY_CHANGE - prev_cl_pr = curr_date_row['PREV_CL_PR'].iloc[0] - close_price = curr_date_row['CLOSE_PRICE'].iloc[0] - if prev_cl_pr != 0: - daily_change = ((close_price / prev_cl_pr - 1) * 100) - date_data.loc[curr_date_idx, 'DAILY_CHANGE'] = daily_change - else: - date_data.loc[curr_date_idx, 'DAILY_CHANGE'] = 0.0 - - # Calculate various period changes - periods = { - 'WEEK_CHANGE': 7, - 'MONTH_CHANGE': 30, - 'THREE_MONTH_CHANGE': 90, - 'TWO_YEAR_CHANGE': 365 * 2 - } - - for col, days_back in periods.items(): - start_period = curr_date - timedelta(days=days_back) - period_prices = symbol_data[(symbol_data['TRAN_DATE'] <= curr_date) & - (symbol_data['TRAN_DATE'] >= start_period)]['CLOSE_PRICE'] - - if len(period_prices) > 1: - change = ((period_prices.iloc[-1] / period_prices.iloc[0] - 1) * 100) - date_data.loc[curr_date_idx, col] = change - else: - date_data.loc[curr_date_idx, col] = 0.0 - - # 2-year data for history (for graph plotting, though not directly used in console output) - two_years_ago_for_history = curr_date - timedelta(days=365 * 2) - two_year_history_data = symbol_data[(symbol_data['TRAN_DATE'] <= curr_date) & - (symbol_data['TRAN_DATE'] >= two_years_ago_for_history)] - - if not two_year_history_data.empty: - # Store price history as list - history = two_year_history_data[['TRAN_DATE', 'CLOSE_PRICE']].values.tolist() - date_data.at[curr_date_idx, 'PRICE_HISTORY'] = history - - # Store volume history as list - volume_history = two_year_history_data[['TRAN_DATE', 'NET_TRDQTY']].values.tolist() - date_data.at[curr_date_idx, 'VOLUME_HISTORY'] = volume_history - - # Calculate and store average volume over the period - avg_volume = two_year_history_data['NET_TRDQTY'].mean() - date_data.at[curr_date_idx, 'AVG_VOLUME'] = avg_volume - else: - date_data.at[curr_date_idx, 'AVG_VOLUME'] = 0.0 - - # Calculate last 5 volume changes (D1 to D5) - volumes_for_changes = [] - temp_date = curr_date - days_checked = 0 - while len(volumes_for_changes) < 6 and days_checked < 30: # Max 6 volumes needed for 5 changes - day_data = symbol_data[symbol_data['TRAN_DATE'].dt.date == temp_date.date()] - if not day_data.empty: - volumes_for_changes.append(float(day_data['NET_TRDQTY'].iloc[0])) - temp_date -= timedelta(days=1) - days_checked += 1 - - # Pad with zeros if less than 6 volumes were collected - while len(volumes_for_changes) < 6: - volumes_for_changes.append(0.0) - - # Reverse to get oldest to newest: [V_6_days_ago, ..., V_today] - volumes_for_changes.reverse() - volume_changes = [] - for i in range(1, len(volumes_for_changes)): - prev_vol = volumes_for_changes[i-1] - curr_vol = volumes_for_changes[i] - if prev_vol != 0: - change = ((curr_vol - prev_vol) / prev_vol) * 100 - else: - change = 0.0 - volume_changes.append(round(change, 2)) - - # Store the last 5 calculated daily volume changes (D1 to D5 in original context) - date_data.at[curr_date_idx, 'VOLUME_CHANGES'] = volume_changes[-5:] - - logging.info(f"Data preparation for {date_obj.strftime('%Y-%m-%d')} completed. Elapsed time: {time.time() - start_time:.2f} seconds.") - return date_data - - except Exception as e: - logging.error(f"Failed to get data for {date.strftime('%Y-%m-%d')}: {e}") - return pd.DataFrame() - - def _calculate_average_volume_change(self, symbol_data, days_back): - """ - Calculates the average daily volume percentage change for a given stock - over a specified number of historical days. - """ - if symbol_data.empty or len(symbol_data) < 2: - return 0.0 - - symbol_data = symbol_data.sort_values(by='TRAN_DATE', ascending=True) - - end_date = symbol_data['TRAN_DATE'].iloc[-1] # Most recent date in the subset - start_date = end_date - timedelta(days=days_back) - - period_data = symbol_data[(symbol_data['TRAN_DATE'] <= end_date) & - (symbol_data['TRAN_DATE'] >= start_date)].copy() - - if period_data.empty or len(period_data) < 2: - return 0.0 - - volume_changes = [] - for i in range(1, len(period_data)): - current_volume = period_data['NET_TRDQTY'].iloc[i] - previous_volume = period_data['NET_TRDQTY'].iloc[i-1] - - if previous_volume != 0: - change = ((current_volume - previous_volume) / previous_volume) * 100 - else: - change = 0.0 - volume_changes.append(change) - - if volume_changes: - return sum(volume_changes) / len(volume_changes) - return 0.0 - - def _save_watchlist_json(self, df_to_save, file_path, name): - """Helper to save a DataFrame to a JSON watchlist format.""" - watchlist_data = { - "name": name, - "symbols": [], - "last_updated": datetime.now().isoformat() - } - for _, row in df_to_save.iterrows(): - watchlist_data["symbols"].append({ - "SYMBOL": row['SYMBOL'], - "Quantity": 0.0, - "Price": 0.0 - }) - try: - file_path.parent.mkdir(parents=True, exist_ok=True) # Ensure directory exists - with open(file_path, 'w') as f: - json.dump(watchlist_data, f, indent=4) - logging.info(f"Saved '{name}' watchlist to {file_path}") - return True - except Exception as e: - logging.error(f"Error saving '{name}' watchlist to {file_path}: {e}") - return False - - def _load_analysis_json_if_recent(self, file_path, current_data_tran_date): - """ - Loads symbols from a JSON file if its 'last_updated' date is equal to or later than - the current_data_tran_date. Returns a list of symbols if valid, else None. - """ - if not file_path.exists(): - logging.debug(f"Analysis JSON file not found: {file_path}") - return None - - try: - with open(file_path, 'r') as f: - data = json.load(f) - - last_updated_str = data.get('last_updated') - if not last_updated_str: - logging.warning(f"'{file_path}' has no 'last_updated' field. Will recalculate.") - return None - - last_updated_date = datetime.fromisoformat(last_updated_str).date() - - if last_updated_date >= current_data_tran_date.date(): - logging.info(f"Loading '{file_path}' from cache (last_updated: {last_updated_date}, current_data_tran_date: {current_data_tran_date.date()}).") - symbols_to_load = [] - for entry in data.get('symbols', []): - if isinstance(entry, dict) and 'SYMBOL' in entry: - symbols_to_load.append(entry['SYMBOL']) - return symbols_to_load - else: - logging.info(f"'{file_path}' is outdated (last_updated: {last_updated_date}, current_data_tran_date: {current_data_tran_date.date()}). Will recalculate.") - return None - except (json.JSONDecodeError, FileNotFoundError, ValueError) as e: - logging.warning(f"Error reading or parsing '{file_path}': {e}. Will recalculate.") - return None - except Exception as e: - logging.error(f"An unexpected error occurred while loading analysis JSON from {file_path}: {e}. Will recalculate.") - return None - - def perform_max_min_analysis(self, duration, change_type, current_data_max_tran_date, current_date_data): - """ - Performs Max/Min % change analysis for a given duration and saves to JSON. - """ - file_map = { - ("1D", "Max"): (self.hist_max_price_1d_file, "1D Max Price Portfolio", "DAILY_CHANGE"), - ("1D", "Min"): (self.hist_min_price_1d_file, "1D Min Price Portfolio", "DAILY_CHANGE"), - ("1W", "Max"): (self.hist_max_price_1w_file, "1W Max Price Portfolio", "WEEK_CHANGE"), - ("1W", "Min"): (self.hist_min_price_1w_file, "1W Min Price Portfolio", "WEEK_CHANGE"), - ("1M", "Max"): (self.hist_max_price_1m_file, "1M Max Price Portfolio", "MONTH_CHANGE"), - ("1M", "Min"): (self.hist_min_price_1m_file, "1M Min Price Portfolio", "MONTH_CHANGE"), - ("3M", "Max"): (self.hist_max_price_3m_file, "3M Max Price Portfolio", "THREE_MONTH_CHANGE"), - ("3M", "Min"): (self.hist_min_price_3m_file, "3M Min Price Portfolio", "THREE_MONTH_CHANGE"), - } - - file_path, json_name, change_column = file_map.get((duration, change_type)) - if not file_path: - logging.error(f"Invalid duration or change type for Max Min Analysis: {duration}, {change_type}") - return False - - logging.info(f"Performing '{json_name}' analysis...") - loaded_symbols = self._load_analysis_json_if_recent(file_path, current_data_max_tran_date) - - if loaded_symbols: - logging.info(f" '{json_name}' data loaded from cache. No recalculation needed.") - return True - else: - if current_date_data.empty: - logging.warning(f" No stock data available for analysis date for '{json_name}'. Skipping.") - return False - - filtered_data = current_date_data.dropna(subset=[change_column]).copy() - - if filtered_data.empty: - logging.warning(f" No valid data in '{change_column}' for '{json_name}'. Skipping.") - return False - - if change_type == "Max": - # For Max analysis, we typically want positive changes - filtered_data = filtered_data[filtered_data[change_column] > 0] - filtered_data = filtered_data.sort_values(by=change_column, ascending=False) - elif change_type == "Min": - # For Min analysis, we typically want negative changes (falls) - filtered_data = filtered_data[filtered_data[change_column] < 0] - filtered_data = filtered_data.sort_values(by=change_column, ascending=True) - - stocks_to_save = filtered_data.head(50).copy() - - if stocks_to_save.empty: - logging.warning(f" No stocks met the criteria for '{json_name}'. No file saved.") - return False - - return self._save_watchlist_json(stocks_to_save, file_path, json_name) - - def perform_volume_analysis(self, duration, current_data_max_tran_date, current_date_data): - """ - Performs Volume Analysis for a given duration and saves to JSON. - """ - file_map = { - "1D": (self.hist_max_vol_1d_file, "1D Max Volume Portfolio"), - "1W": (self.hist_max_vol_1w_file, "1W Max Volume Portfolio"), - "1M": (self.hist_max_vol_1m_file, "1M Max Volume Portfolio"), - "3M": (self.hist_max_vol_3m_file, "3M Max Volume Portfolio"), - } - - file_path, json_name = file_map.get(duration) - if not file_path: - logging.error(f"Invalid duration for Volume Analysis: {duration}") - return False - - logging.info(f"Performing '{json_name}' analysis...") - loaded_symbols = self._load_analysis_json_if_recent(file_path, current_data_max_tran_date) - - if loaded_symbols: - logging.info(f" '{json_name}' data loaded from cache. No recalculation needed.") - return True - else: - if current_date_data.empty: - logging.warning(f" No stock data available for analysis date for '{json_name}'. Skipping.") - return False - - top_movers = [] - symbols_to_process = current_date_data['SYMBOL'].unique().tolist() - total_symbols_for_vol_calc = len(symbols_to_process) - processed_symbols_for_vol_calc = 0 - - for symbol in symbols_to_process: - processed_symbols_for_vol_calc += 1 - if processed_symbols_for_vol_calc % 100 == 0 or processed_symbols_for_vol_calc == total_symbols_for_vol_calc: - logging.info(f" Processing volume for {symbol} ({processed_symbols_for_vol_calc}/{total_symbols_for_vol_calc})") - - # Exclude ETF symbols - if symbol in self.etf_symbols: - logging.debug(f" Excluding ETF: {symbol} from volume analysis.") - continue - - # Also exclude if "ETF" in SECURITY name from the current_date_data - security_name_series = current_date_data[current_date_data['SYMBOL'] == symbol].get('SECURITY', pd.Series([''])) - security_name = security_name_series.iloc[0] if not security_name_series.empty else '' - if "ETF" in str(security_name).upper(): - logging.debug(f" Excluding ETF (by name): {symbol} ({security_name}) from volume analysis.") - continue - - # Exclude stocks with average volume below the cutoff - avg_volume_series = current_date_data[current_date_data['SYMBOL'] == symbol].get('AVG_VOLUME', pd.Series([0.0])) - avg_volume = avg_volume_series.iloc[0] if not avg_volume_series.empty else 0.0 - if avg_volume < self.volume_cutoff_threshold: - logging.debug(f" Excluding {symbol} due to average volume ({avg_volume:,.0f}) below cutoff ({self.volume_cutoff_threshold:,.0f}).") - continue - - volume_pct_change = 0.0 - if duration == "1D": - # Use pre-calculated VOLUME_CHANGES for 1D - # Ensure that the column exists and is not empty before attempting to access - if 'VOLUME_CHANGES' in current_date_data.columns and not current_date_data[current_date_data['SYMBOL'] == symbol]['VOLUME_CHANGES'].isnull().all(): - volume_changes_list = current_date_data[current_date_data['SYMBOL'] == symbol]['VOLUME_CHANGES'].iloc[0] - if volume_changes_list and len(volume_changes_list) >= 5: - volume_pct_change = volume_changes_list[4] # This is D1 (most recent daily change) - else: - # For 1W, 1M, 3M, calculate average volume change over the period - days_map = {"1W": 7, "1M": 30, "3M": 90} - days_back = days_map.get(duration) - if days_back: - symbol_historic_data = self.df[self.df['SYMBOL'] == symbol].copy() - volume_pct_change = self._calculate_average_volume_change(symbol_historic_data, days_back) - - top_movers.append({ - "SYMBOL": symbol, - "VOLUME_CHANGE_PCT": volume_pct_change, - "SECURITY": security_name, - "AVG_VOLUME": avg_volume - }) - - movers_df = pd.DataFrame(top_movers) - - if movers_df.empty: - logging.warning(f" No top volume movers found for '{json_name}'. No file saved.") - return False - - movers_df = movers_df.sort_values(by="VOLUME_CHANGE_PCT", ascending=False).head(50) - - if movers_df.empty: - logging.warning(f" No stocks met the criteria after sorting for '{json_name}'. No file saved.") - return False - - return self._save_watchlist_json(movers_df, file_path, json_name) - - def run_all_analysis(self): - """ - Consolidates and runs both Max Min Analysis and Volume Analysis - for all specified durations in a single pass. - """ - logging.info("Starting consolidated Max Min and Volume Analysis...") - overall_start_time = time.time() - - if self.df.empty: - logging.error("No historical data loaded. Cannot perform analysis.") - return - - # Get the latest transaction date from the loaded data for comparison - current_data_max_tran_date = self.df['TRAN_DATE'].max() - logging.info(f"Latest historical data date: {current_data_max_tran_date.strftime('%Y-%m-%d')}") - - # Get the full processed data for the latest date once - # This includes all calculated changes needed for both analysis types - current_date_data = self.get_data_for_date(current_data_max_tran_date) - if current_date_data.empty: - logging.error("Failed to get data for the latest date. Analysis cannot proceed.") - return - - durations = ["1D", "1W", "1M", "3M"] - change_types = ["Max", "Min"] # Only "Max" for Volume Analysis - - total_tasks = (len(durations) * len(change_types)) + len(durations) - completed_tasks = 0 - - # Max Min Analysis - for duration in durations: - for change_type in change_types: - completed_tasks += 1 - logging.info(f"\n--- Task {completed_tasks}/{total_tasks}: Max Min Analysis ({duration} {change_type}) ---") - try: - self.perform_max_min_analysis(duration, change_type, current_data_max_tran_date, current_date_data) - except Exception as e: - logging.error(f"Error during Max Min Analysis for {duration} {change_type}: {e}") - # Continue to next task even if one fails - - # Volume Analysis (only Max) - for duration in durations: - completed_tasks += 1 - logging.info(f"\n--- Task {completed_tasks}/{total_tasks}: Volume Analysis ({duration} Max) ---") - try: - self.perform_volume_analysis(duration, current_data_max_tran_date, current_date_data) - except Exception as e: - logging.error(f"Error during Volume Analysis for {duration} Max: {e}") - # Continue to next task even if one fails - - overall_elapsed_time = time.time() - overall_start_time - logging.info(f"\nConsolidated analysis finished! Total elapsed time: {overall_elapsed_time:.2f} seconds.") - logging.info("All analysis JSON files should be in './processed/watchlists/'.") - - -def main(): - parser = argparse.ArgumentParser( - description="NSE Symbol Data and Historical Volume Analyzer CLI", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - "--base-dir", - type=str, - default=None, - help="Base directory containing the 'processed' and 'downloads' folders." - ) - parser.add_argument( - "--volume-cutoff", - type=float, - default=10000.0, - help="Volume cutoff threshold for excluding illiquid stocks from volume analysis." - ) - parser.add_argument( - "--no-splash", - action="store_true", - help="Suppress the startup splash screen." - ) - parser.add_argument( - "--log-level", - type=str, - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - help="Set the logging level." - ) - - args = parser.parse_args() - - # Configure logging for console output - logging.basicConfig( - level=getattr(logging, args.log_level), - format='%(asctime)s - %(levelname)s - %(message)s' - ) - - if not args.no_splash: - show_splash_screen() - - try: - # Create an instance of the utility - analysis_utility = NSEAnalysisUtility( - base_dir=args.base_dir, - volume_cutoff_threshold=args.volume_cutoff - ) - - # Run the consolidated analysis - analysis_utility.run_all_analysis() - - except Exception as e: - logging.critical(f"A critical error occurred: {e}", exc_info=True) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/nsevolmaxmin.py b/nsevolmaxmin.py deleted file mode 100644 index c2fd6d25aa140d8d9b7f65b5177c4a5a98a3eba1..0000000000000000000000000000000000000000 --- a/nsevolmaxmin.py +++ /dev/null @@ -1,147 +0,0 @@ -import pandas as pd -import json -import os -from datetime import datetime - -def check_and_convert_csv_to_parquet(csv_path: str, parquet_path: str) -> None: - """If a CSV file is found, convert it to Parquet and delete the CSV.""" - if os.path.exists(csv_path): - try: - df = pd.read_csv(csv_path) - df.to_parquet(parquet_path, index=False) - os.remove(csv_path) - print(f"Auto-converted {csv_path} to {parquet_path} and removed CSV.") - except Exception as e: - print(f"Failed to auto-convert {csv_path} to Parquet: {e}") - -def process_parquet_to_json( - input_parquet_path: str, - output_json_path: str, - portfolio_name: str, - symbol_col: str, - sort_col: str, - sort_ascending: bool = False, - top_n: int = 50, - header_clean_type: str = "strip_quotes_and_spaces" # "strip_quotes_and_spaces" or "strip_spaces" -) -> None: - """ - Processes a Parquet file, extracts top N symbols based on sorting criteria, - and saves them to a JSON file in the specified format. - """ - # Check for CSV format and convert to Parquet if found - csv_path = input_parquet_path.replace(".parquet", ".csv") - check_and_convert_csv_to_parquet(csv_path, input_parquet_path) - - try: - df = pd.read_parquet(input_parquet_path) - - # Clean up column names based on the specified type - cleaned_columns = [] - for col in df.columns: - if header_clean_type == "strip_quotes_and_spaces": - cleaned_columns.append(col.strip().replace('"', '')) - elif header_clean_type == "strip_spaces": - cleaned_columns.append(col.strip()) - else: - cleaned_columns.append(col) # No cleaning - - df.columns = cleaned_columns - - # Ensure sort_col and symbol_col exist after cleaning - if sort_col not in df.columns: - print(f"Error: Sort column '{sort_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}") - return - if symbol_col not in df.columns: - print(f"Error: Symbol column '{symbol_col}' not found in {input_parquet_path} after cleaning. Available columns: {df.columns.tolist()}") - return - - # Convert sort_col to numeric, handling errors by coercing to NaN - df[sort_col] = pd.to_numeric(df[sort_col], errors='coerce') - # Drop rows where sort_col is NaN (if conversion failed) - df.dropna(subset=[sort_col], inplace=True) - - # Sort the DataFrame - df_sorted = df.sort_values(by=sort_col, ascending=sort_ascending) - - # Select the top N symbols - top_symbols_df = df_sorted.head(top_n) - - # Prepare the list of symbols for JSON output - symbols_list = [] - for _, row in top_symbols_df.iterrows(): - symbols_list.append({ - "SYMBOL": row[symbol_col], - "Quantity": 0.0, - "Price": 0.0 - }) - - # Get current timestamp - last_updated = datetime.now().isoformat() - - # Create the final JSON structure - json_output = { - "name": portfolio_name, - "symbols": symbols_list, - "last_updated": last_updated - } - - # Create output directory if it doesn't exist - output_dir = os.path.dirname(output_json_path) - if output_dir and not os.path.exists(output_dir): - os.makedirs(output_dir) - - # Write to JSON file - with open(output_json_path, 'w') as f: - json.dump(json_output, f, indent=2) - - print(f"Successfully processed {input_parquet_path} and saved to {output_json_path}") - - except FileNotFoundError: - print(f"Error: Input file not found at {input_parquet_path}") - except KeyError as e: - print(f"Error: Missing expected column in {input_parquet_path}: {e}") - except Exception as e: - print(f"An unexpected error occurred while processing {input_parquet_path}: {e}") - -# --- Main execution --- -if __name__ == "__main__": - # Ensure the 'processed/watchlists' directory exists - output_base_dir = "./processed/watchlists" - if not os.path.exists(output_base_dir): - os.makedirs(output_base_dir) - - # 1. Process Volume.parquet - process_parquet_to_json( - input_parquet_path="./processed/Volume.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxvol50.json", - portfolio_name="Max Volume Portfolio", - symbol_col="SYMBOL", - sort_col="TODAY - VOLUME", - sort_ascending=False, # Descending for Max Volume - top_n=50, - header_clean_type="strip_quotes_and_spaces" - ) - - # 2. Process Advance.parquet - process_parquet_to_json( - input_parquet_path="./processed/Advance.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxprice50.json", - portfolio_name="Max Price Portfolio", - symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap - sort_col="%chng", - sort_ascending=False, # Descending for Max Price Change - top_n=50, - header_clean_type="strip_spaces" - ) - - # 3. Process Decline.parquet - process_parquet_to_json( - input_parquet_path="./processed/Decline.parquet", - output_json_path="./processed/watchlists/nse_dashboard_maxfall50.json", - portfolio_name="Max Fall Portfolio", - symbol_col="Symbol", # Note the exact column name 'Symbol' with initial cap - sort_col="%chng", - sort_ascending=True, # Ascending for Max Fall (most negative %chng) - top_n=50, - header_clean_type="strip_spaces" - ) diff --git a/processed/nse_processed_data.csv b/processed/nse_processed_data.csv deleted file mode 100644 index 0dad50ff8ec5e6dc96d7a465d288e97cbd007064..0000000000000000000000000000000000000000 --- a/processed/nse_processed_data.csv +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:37123a9552fa1a26ff2fb6595fab69c84f80ece55a8d2d456493bdbd24c53f4b -size 11419978 diff --git a/processed/security/userdata.db b/processed/security/userdata.db index 381c226ba452f3ee95779916ff5c9131245e8e3e..a73372cc9ed14636f86265a269beac6c36a8c10a 100644 Binary files a/processed/security/userdata.db and b/processed/security/userdata.db differ diff --git a/processed/security/userdata.db-journal b/processed/security/userdata.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..165e1f4354c351b273fd54d89eb88e74c28eb260 Binary files /dev/null and b/processed/security/userdata.db-journal differ