Update app.py

app.py

@@ -5,10 +5,6 @@ import logging
 import plotly.express as px
 import plotly.graph_objects as go
 from sklearn.ensemble import IsolationForest
-from transformers import pipeline
-import torch
-from concurrent.futures import ThreadPoolExecutor
-from simple_salesforce import Salesforce
 import os
 import io
 import time
@@ -17,19 +13,6 @@ import asyncio
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Salesforce configuration
-try:
-    sf = Salesforce(
-        username='multi-devicelabopsdashboard@sathkrutha.com',
-        password='Team@1234',
-        security_token=os.getenv('SF_SECURITY_TOKEN', ''),
-        domain='login'
-    )
-    logging.info("Salesforce connection established")
-except Exception as e:
-    logging.error(f"Failed to connect to Salesforce: {str(e)}")
-    sf = None
-
 # Try to import reportlab
 try:
     from reportlab.lib.pagesizes import letter
@@ -42,120 +25,15 @@ except ImportError:
     logging.warning("reportlab module not found. PDF generation disabled.")
     reportlab_available = False
 
-# Preload Hugging Face model with optimization
-logging.info("Preloading Hugging Face model...")
-try:
-    device = 0 if torch.cuda.is_available() else -1
-    summarizer = pipeline(
-        "summarization",
-        model="t5-small",
-        device=device,
-        max_length=20,  # Further reduced for speed
-        min_length=10,
-        num_beams=2
-    )
-    logging.info(f"Hugging Face model preloaded on {'GPU' if device == 0 else 'CPU'}")
-except Exception as e:
-    logging.error(f"Failed to preload model: {str(e)}")
-    raise e
-
-# Cache picklist values at startup
-def get_picklist_values(field_name):
-    if sf is None:
-        return []
-    try:
-        obj_desc = sf.SmartLog__c.describe()
-        for field in obj_desc['fields']:
-            if field['name'] == field_name:
-                return [value['value'] for value in field['picklistValues'] if value['active']]
-        return []
-    except Exception as e:
-        logging.error(f"Failed to fetch picklist values for {field_name}: {str(e)}")
-        return []
-
-status_values = get_picklist_values('Status__c') or ["Active", "Inactive", "Pending"]
-log_type_values = get_picklist_values('Log_Type__c') or ["Smart Log", "Cell Analysis", "UV Verification"]
-logging.info(f"Valid Status__c values: {status_values}")
-logging.info(f"Valid Log_Type__c values: {log_type_values}")
-
-# Map invalid picklist values
-picklist_mapping = {
-    'Status__c': {
-        'normal': 'Active',
-        'error': 'Inactive',
-        'warning': 'Pending',
-        'ok': 'Active',
-        'failed': 'Inactive'
-    },
-    'Log_Type__c': {
-        'maint': 'Smart Log',
-        'error': 'Cell Analysis',
-        'ops': 'UV Verification',
-        'maintenance': 'Smart Log',
-        'cell': 'Cell Analysis',
-        'uv': 'UV Verification',
-        'weight log': 'Smart Log'
-    }
-}
-
-# Save to Salesforce
-def save_to_salesforce(df, reminders_df):
-    if sf is None:
-        return
-    try:
-        current_date = datetime.now()
-        next_30_days = current_date + timedelta(days=30)
-        records = []
-        reminder_device_ids = set(reminders_df['device_id']) if not reminders_df.empty else set()
-
-        for _, row in df.iterrows():
-            status = str(row['status'])
-            log_type = str(row['log_type'])
-            status = picklist_mapping['Status__c'].get(status.lower(), status_values[0] if status_values else None)
-            log_type = picklist_mapping['Log_Type__c'].get(log_type.lower(), log_type_values[0] if log_type_values else None)
-            if status is None or log_type is None:
-                continue
-
-            amc_date_str = None
-            if pd.notna(row['amc_date']):
-                try:
-                    amc_date = pd.to_datetime(row['amc_date']).strftime('%Y-%m-%d')
-                    amc_date_dt = datetime.strptime(amc_date, '%Y-%m-%d')
-                    if status == "Active" and current_date.date() <= amc_date_dt.date() <= next_30_days.date():
-                        logging.info(f"AMC Reminder for Device ID {row['device_id']}")
-                    amc_date_str = amc_date
-                except:
-                    amc_date_str = None
-
-            record = {
-                'Device_Id__c': str(row['device_id'])[:50],
-                'Log_Type__c': log_type,
-                'Status__c': status,
-                'Timestamp__c': row['timestamp'].isoformat() if pd.notna(row['timestamp']) else None,
-                'Usage_Hours__c': float(row['usage_hours']) if pd.notna(row['usage_hours']) else 0.0,
-                'Downtime__c': float(row['downtime']) if pd.notna(row['downtime']) else 0.0,
-                'AMC_Date__c': amc_date_str
-            }
-            if row['device_id'] not in reminder_device_ids:
-                records.append(record)
-
-        if records:
-            sf.bulk.SmartLog__c.insert(records)
-            logging.info(f"Saved {len(records)} records to Salesforce")
-    except Exception as e:
-        logging.error(f"Failed to save to Salesforce: {str(e)}")
-
-# Summarize logs
+# Summarize logs (without Hugging Face)
 def summarize_logs(df):
     try:
         total_devices = df["device_id"].nunique()
-        most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
-        prompt = f"Logs: {total_devices} devices. Most used: {most_used}."
-        summary = summarizer(prompt, max_length=20, min_length=10, do_sample=False)[0]["summary_text"]
-        return summary
+        total_usage = df["usage_hours"].sum() if "usage_hours" in df.columns else 0
+        return f"{total_devices} devices processed with {total_usage:.2f} total usage hours."
     except Exception as e:
         logging.error(f"Summary generation failed: {str(e)}")
-        return f"Failed to generate summary: {str(e)}"
+        return "Failed to generate summary."
 
 # Anomaly detection
 def detect_anomalies(df):
@@ -163,8 +41,8 @@ def detect_anomalies(df):
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns.", pd.DataFrame()
         features = df[["usage_hours", "downtime"]].fillna(0)
-        if len(features) > 100:  # Further reduced sample size
-            features = features.sample(n=100, random_state=42)
+        if len(features) > 50:  # Reduced sample size
+            features = features.sample(n=50, random_state=42)
         iso_forest = IsolationForest(contamination=0.1, random_state=42)
         df["anomaly"] = iso_forest.fit_predict(features)
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
@@ -191,17 +69,15 @@ def check_amc_reminders(df, current_date):
         logging.error(f"AMC reminder generation failed: {str(e)}")
         return f"AMC reminder generation failed: {str(e)}", pd.DataFrame()
 
-# Dashboard insights
+# Dashboard insights (without Hugging Face)
 def generate_dashboard_insights(df):
     try:
         total_devices = df["device_id"].nunique()
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
-        prompt = f"Insights: {total_devices} devices, avg usage {avg_usage:.2f} hours."
-        insights = summarizer(prompt, max_length=20, min_length=10, do_sample=False)[0]["summary_text"]
-        return insights
+        return f"{total_devices} devices with average usage of {avg_usage:.2f} hours."
     except Exception as e:
         logging.error(f"Dashboard insights generation failed: {str(e)}")
-        return f"Dashboard insights generation failed: {str(e)}"
+        return "Failed to generate insights."
 
 # Placeholder chart for empty data
 def create_placeholder_chart(title):
@@ -265,7 +141,7 @@ def create_daily_log_trends_chart(df):
         if df.empty or "timestamp" not in df.columns:
             logging.warning("Insufficient data for daily log trends chart")
             return create_placeholder_chart("Daily Log Trends")
-        df['date'] = pd.to_datetime(df['timestamp']).dt.date
+        df['date'] = pd.to_datetime(df['timestamp'], errors='coerce').dt.date
         daily_logs = df.groupby('date').size().reset_index(name='log_count')
         if daily_logs.empty:
             return create_placeholder_chart("Daily Log Trends")
@@ -288,8 +164,8 @@ def create_weekly_uptime_chart(df):
         if df.empty or "timestamp" not in df.columns or "usage_hours" not in df.columns or "downtime" not in df.columns:
             logging.warning("Insufficient data for weekly uptime chart")
             return create_placeholder_chart("Weekly Uptime Percentage")
-        df['week'] = pd.to_datetime(df['timestamp']).dt.isocalendar().week
-        df['year'] = pd.to_datetime(df['timestamp']).dt.year
+        df['week'] = pd.to_datetime(df['timestamp'], errors='coerce').dt.isocalendar().week
+        df['year'] = pd.to_datetime(df['timestamp'], errors='coerce').dt.year
         weekly_data = df.groupby(['year', 'week']).agg({
             'usage_hours': 'sum',
             'downtime': 'sum'
@@ -317,7 +193,7 @@ def create_anomaly_alerts_chart(anomalies_df):
         if anomalies_df is None or anomalies_df.empty or "timestamp" not in anomalies_df.columns:
             logging.warning("Insufficient data for anomaly alerts chart")
             return create_placeholder_chart("Anomaly Alerts Over Time")
-        anomalies_df['date'] = pd.to_datetime(anomalies_df['timestamp']).dt.date
+        anomalies_df['date'] = pd.to_datetime(anomalies_df['timestamp'], errors='coerce').dt.date
         anomaly_counts = anomalies_df.groupby('date').size().reset_index(name='anomaly_count')
         if anomaly_counts.empty:
             return create_placeholder_chart("Anomaly Alerts Over Time")
@@ -438,100 +314,94 @@ def generate_pdf_content(summary, preview_df, anomalies, amc_reminders, insights
         return None
 
 # Main processing function
-async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state):
+async def process_logs(file_obj, lab_site_filter, equipment_type_filter, date_range, last_modified_state, cached_df_state, cached_filtered_df_state):
     start_time = time.time()
     try:
         if not file_obj:
-            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state
+            return "No file uploaded.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "No anomalies detected.", "No AMC reminders.", "No insights generated.", None, last_modified_state, cached_df_state, cached_filtered_df_state
 
         file_path = file_obj.name
         current_modified_time = os.path.getmtime(file_path)
-        if last_modified_state and current_modified_time == last_modified_state and cached_df_state is not None:
-            df = cached_df_state
+        if last_modified_state and current_modified_time == last_modified_state and cached_filtered_df_state is not None:
+            filtered_df = cached_filtered_df_state
         else:
-            logging.info(f"Processing file: {file_path}")
-            if not file_path.endswith(".csv"):
-                return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state
-
-            required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
-            dtypes = {
-                "device_id": "string",
-                "log_type": "string",
-                "status": "string",
-                "usage_hours": "float32",
-                "downtime": "float32",
-                "amc_date": "string"
-            }
-            df = pd.read_csv(file_path, dtype=dtypes)
-            missing_columns = [col for col in required_columns if col not in df.columns]
-            if missing_columns:
-                return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
-
-            df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
-            df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
-            if df["timestamp"].dt.tz is None:
-                df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
-            if df.empty:
-                return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
-
-        # Apply filters
-        filtered_df = df.copy()
-        if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
-            filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
-        if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
-            filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
-        if date_range and len(date_range) == 2:
-            days_start, days_end = date_range
-            today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
-            start_date = today + pd.Timedelta(days=days_start)
-            end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
-            filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
-
-        if filtered_df.empty:
-            return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df
+            if cached_df_state is None or current_modified_time != last_modified_state:
+                logging.info(f"Processing file: {file_path}")
+                if not file_path.endswith(".csv"):
+                    return "Please upload a CSV file.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, "", "", "", None, last_modified_state, cached_df_state, cached_filtered_df_state
+
+                required_columns = ["device_id", "log_type", "status", "timestamp", "usage_hours", "downtime", "amc_date"]
+                dtypes = {
+                    "device_id": "string",
+                    "log_type": "string",
+                    "status": "string",
+                    "usage_hours": "float32",
+                    "downtime": "float32",
+                    "amc_date": "string"
+                }
+                df = pd.read_csv(file_path, dtype=dtypes)
+                missing_columns = [col for col in required_columns if col not in df.columns]
+                if missing_columns:
+                    return f"Missing columns: {missing_columns}", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, cached_filtered_df_state
+
+                df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
+                df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
+                if df["timestamp"].dt.tz is None:
+                    df["timestamp"] = df["timestamp"].dt.tz_localize('UTC').dt.tz_convert('Asia/Kolkata')
+                if df.empty:
+                    return "No data available.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, cached_filtered_df_state
+            else:
+                df = cached_df_state
+
+            # Apply filters
+            filtered_df = df.copy()
+            if lab_site_filter and lab_site_filter != 'All' and 'lab_site' in filtered_df.columns:
+                filtered_df = filtered_df[filtered_df['lab_site'] == lab_site_filter]
+            if equipment_type_filter and equipment_type_filter != 'All' and 'equipment_type' in filtered_df.columns:
+                filtered_df = filtered_df[filtered_df['equipment_type'] == equipment_type_filter]
+            if date_range and len(date_range) == 2:
+                days_start, days_end = date_range
+                today = pd.to_datetime(datetime.now().date()).tz_localize('Asia/Kolkata')
+                start_date = today + pd.Timedelta(days=days_start)
+                end_date = today + pd.Timedelta(days=days_end) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)
+                filtered_df = filtered_df[(filtered_df['timestamp'] >= start_date) & (filtered_df['timestamp'] <= end_date)]
+
+            if filtered_df.empty:
+                return "No data after applying filters.", pd.DataFrame(), None, '<p>No device cards available.</p>', None, None, None, None, None, None, None, None, last_modified_state, df, filtered_df
 
         # Generate table for preview
         preview_df = filtered_df[['device_id', 'log_type', 'status', 'timestamp', 'usage_hours', 'downtime', 'amc_date']].head(5)
         preview_html = preview_df.to_html(index=False, classes='table table-striped', border=0)
 
         # Run critical tasks concurrently
-        with ThreadPoolExecutor(max_workers=4) as executor:  # Reduced workers
-            future_summary = executor.submit(summarize_logs, filtered_df)
+        with ThreadPoolExecutor(max_workers=2) as executor:  # Reduced workers
             future_anomalies = executor.submit(detect_anomalies, filtered_df)
             future_amc = executor.submit(check_amc_reminders, filtered_df, datetime.now())
-            future_insights = executor.submit(generate_dashboard_insights, filtered_df)
-            future_usage_chart = executor.submit(create_usage_chart, filtered_df)
-            future_downtime_chart = executor.submit(create_downtime_chart, filtered_df)
-            future_daily_log_chart = executor.submit(create_daily_log_chart, filtered_df)
-            future_weekly_uptime_chart = executor.submit(create_weekly_uptime_chart, filtered_df)
-            future_device_cards = executor.submit(generate_device_cards, filtered_df)
-
-            summary = f"Step 1: Summary Report\n{future_summary.result()}"
+
+            summary = f"Step 1: Summary Report\n{summarize_logs(filtered_df)}"
             anomalies, anomalies_df = future_anomalies.result()
             anomalies = f"Anomaly Detection\n{anomalies}"
             amc_reminders, reminders_df = future_amc.result()
             amc_reminders = f"AMC Reminders\n{amc_reminders}"
-            insights = f"Dashboard Insights (AI)\n{future_insights.result()}"
-            usage_chart = future_usage_chart.result()
-            downtime_chart = future_downtime_chart.result()
-            daily_log_chart = future_daily_log_chart.result()
-            weekly_uptime_chart = future_weekly_uptime_chart.result()
-            device_cards = future_device_cards.result()
-
-        # Generate anomaly alerts chart after anomalies are computed
-        anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
+            insights = f"Dashboard Insights\n{generate_dashboard_insights(filtered_df)}"
 
-        save_to_salesforce(filtered_df, reminders_df)
+        # Generate charts sequentially to avoid contention
+        usage_chart = create_usage_chart(filtered_df)
+        downtime_chart = create_downtime_chart(filtered_df)
+        daily_log_chart = create_daily_log_trends_chart(filtered_df)
+        weekly_uptime_chart = create_weekly_uptime_chart(filtered_df)
+        anomaly_alerts_chart = create_anomaly_alerts_chart(anomalies_df)
+        device_cards = generate_device_cards(filtered_df)
 
         elapsed_time = time.time() - start_time
         logging.info(f"Processing completed in {elapsed_time:.2f} seconds")
-        if elapsed_time > 5:  # Stricter threshold
-            logging.warning(f"Processing time exceeded 5 seconds: {elapsed_time:.2f} seconds")
+        if elapsed_time > 3:  # Very strict threshold
+            logging.warning(f"Processing time exceeded 3 seconds: {elapsed_time:.2f} seconds")
 
-        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, df)
+        return (summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights, None, current_modified_time, df, filtered_df)
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
-        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state
+        return f"Error: {str(e)}", pd.DataFrame(), None, '<p>Error processing data.</p>', None, None, None, None, None, None, None, None, last_modified_state, cached_df_state, cached_filtered_df_state
 
 # Generate PDF separately
 async def generate_pdf(summary, preview_html, usage_chart, device_cards, daily_log_chart, weekly_uptime_chart, anomaly_alerts_chart, downtime_chart, anomalies, amc_reminders, insights):
@@ -576,12 +446,13 @@ try:
         .table th {background-color: #f2f2f2;}
         .table tr:nth-child(even) {background-color: #f9f9f9;}
     """) as iface:
-        gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
+        gr.Markdown("<h1>LabOps Log Analyzer Dashboard</h1>")
         gr.Markdown("Upload a CSV file to analyze. Click 'Analyze' to refresh the dashboard. Use 'Export PDF' for report download.")
 
         last_modified_state = gr.State(value=None)
         current_file_state = gr.State(value=None)
         cached_df_state = gr.State(value=None)
+        cached_filtered_df_state = gr.State(value=None)
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -625,7 +496,7 @@ try:
                         gr.Markdown("### Step 5: AMC Reminders")
                         amc_output = gr.Markdown()
                     with gr.Group(elem_classes="dashboard-section"):
-                        gr.Markdown("### Step 6: Insights (AI)")
+                        gr.Markdown("### Step 6: Insights")
                         insights_output = gr.Markdown()
                     with gr.Group(elem_classes="dashboard-section"):
                         gr.Markdown("### Export Report")
@@ -640,8 +511,8 @@ try:
 
         submit_button.click(
             fn=process_logs,
-            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state],
-            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state]
+            inputs=[file_input, lab_site_filter, equipment_type_filter, date_range_filter, last_modified_state, cached_df_state, cached_filtered_df_state],
+            outputs=[summary_output, preview_output, usage_chart_output, device_cards_output, daily_log_trends_output, weekly_uptime_output, anomaly_alerts_output, downtime_chart_output, anomaly_output, amc_output, insights_output, pdf_output, last_modified_state, cached_df_state, cached_filtered_df_state]
         )
 
         pdf_button.click(