Update app.py

app.py

@@ -5,26 +5,27 @@ import logging
 import plotly.express as px
 from sklearn.ensemble import IsolationForest  # For anomaly detection
 from transformers import pipeline
+import dask.dataframe as dd  # For handling large datasets
 
 # Configure logging for debugging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Load Hugging Face summarization model
+# Preload Hugging Face summarization model at startup
+logging.info("Preloading Hugging Face model...")
 try:
-    logging.info("Attempting to load Hugging Face model...")
-    summarizer = pipeline("text2text-generation", model="google/flan-t5-base")
-    logging.info("Hugging Face model loaded successfully")
+    summarizer = pipeline("text2text-generation", model="google/flan-t5-base", device=0 if gr.cuda.is_available() else -1)  # Use GPU if available
+    logging.info("Hugging Face model preloaded successfully")
 except Exception as e:
-    logging.error(f"Failed to load model: {str(e)}")
+    logging.error(f"Failed to preload model: {str(e)}")
     raise e
 
 # Format summary prompt and generate report
-def summarize_logs(df):
+def summarize_logs(df, progress=gr.Progress()):
+    progress(0.1, "Generating summary report...")
     try:
         total_devices = df["device_id"].nunique()
         avg_uptime = "97%"  # Placeholder
         most_used = df.groupby("device_id")["usage_hours"].sum().idxmax() if not df.empty else "N/A"
-        downtime_events = 3  # Placeholder
 
         prompt = (
             f"Summarize maintenance and usage logs. "
@@ -38,15 +39,21 @@ def summarize_logs(df):
         logging.error(f"Summary generation failed: {str(e)}")
         return "Failed to generate summary."
 
-# Anomaly Detection using Isolation Forest
-def detect_anomalies(df):
+# Anomaly Detection using Isolation Forest with sampling for large datasets
+def detect_anomalies(df, progress=gr.Progress()):
+    progress(0.4, "Detecting anomalies...")
     try:
         if "usage_hours" not in df.columns or "downtime" not in df.columns:
             logging.warning("Required columns for anomaly detection not found")
             return "Anomaly detection requires 'usage_hours' and 'downtime' columns."
 
+        # Sample data if too large
+        if len(df) > 10000:  # Adjust threshold as needed
+            df = df.sample(n=10000, random_state=42)
+            logging.info("Sampled data for anomaly detection to 10,000 rows")
+
         features = df[["usage_hours", "downtime"]].fillna(0)
-        iso_forest = IsolationForest(contamination=0.1, random_state=42)
+        iso_forest = IsolationForest(contamination=0.1, random_state=42, n_jobs=-1)  # Use all CPU cores
         df["anomaly"] = iso_forest.fit_predict(features)
         
         anomalies = df[df["anomaly"] == -1][["device_id", "usage_hours", "downtime", "timestamp"]]
@@ -68,13 +75,14 @@ def detect_anomalies(df):
         return f"Anomaly detection failed: {str(e)}"
 
 # AMC Reminders based on device and AMC date
-def check_amc_reminders(df, current_date):
+def check_amc_reminders(df, current_date, progress=gr.Progress()):
+    progress(0.6, "Checking AMC reminders...")
     try:
         if "device_id" not in df.columns or "amc_date" not in df.columns:
             logging.warning("Required columns for AMC reminders not found")
             return "AMC reminders require 'device_id' and 'amc_date' columns."
 
-        df["amc_date"] = pd.to_datetime(df["amc_date"])
+        df["amc_date"] = pd.to_datetime(df["amc_date"], errors='coerce')
         current_date = pd.to_datetime(current_date)
         
         df["days_to_amc"] = (df["amc_date"] - current_date).dt.days
@@ -96,7 +104,8 @@ def check_amc_reminders(df, current_date):
         return f"AMC reminder generation failed: {str(e)}"
 
 # Dashboard Insights (AI-generated executive-level insights)
-def generate_dashboard_insights(df):
+def generate_dashboard_insights(df, progress=gr.Progress()):
+    progress(0.8, "Generating dashboard insights...")
     try:
         total_devices = df["device_id"].nunique()
         avg_usage = df["usage_hours"].mean() if "usage_hours" in df.columns else 0
@@ -112,10 +121,15 @@ def generate_dashboard_insights(df):
         return f"Dashboard insights generation failed: {str(e)}"
 
 # Create a bar chart for usage hours per device
-def create_usage_chart(df):
+def create_usage_chart(df, progress=gr.Progress()):
+    progress(0.9, "Creating usage chart...")
     try:
+        # Limit the data for chart to top 10 devices to reduce load
         usage_data = df.groupby("device_id")["usage_hours"].sum().reset_index()
-        # Define a custom color palette for multiple colors
+        if len(usage_data) > 10:
+            usage_data = usage_data.nlargest(10, "usage_hours")
+            logging.info("Limited chart data to top 10 devices")
+
         custom_colors = ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4']
         fig = px.bar(
             usage_data,
@@ -123,8 +137,8 @@ def create_usage_chart(df):
             y="usage_hours",
             title="Usage Hours per Device",
             labels={"device_id": "Device ID", "usage_hours": "Usage Hours"},
-            color="device_id",  # Color by device_id to assign different colors
-            color_discrete_sequence=custom_colors  # Use custom color palette
+            color="device_id",
+            color_discrete_sequence=custom_colors
         )
         fig.update_layout(
             title_font_size=16,
@@ -139,8 +153,9 @@ def create_usage_chart(df):
         return None
 
 # Main Gradio function
-def process_logs(file_obj):
+def process_logs(file_obj, progress=gr.Progress()):
     try:
+        progress(0, "Starting file processing...")
         if file_obj is None:
             logging.warning("No file uploaded, returning empty results")
             return "No file uploaded.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated."
@@ -152,11 +167,19 @@ def process_logs(file_obj):
             logging.error("Unsupported file format")
             return "Unsupported file format. Please upload a CSV file.", None, None, None, None, None
 
-        df = pd.read_csv(file_name)
-        logging.info(f"File loaded successfully with {len(df)} rows")
+        # Use Dask for large CSV files to read in chunks
+        progress(0.05, "Loading CSV file...")
+        try:
+            ddf = dd.from_pandas(pd.read_csv(file_name), npartitions=4)  # Split into 4 partitions
+            df = ddf.compute()  # Convert back to pandas after processing
+            logging.info(f"File loaded successfully with {len(df)} rows")
+        except Exception as e:
+            logging.error(f"Failed to load CSV: {str(e)}")
+            return f"Failed to load CSV: {str(e)}", None, None, None, None, None
 
+        progress(0.1, "Converting timestamps...")
         try:
-            df["timestamp"] = pd.to_datetime(df["timestamp"])
+            df["timestamp"] = pd.to_datetime(df["timestamp"], errors='coerce')
         except Exception as e:
             logging.error(f"Date conversion failed: {str(e)}")
             return f"Failed to convert timestamp to datetime: {str(e)}", None, None, None, None, None
@@ -166,9 +189,11 @@ def process_logs(file_obj):
             return "No data available in the file.", "No data to preview.", None, "No anomalies detected.", "No AMC reminders.", "No insights generated."
 
         # Step 1: Summary Report
-        summary = f"**Step 1: Summary Report**\n{summarize_logs(df)}\n---\n"
+        progress(0.2, "Generating summary...")
+        summary = f"**Step 1: Summary Report**\n{summarize_logs(df, progress)}\n---\n"
 
         # Step 2: Log Preview
+        progress(0.3, "Previewing logs...")
         if not df.empty:
             preview_lines = ["**Step 2: Log Preview (First 5 Rows)**"]
             for idx, row in df.head().iterrows():
@@ -184,23 +209,24 @@ def process_logs(file_obj):
             preview = "**Step 2: Log Preview**\nNo data available.\n---\n"
 
         # Step 3: Usage Chart
-        chart = create_usage_chart(df)
+        chart = create_usage_chart(df, progress)
 
         # Step 4: Anomaly Detection
-        anomalies = f"**Step 3: Anomaly Detection**\n{detect_anomalies(df)}\n---\n"
+        anomalies = f"**Step 3: Anomaly Detection**\n{detect_anomalies(df, progress)}\n---\n"
 
         # Step 5: AMC Reminders
-        amc_reminders = f"**Step 4: AMC Reminders**\n{check_amc_reminders(df, datetime.now())}\n---\n"
+        amc_reminders = f"**Step 4: AMC Reminders**\n{check_amc_reminders(df, datetime.now(), progress)}\n---\n"
 
-        # Step 6: Dashboard Insights
-        insights = f"**Step 5: Dashboard Insights (AI)**\n{generate_dashboard_insights(df)}\n---\n"
+        # Stepabba 6: Dashboard Insights
+        insights = f"**Step 5: Dashboard Insights (AI)**\n{generate_dashboard_insights(df, progress)}\n---\n"
 
+        progress(1.0, "Processing complete!")
         return summary, preview, chart, anomalies, amc_reminders, insights
     except Exception as e:
         logging.error(f"Failed to process file: {str(e)}")
         return f"Failed to process file: {str(e)}", None, None, None, None, None
 
-# Gradio Interface with Step-by-Step Layout
+# Gr{dimadio Interface with Step-by-Step Layout
 try:
     logging.info("Initializing Gradio Blocks interface...")
     with gr.Blocks(css="""
@@ -208,7 +234,7 @@ try:
         .dashboard-title {font-size: 24px; font-weight: bold; margin-bottom: 10px;}
         .dashboard-section {margin-bottom: 15px;}
         .dashboard-section h3 {font-size: 18px; margin-bottom: 5px;}
-        .dashboard-section p {margin: 2px 0;}  /* Reduce spacing between lines */
+        .dashboard-section p {margin: 2px 0;}
     """) as iface:
         gr.Markdown("<h1>LabOps Log Analyzer Dashboard (Hugging Face AI)</h1>")
         gr.Markdown("Upload a CSV file containing lab equipment logs to analyze usage.")