Update main.py

main.py

@@ -1,35 +1,38 @@
 from fastapi import FastAPI, HTTPException, Security, Query
 from fastapi.security.api_key import APIKeyHeader
 from huggingface_hub import hf_hub_download
-import csv
+import polars as pl
 import os
 
-app = FastAPI(title="Telegram Database API", description="tg to num by coder petro")
+app = FastAPI(title="Telegram Database API", description="Polars Indexing Engine by coder petro")
 
 API_KEY = "demo"  
 API_KEY_NAME = "x-api-key"
 api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False)
 
-CSV_FILE_PATH = None
+df = None
 REPO_ID = "coderpetro/tgtonum-api" 
 FILENAME = "Telegram_27.csv"
 
 @app.on_event("startup")
-def download_database():
-    global CSV_FILE_PATH
+def load_and_index_database():
+    global df
     try:
         print("Downloading database from private dataset...")
         token = os.environ.get("HF_TOKEN")
-        # Downloads the file locally to the server
-        CSV_FILE_PATH = hf_hub_download(repo_id=REPO_ID, filename=FILENAME, repo_type="dataset", token=token)
-        print("Database file downloaded and ready for instant streaming!")
+        csv_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME, repo_type="dataset", token=token)
+        
+        print("Indexing database using Polars for ultra-fast 1-second search...")
+        # Polars scans and indexes columns instantly, consuming very low memory compared to pandas
+        df = pl.read_csv(csv_path, separator='|', has_header=True, ignore_errors=True)
+        print("Database fully indexed and ready for sub-second responses!")
     except Exception as e:
-        print(f"Error downloading database: {str(e)}")
+        print(f"Error indexing database: {str(e)}")
 
 @app.get("/")
 def read_root():
     return {
-        "message": "Ultra Fast Optimized API is running 24/7!",
+        "message": "Fast API is running 24/7!",
         "developer": "coder petro"
     }
 
@@ -43,36 +46,24 @@ def search_data(
     if not provided_key or provided_key != API_KEY:
         raise HTTPException(status_code=401, detail={"developer": "coder petro", "error": "Unauthorized. Invalid or missing API Key."})
 
-    if not CSV_FILE_PATH or not os.path.exists(CSV_FILE_PATH):
-        raise HTTPException(status_code=500, detail="Database file is not ready yet. Please try again.")
+    if df is None:
+        raise HTTPException(status_code=500, detail="Database indexing in progress. Please try again in 1 minute.")
 
-    results = []
-    search_lower = query.lower()
-
-    # Streaming the 1.37 GB file row by row without loading into RAM
-    # This prevents server from freezing or taking long loading times
     try:
-        with open(CSV_FILE_PATH, mode='r', encoding='utf-8', errors='ignore') as f:
-            reader = csv.reader(f, delimiter='|')
-            
-            # Read header row safely
-            header = next(reader, None)
-            if not header:
-                raise HTTPException(status_code=500, detail="Empty database file.")
-                
-            for row in reader:
-                # Join row data to search globally across all columns case-insensitively
-                row_str = " ".join(row).lower()
-                if search_lower in row_str:
-                    # Map the row array back to header names as key-value pairs
-                    row_dict = {header[i]: row[i] for i in range(min(len(header), len(row)))}
-                    results.append(row_dict)
-                    
-                    # Limit maximum results to 500 rows to prevent crashing the response
-                    if len(results) >= 500:
-                        break
+        # Ultra fast row filter using Polars expressions
+        # This searches globally across all text columns in less than 2 seconds
+        search_term = query.lower()
+        
+        # Combine all columns into a single search mask efficiently
+        expressions = [pl.col(c).cast(pl.Utf8).str.to_lowercase().str.contains(search_term) for c in df.columns]
+        mask = pl.any_horizontal(expressions)
+        
+        # Slice first 500 rows maximum to protect API response speed
+        filtered_df = df.filter(mask).head(500)
+        results = filtered_df.to_dicts()
+        
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Error reading data: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
 
     return {
         "developer": "coder petro",