File size: 7,955 Bytes
a1fb95c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
import pandas as pd
import numpy as np
import re
from src.model import TransactionClassifier
from src.logger import logger

# Initialize AI Brain
ai_brain = TransactionClassifier()

def clean_statement_csv(file_path):
    """
    Super-Robust Cleaner: Finds the actual table using fuzzy matching, 
    fixes headers, and cleans numbers.
    """
    logger.info(f"Cleaning file: {file_path}")
    try:
        # Read without headers initially to capture everything
        df = pd.read_csv(file_path, header=None)
    except Exception as e:
        logger.error(f"Could not read CSV: {e}")
        return None

    # --- 1. SUPER-ROBUST HEADER DETECTION ---
    header_idx = -1
    for idx, row in df.iterrows():
        # Convert row to a single lowercase string (e.g., "date description withdrawal")
        row_text = " ".join(row.astype(str)).lower()
        
        # Check if it contains 'date' AND ('description' OR 'particulars')
        has_date = 'date' in row_text
        has_desc = any(x in row_text for x in ['description', 'particulars', 'narration', 'details'])
        
        if has_date and has_desc:
            header_idx = idx
            logger.debug(f"Found Header at Row {idx}: {row_text[:50]}...")
            break
    
    if header_idx != -1:
        # Set this row as the header
        df.columns = df.iloc[header_idx].astype(str).str.strip()
        df = df.iloc[header_idx+1:].reset_index(drop=True)
    else:
        logger.warning("Header not found. Using fallback column names.")
        df.columns = [f"Col_{i}" for i in range(len(df.columns))]

    # --- 2. SMART COLUMN MAPPING ---
    col_map = {}
    for col in df.columns:
        c = col.lower()
        if 'date' in c: col_map[col] = 'Date'
        elif any(x in c for x in ['description', 'particulars', 'narration', 'details']): col_map[col] = 'Description'
        # Check for 'dr' (debit) or 'withdrawal'
        elif any(x in c for x in ['debit', 'withdrawal', 'dr.']): col_map[col] = 'Withdrawal'
        # Check for 'cr' (credit) or 'deposit'
        elif any(x in c for x in ['credit', 'deposit', 'cr.']): col_map[col] = 'Deposit'

    df.rename(columns=col_map, inplace=True)

    # --- 3. SAFETY NET: Ensure Columns Exist ---
    if 'Withdrawal' not in df.columns: df['Withdrawal'] = 0.0
    if 'Deposit' not in df.columns: df['Deposit'] = 0.0
    
    # If Description is missing, pick the column with the longest text
    if 'Description' not in df.columns:
        desc_col = df.astype(str).apply(lambda x: x.str.len()).mean().idxmax()
        df.rename(columns={desc_col: 'Description'}, inplace=True)

    # --- 4. ROBUST NUMBER CLEANING ---
    def clean_money(val):
        if pd.isna(val) or val == '': return 0.0
        s = str(val).replace(',', '').strip()
        # Remove everything that isn't a digit or a dot
        s = re.sub(r'[^\d.]', '', s)
        try:
            return float(s)
        except:
            return 0.0

    df['Withdrawal'] = df['Withdrawal'].apply(clean_money)
    df['Deposit'] = df['Deposit'].apply(clean_money)

    # --- 5. REMOVE JUNK ROWS ---
    # Keep rows where at least one money column has a value > 0
    original_count = len(df)
    df = df[(df['Withdrawal'] > 0) | (df['Deposit'] > 0)]
    logger.info(f"Extracted {len(df)} valid transactions from {original_count} raw rows.")
    
    return df.reset_index(drop=True)

def tag_transactions(df):
    """Tags transactions using Regex Rules + AI Prediction + Pseudo-RAG Knowledge Base."""
    logger.info("Classifying transactions via Cognitive Pipeline...")
    
    # 1. Load Knowledge Base
    import json
    import os
    kb_path = os.path.join("data", "tax_knowledge.json")
    kb = {}
    if os.path.exists(kb_path):
        with open(kb_path, 'r') as f:
            kb = json.load(f)

    def get_category(desc):
        d = str(desc).upper()
        # Fast Regex Tier
        if any(x in d for x in ["AMITY", "LIC", "PPF", "TUITION", "SCHOOL"]): return "80C_INVEST"
        if any(x in d for x in ["AKSHAT", "NEFT", "IMPS", "CREDIT", "UPI-RECEIVE"]): return "BUSINESS_REV"
        if any(x in d for x in ["SALARY", "ACH CREDIT", "TCS", "WIPRO"]): return "SALARY"
        if any(x in d for x in ["SWIGGY", "NETFLIX", "ZOMATO", "UBER", "STARBUCKS", "STEAM"]): return "PERSONAL"
        if any(x in d for x in ["ZERODHA", "GROWW", "UPSTOX", "KITE"]): return "STCG"
        if any(x in d for x in ["SELF", "OWN A/C", "TRANSFER TO"]): return "SELF_TRANSFER"
        return "UNCATEGORIZED"

    # Initialize columns if they don't exist
    if 'Tax_Category' not in df.columns:
        df['Tax_Category'] = "UNCATEGORIZED"
    if 'Confidence_Score' not in df.columns:
        df['Confidence_Score'] = 0.0
    if 'Source' not in df.columns:
        df['Source'] = "Unknown"

    # Only process rows that are still UNCATEGORIZED
    mask = df['Tax_Category'] == 'UNCATEGORIZED'
    
    # 2. Heuristic Tier for Uncategorized
    df.loc[mask, 'Tax_Category'] = df.loc[mask, 'Description'].apply(get_category)
    df.loc[mask, 'Confidence_Score'] = 1.0
    df.loc[mask, 'Source'] = "Heuristic"

    # 3. Sequential Inference Tier (for items still uncategorized after regex)
    for idx, row in df[df['Tax_Category'] == 'UNCATEGORIZED'].iterrows():
        desc = str(row['Description'])
        
        # Tier A: Neural Prediction
        cat, conf = ai_brain.predict(desc)
        if conf > 0.85:
            df.at[idx, 'Tax_Category'] = cat
            df.at[idx, 'Confidence_Score'] = conf
            df.at[idx, 'Source'] = "Neural"
        else:
            # Tier B: Pseudo-RAG Semantic Search
            rag_cat, rag_sim = ai_brain.semantic_search(desc, kb)
            if rag_sim > 0.70:
                df.at[idx, 'Tax_Category'] = rag_cat
                df.at[idx, 'Confidence_Score'] = rag_sim
                df.at[idx, 'Source'] = "Knowledge-Augmented"
    
    # 4. Confidence Calibration (Entropy) for all AI-tagged rows
    ai_mask = df['Source'].isin(["Neural", "Knowledge-Augmented"])
    for idx, row in df[ai_mask].iterrows():
        desc = str(row['Description'])
        entropy = ai_brain.calculate_entropy(desc)
        if entropy > 1.5:  # High uncertainty threshold
            df.at[idx, 'Source'] += " (Uncertain)"

    logger.info(f"Classification complete. Sources: {df['Source'].value_counts().to_dict()}")
    return df

def train_model(texts, labels):
    """
    Exposes the training function to the GUI.
    texts: list of description strings
    labels: list of category strings
    """
    if not texts or not labels: return
    logger.info(f"Training AI with {len(texts)} new examples...")
    ai_brain.train(texts, labels)
    logger.info("AI Training Complete & Saved.")

def interactive_classification(df):
    """Human-in-the-Loop Training."""
    unknowns = df[df['Tax_Category'] == 'UNCATEGORIZED']
    
    if unknowns.empty:
        return df

    print(f"\n--- 🤖 Human Help Needed ({len(unknowns)} items) ---")
    
    TAX_CATS = {
        "1": "BUSINESS_REV", "2": "BUSINESS_EXP", "3": "SALARY",
        "4": "80C_INVEST", "5": "STCG", "6": "LTCG", 
        "7": "PERSONAL", "8": "SELF_TRANSFER"
    }
    
    updates = {}
    
    # Ask for max 5 items
    unique_desc = unknowns['Description'].unique()
    for desc in unique_desc[:5]: 
        print(f" ? What is: '{str(desc)[:50]}...'?")
        print("   [1] Biz Inc [2] Biz Exp [3] Salary [4] 80C")
        print("   [5] STCG    [6] LTCG    [7] Person [8] Self")
        
        choice = input(" > Choice: ").strip()
        if choice in TAX_CATS:
            selected_cat = TAX_CATS[choice]
            updates[desc] = selected_cat
            ai_brain.train([desc], [selected_cat])
            logger.info(f"User trained AI: {desc[:20]}... -> {selected_cat}")

    for idx, row in df.iterrows():
        if row['Description'] in updates:
            df.at[idx, 'Tax_Category'] = updates[row['Description']]
            
    return df