Update app.py

app.py

@@ -31,17 +31,23 @@ def drop_unnamed_columns(df):
     return df.loc[:, ~df.columns.str.contains("Unnamed", case=False, na=False)]
 
 def flatten_columns(df):
-    """If the DataFrame has a MultiIndex for columns, flatten it."""
+    """
+    If the DataFrame has a MultiIndex for columns, flatten it.
+    This version joins non-empty levels with a space.
+    """
     if isinstance(df.columns, pd.MultiIndex):
-        df.columns = [' '.join([str(x) for x in col if x is not None and x != 'nan']).strip() 
-                      for col in df.columns.values]
+        new_cols = []
+        for col in df.columns:
+            parts = [str(x).strip() for x in col if pd.notna(x) and str(x).strip() not in ["", "nan"]]
+            new_cols.append(" ".join(parts))
+        df.columns = new_cols
     return df
 
 def read_excel_file(file, header_option=0):
     """
     Read an Excel file using the given header option,
-    flatten and normalize its columns,
-    drop columns whose header contains 'Unnamed',
+    flatten multi-index columns, normalize column names,
+    drop columns with 'Unnamed' in their header,
     and drop completely empty rows/columns.
     """
     df = pd.read_excel(file, header=header_option)
@@ -54,7 +60,8 @@ def read_excel_file(file, header_option=0):
 
 def get_column(df, possible_names):
     """
-    Return the first column name found in df.columns matching any name in possible_names (case-insensitive).
+    Return the first matching column name from df.columns (case-insensitive)
+    that appears in possible_names.
     """
     lower_cols = {col.lower(): col for col in df.columns}
     for name in possible_names:
@@ -62,19 +69,24 @@ def get_column(df, possible_names):
             return lower_cols[name.lower()]
     return None
 
-# --- Processing Functions for Each File ---
+# --- Processing Functions ---
 def process_employee_data(df):
-    """Process employee data: ensure a proper TIN and build Employee Name if needed."""
+    """Process employee data: build Employee Name (if needed) and standardize TIN."""
     df = normalize_columns(df)
-    # If "Employee Name" is missing or all blank, try to build it.
+    
+    # Build Employee Name if not present or all blank.
     if 'Employee Name' not in df.columns or df['Employee Name'].isna().all():
         first_name = get_column(df, ["First Name", "First", "Forename"])
         last_name  = get_column(df, ["Last Name", "Surname", "Family Name", "Last"])
         if first_name and last_name:
             df["Employee Name"] = df[first_name].apply(clean_name) + " " + df[last_name].apply(clean_name)
-    # Standardize TIN from either 'TIN' or 'Personal ID of Employee'
-    if 'TIN' in df.columns:
-        df['TIN'] = df['TIN'].apply(standardize_tin)
+    
+    # Standardize TIN: look for "TIN" first, else "Personal ID of Employee"
+    tin_col = get_column(df, ["TIN"])
+    if tin_col:
+        df['TIN'] = df[tin_col].apply(standardize_tin)
+        if tin_col != "TIN":
+            df.rename(columns={tin_col: "TIN"}, inplace=True)
     else:
         alt = get_column(df, ["Personal ID of Employee"])
         if alt:
@@ -85,10 +97,14 @@ def process_employee_data(df):
     return df
 
 def process_salary_data(df):
-    """Process the salary (earnings) data (the master): standardize TIN and convert earnings figures to numeric."""
+    """Process the salary (earnings) data (master): standardize TIN and convert earnings figures to numeric."""
     df = normalize_columns(df)
-    if 'TIN' in df.columns:
-        df['TIN'] = df['TIN'].apply(standardize_tin)
+    # Use get_column to find TIN (case-insensitive)
+    tin_col = get_column(df, ["TIN"])
+    if tin_col:
+        df['TIN'] = df[tin_col].apply(standardize_tin)
+        if tin_col != "TIN":
+            df.rename(columns={tin_col: "TIN"}, inplace=True)
     else:
         alt = get_column(df, ["TIN or Personal ID of Employee"])
         if alt:
@@ -96,6 +112,8 @@ def process_salary_data(df):
             df.drop(columns=[alt], inplace=True)
         else:
             raise KeyError("Salary data must contain a TIN column.")
+    
+    # Convert non-key columns to numeric.
     ignore = {'TIN', 'Employee Name', 'Currency'}
     for col in df.columns:
         if col not in ignore:
@@ -105,8 +123,11 @@ def process_salary_data(df):
 def process_paye_data(df):
     """Process PAYE data: standardize TIN and convert other columns to numeric."""
     df = normalize_columns(df)
-    if 'TIN' in df.columns:
-        df['TIN'] = df['TIN'].apply(standardize_tin)
+    tin_col = get_column(df, ["TIN"])
+    if tin_col:
+        df['TIN'] = df[tin_col].apply(standardize_tin)
+        if tin_col != "TIN":
+            df.rename(columns={tin_col: "TIN"}, inplace=True)
     else:
         alt = get_column(df, ["TIN or Personal ID of Employee"])
         if alt:
@@ -114,6 +135,7 @@ def process_paye_data(df):
             df.drop(columns=[alt], inplace=True)
         else:
             raise KeyError("PAYE data must contain a TIN column.")
+    
     ignore = {'TIN', 'Employed From date', 'Employed To date'}
     for col in df.columns:
         if col not in ignore:
@@ -123,17 +145,16 @@ def process_paye_data(df):
 # --- Merging Function ---
 def merge_dataframes(employee_df, salary_df, paye_df):
     """
-    Use the salary file as the master.
+    Use the salary file as master.
     Left-join key employee fields onto salary data,
     then left-join PAYE data.
-    Finally, drop any remaining columns whose header contains 'Unnamed'.
+    Drop any remaining "Unnamed" columns.
     """
-    # Restrict employee data to key columns (if available).
-    key_emp = ['TIN', 'Employee Name', 'First Name', 'Middle Name', 'Last Name',
-               'Birth Date', 'Employed From date', 'Employed To date', 'Position']
+    key_emp = ['TIN', 'Employee Name', 'First Name', 'Middle Name', 
+               'Last Name', 'Birth Date', 'Employed From date', 
+               'Employed To date', 'Position']
     emp_subset = employee_df[[col for col in key_emp if col in employee_df.columns]].copy()
     
-    # Merge employee data (using a left join onto the salary file).
     merged_df = salary_df.merge(emp_subset, on='TIN', how='left', suffixes=('', '_emp'))
     for col in ['Employee Name', 'First Name', 'Middle Name', 'Last Name', 
                 'Birth Date', 'Employed From date', 'Employed To date', 'Position']:
@@ -142,9 +163,7 @@ def merge_dataframes(employee_df, salary_df, paye_df):
             merged_df[col] = merged_df[col].combine_first(merged_df[alt])
             merged_df.drop(columns=[alt], inplace=True)
     
-    # Merge PAYE data.
     merged_df = merged_df.merge(paye_df, on='TIN', how='left', suffixes=('', '_paye'))
-    # Combine any duplicate columns from PAYE.
     for col in list(merged_df.columns):
         if col.endswith('_paye'):
             base = col[:-5]
@@ -154,13 +173,9 @@ def merge_dataframes(employee_df, salary_df, paye_df):
                 merged_df.rename(columns={col: base}, inplace=True)
             merged_df.drop(columns=[col], inplace=True)
     
-    # Drop any columns with 'Unnamed' in the header.
     merged_df = drop_unnamed_columns(merged_df)
-    
-    # For numeric columns, fill remaining NaNs with 0.
     num_cols = merged_df.select_dtypes(include=[np.number]).columns
     merged_df[num_cols] = merged_df[num_cols].fillna(0)
-    
     return merged_df
 
 # --- Main (Streamlit UI) ---
@@ -170,24 +185,27 @@ def main():
 **Upload the following files:**
 
 1. **Employee Information File** (single header row)
-2. **Salary (Earnings) Information File** – this file is the master; if it has two header rows (e.g. one for metric and one for currency), it will be read accordingly.
+2. **Salary (Earnings) Information File** – this file is the master.
 3. **PAYE Information File** (single header row)
+
+*If your salary file has two header rows (e.g. one for metrics and one for currency),
+it will be read with header_option=[0,1].*
     """)
     
     employee_file = st.file_uploader("Upload Employee File", type=['xlsx', 'xls'])
-    # For the salary file, we use header_option=[0,1] to capture two header rows.
     salary_file = st.file_uploader("Upload Salary (Earnings) File (Master)", type=['xlsx', 'xls'])
     paye_file = st.file_uploader("Upload PAYE File", type=['xlsx', 'xls'])
     
     if employee_file and salary_file and paye_file:
         try:
             employee_df = read_excel_file(employee_file, header_option=0)
-            salary_df   = read_excel_file(salary_file, header_option=[0,1])
-            paye_df     = read_excel_file(paye_file, header_option=0)
+            # Read salary file with header_option=[0,1] to capture two header rows, if present.
+            salary_df = read_excel_file(salary_file, header_option=[0,1])
+            paye_df = read_excel_file(paye_file, header_option=0)
             
             employee_df = process_employee_data(employee_df)
-            salary_df   = process_salary_data(salary_df)
-            paye_df     = process_paye_data(paye_df)
+            salary_df = process_salary_data(salary_df)
+            paye_df = process_paye_data(paye_df)
             
             final_df = merge_dataframes(employee_df, salary_df, paye_df)