Spaces:
Sleeping
Sleeping
Fix: Robust timestamp repair and flexible date parsing for PDF/CSV
Browse files
telecom_anomaly/utils/pdf_processor.py
CHANGED
|
@@ -150,7 +150,12 @@ def _process_lines_logic(lines, base_name, save_files=False):
|
|
| 150 |
current_row = row.copy()
|
| 151 |
elif current_row is not None:
|
| 152 |
# Append fragments to current row
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
for col in df_raw.columns:
|
| 155 |
if col != main_col and pd.api.types.is_string_dtype(df_raw[col]):
|
| 156 |
fragment = str(row[col]).strip()
|
|
@@ -165,13 +170,35 @@ def _process_lines_logic(lines, base_name, save_files=False):
|
|
| 165 |
return None
|
| 166 |
|
| 167 |
df_calls = pd.DataFrame(fixed_rows)
|
|
|
|
|
|
|
| 168 |
df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
|
| 169 |
|
| 170 |
# Robust date parsing
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
df_calls['event_date'] = dt_series.dt.date
|
| 173 |
df_calls['event_time'] = dt_series.dt.time
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
# If the main column was NOT the first one, we should keep it for debugging or drop it if it's already split
|
| 176 |
if main_col in df_calls.columns:
|
| 177 |
df_calls.drop(columns=[main_col], inplace=True)
|
|
|
|
| 150 |
current_row = row.copy()
|
| 151 |
elif current_row is not None:
|
| 152 |
# Append fragments to current row
|
| 153 |
+
# CRITICAL FIX: If the current value ends in a colon, don't add a space when joining
|
| 154 |
+
if str(current_row[main_col]).strip().endswith(':'):
|
| 155 |
+
current_row[main_col] = (str(current_row[main_col]).strip() + val).replace('\r', '').replace('\n', '')
|
| 156 |
+
else:
|
| 157 |
+
current_row[main_col] = (str(current_row[main_col]) + " " + val).replace('\r', '').replace('\n', '')
|
| 158 |
+
|
| 159 |
for col in df_raw.columns:
|
| 160 |
if col != main_col and pd.api.types.is_string_dtype(df_raw[col]):
|
| 161 |
fragment = str(row[col]).strip()
|
|
|
|
| 170 |
return None
|
| 171 |
|
| 172 |
df_calls = pd.DataFrame(fixed_rows)
|
| 173 |
+
# Clean up the joined column: remove any "01: 52:44" -> "01:52:44"
|
| 174 |
+
df_calls[main_col] = df_calls[main_col].str.replace(r'(\d{2}:)\s+(\d{2})', r'\1\2', regex=True)
|
| 175 |
df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
|
| 176 |
|
| 177 |
# Robust date parsing
|
| 178 |
+
try:
|
| 179 |
+
dt_series = pd.to_datetime(df_calls[main_col], errors='coerce')
|
| 180 |
+
# If too many NaT, try with dayfirst
|
| 181 |
+
if dt_series.isna().sum() > len(dt_series) * 0.5:
|
| 182 |
+
dt_series = pd.to_datetime(df_calls[main_col], errors='coerce', dayfirst=True)
|
| 183 |
+
except:
|
| 184 |
+
dt_series = pd.Series([None] * len(df_calls))
|
| 185 |
+
|
| 186 |
df_calls['event_date'] = dt_series.dt.date
|
| 187 |
df_calls['event_time'] = dt_series.dt.time
|
| 188 |
|
| 189 |
+
# Manual fallback for the specific "2024-01-01 01:52:44" format if pandas fails
|
| 190 |
+
mask = df_calls['event_date'].isna()
|
| 191 |
+
if mask.any():
|
| 192 |
+
for i in df_calls[mask].index:
|
| 193 |
+
try:
|
| 194 |
+
# Try simple split
|
| 195 |
+
parts = str(df_calls.loc[i, main_col]).split(' ')
|
| 196 |
+
if len(parts) >= 2:
|
| 197 |
+
df_calls.loc[i, 'event_date'] = pd.to_datetime(parts[0]).date()
|
| 198 |
+
df_calls.loc[i, 'event_time'] = pd.to_datetime(parts[1]).time()
|
| 199 |
+
except:
|
| 200 |
+
pass
|
| 201 |
+
|
| 202 |
# If the main column was NOT the first one, we should keep it for debugging or drop it if it's already split
|
| 203 |
if main_col in df_calls.columns:
|
| 204 |
df_calls.drop(columns=[main_col], inplace=True)
|
telecom_anomaly/validation/validator.py
CHANGED
|
@@ -53,23 +53,39 @@ class DataValidator:
|
|
| 53 |
|
| 54 |
@staticmethod
|
| 55 |
def parse_datetime(date_str: str, time_str: str) -> Optional[datetime]:
|
| 56 |
-
"""Parse datetime from date and time strings"""
|
|
|
|
|
|
|
|
|
|
| 57 |
try:
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
except
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
return None
|
| 63 |
|
| 64 |
@staticmethod
|
| 65 |
def is_night_call(time_str: str, night_start: int = 23, night_end: int = 5) -> bool:
|
| 66 |
"""Check if call occurred at night"""
|
| 67 |
try:
|
| 68 |
-
if not time_str or time_str == 'NULL':
|
| 69 |
return False
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
return hour >= night_start or hour < night_end
|
| 72 |
-
except
|
| 73 |
return False
|
| 74 |
|
| 75 |
@staticmethod
|
|
|
|
| 53 |
|
| 54 |
@staticmethod
|
| 55 |
def parse_datetime(date_str: str, time_str: str) -> Optional[datetime]:
|
| 56 |
+
"""Parse datetime from date and time strings with robust fallback"""
|
| 57 |
+
if not date_str or date_str == 'NULL' or not time_str or time_str == 'NULL':
|
| 58 |
+
return None
|
| 59 |
+
|
| 60 |
try:
|
| 61 |
+
# Try standard format
|
| 62 |
+
return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
|
| 63 |
+
except:
|
| 64 |
+
try:
|
| 65 |
+
# Fallback to pandas for flexible parsing
|
| 66 |
+
import pandas as pd
|
| 67 |
+
combined = f"{date_str} {time_str}"
|
| 68 |
+
dt = pd.to_datetime(combined, errors='coerce')
|
| 69 |
+
if pd.notna(dt):
|
| 70 |
+
return dt.to_pydatetime()
|
| 71 |
+
except:
|
| 72 |
+
pass
|
| 73 |
return None
|
| 74 |
|
| 75 |
@staticmethod
|
| 76 |
def is_night_call(time_str: str, night_start: int = 23, night_end: int = 5) -> bool:
|
| 77 |
"""Check if call occurred at night"""
|
| 78 |
try:
|
| 79 |
+
if not time_str or time_str == 'NULL' or time_str == 'None':
|
| 80 |
return False
|
| 81 |
+
# Handle both "HH:MM:SS" and full "YYYY-MM-DD HH:MM:SS"
|
| 82 |
+
ts = str(time_str).strip()
|
| 83 |
+
if ' ' in ts:
|
| 84 |
+
ts = ts.split(' ')[1]
|
| 85 |
+
|
| 86 |
+
hour = int(ts.split(':')[0])
|
| 87 |
return hour >= night_start or hour < night_end
|
| 88 |
+
except:
|
| 89 |
return False
|
| 90 |
|
| 91 |
@staticmethod
|