winsweb commited on
Commit
df71bed
·
1 Parent(s): 62f645b

Fix: Robust timestamp repair and flexible date parsing for PDF/CSV

Browse files
telecom_anomaly/utils/pdf_processor.py CHANGED
@@ -150,7 +150,12 @@ def _process_lines_logic(lines, base_name, save_files=False):
150
  current_row = row.copy()
151
  elif current_row is not None:
152
  # Append fragments to current row
153
- current_row[main_col] = (str(current_row[main_col]) + " " + val).replace('\r', '').replace('\n', '')
 
 
 
 
 
154
  for col in df_raw.columns:
155
  if col != main_col and pd.api.types.is_string_dtype(df_raw[col]):
156
  fragment = str(row[col]).strip()
@@ -165,13 +170,35 @@ def _process_lines_logic(lines, base_name, save_files=False):
165
  return None
166
 
167
  df_calls = pd.DataFrame(fixed_rows)
 
 
168
  df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
169
 
170
  # Robust date parsing
171
- dt_series = pd.to_datetime(df_calls[main_col], errors='coerce', dayfirst=True)
 
 
 
 
 
 
 
172
  df_calls['event_date'] = dt_series.dt.date
173
  df_calls['event_time'] = dt_series.dt.time
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  # If the main column was NOT the first one, we should keep it for debugging or drop it if it's already split
176
  if main_col in df_calls.columns:
177
  df_calls.drop(columns=[main_col], inplace=True)
 
150
  current_row = row.copy()
151
  elif current_row is not None:
152
  # Append fragments to current row
153
+ # CRITICAL FIX: If the current value ends in a colon, don't add a space when joining
154
+ if str(current_row[main_col]).strip().endswith(':'):
155
+ current_row[main_col] = (str(current_row[main_col]).strip() + val).replace('\r', '').replace('\n', '')
156
+ else:
157
+ current_row[main_col] = (str(current_row[main_col]) + " " + val).replace('\r', '').replace('\n', '')
158
+
159
  for col in df_raw.columns:
160
  if col != main_col and pd.api.types.is_string_dtype(df_raw[col]):
161
  fragment = str(row[col]).strip()
 
170
  return None
171
 
172
  df_calls = pd.DataFrame(fixed_rows)
173
+ # Clean up the joined column: remove any "01: 52:44" -> "01:52:44"
174
+ df_calls[main_col] = df_calls[main_col].str.replace(r'(\d{2}:)\s+(\d{2})', r'\1\2', regex=True)
175
  df_calls[main_col] = df_calls[main_col].str.replace(r'[\r\n]+', '', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
176
 
177
  # Robust date parsing
178
+ try:
179
+ dt_series = pd.to_datetime(df_calls[main_col], errors='coerce')
180
+ # If too many NaT, try with dayfirst
181
+ if dt_series.isna().sum() > len(dt_series) * 0.5:
182
+ dt_series = pd.to_datetime(df_calls[main_col], errors='coerce', dayfirst=True)
183
+ except:
184
+ dt_series = pd.Series([None] * len(df_calls))
185
+
186
  df_calls['event_date'] = dt_series.dt.date
187
  df_calls['event_time'] = dt_series.dt.time
188
 
189
+ # Manual fallback for the specific "2024-01-01 01:52:44" format if pandas fails
190
+ mask = df_calls['event_date'].isna()
191
+ if mask.any():
192
+ for i in df_calls[mask].index:
193
+ try:
194
+ # Try simple split
195
+ parts = str(df_calls.loc[i, main_col]).split(' ')
196
+ if len(parts) >= 2:
197
+ df_calls.loc[i, 'event_date'] = pd.to_datetime(parts[0]).date()
198
+ df_calls.loc[i, 'event_time'] = pd.to_datetime(parts[1]).time()
199
+ except:
200
+ pass
201
+
202
  # If the main column was NOT the first one, we should keep it for debugging or drop it if it's already split
203
  if main_col in df_calls.columns:
204
  df_calls.drop(columns=[main_col], inplace=True)
telecom_anomaly/validation/validator.py CHANGED
@@ -53,23 +53,39 @@ class DataValidator:
53
 
54
  @staticmethod
55
  def parse_datetime(date_str: str, time_str: str) -> Optional[datetime]:
56
- """Parse datetime from date and time strings"""
 
 
 
57
  try:
58
- if date_str and time_str and date_str != 'NULL' and time_str != 'NULL':
59
- return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
60
- except (ValueError, TypeError) as e:
61
- logger.debug(f"Failed to parse datetime: {date_str} {time_str} - {e}")
 
 
 
 
 
 
 
 
62
  return None
63
 
64
  @staticmethod
65
  def is_night_call(time_str: str, night_start: int = 23, night_end: int = 5) -> bool:
66
  """Check if call occurred at night"""
67
  try:
68
- if not time_str or time_str == 'NULL':
69
  return False
70
- hour = int(time_str.split(':')[0])
 
 
 
 
 
71
  return hour >= night_start or hour < night_end
72
- except (ValueError, IndexError):
73
  return False
74
 
75
  @staticmethod
 
53
 
54
  @staticmethod
55
  def parse_datetime(date_str: str, time_str: str) -> Optional[datetime]:
56
+ """Parse datetime from date and time strings with robust fallback"""
57
+ if not date_str or date_str == 'NULL' or not time_str or time_str == 'NULL':
58
+ return None
59
+
60
  try:
61
+ # Try standard format
62
+ return datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %H:%M:%S")
63
+ except:
64
+ try:
65
+ # Fallback to pandas for flexible parsing
66
+ import pandas as pd
67
+ combined = f"{date_str} {time_str}"
68
+ dt = pd.to_datetime(combined, errors='coerce')
69
+ if pd.notna(dt):
70
+ return dt.to_pydatetime()
71
+ except:
72
+ pass
73
  return None
74
 
75
  @staticmethod
76
  def is_night_call(time_str: str, night_start: int = 23, night_end: int = 5) -> bool:
77
  """Check if call occurred at night"""
78
  try:
79
+ if not time_str or time_str == 'NULL' or time_str == 'None':
80
  return False
81
+ # Handle both "HH:MM:SS" and full "YYYY-MM-DD HH:MM:SS"
82
+ ts = str(time_str).strip()
83
+ if ' ' in ts:
84
+ ts = ts.split(' ')[1]
85
+
86
+ hour = int(ts.split(':')[0])
87
  return hour >= night_start or hour < night_end
88
+ except:
89
  return False
90
 
91
  @staticmethod