""" utils/data_loading.py Everything needed to read the raw NASA C-MAPSS text files into tidy pandas DataFrames. The raw files are whitespace separated and have NO header row, so we assign the column names ourselves (see project brief, section 3.2). """ import pandas as pd from pathlib import Path # Column names for the 26 raw columns, in the official order. COLUMN_NAMES = ( ["engine_id", "cycle"] + [f"op_setting_{i}" for i in (1, 2, 3)] + [f"sensor_{i}" for i in range(1, 22)] ) OP_SETTING_COLS = [f"op_setting_{i}" for i in (1, 2, 3)] SENSOR_COLS = [f"sensor_{i}" for i in range(1, 22)] def load_train(data_dir: str, subset: str) -> pd.DataFrame: """Load a train_FDxxx.txt file (run-to-failure trajectories).""" path = Path(data_dir) / f"train_{subset}.txt" df = pd.read_csv(path, sep=r"\s+", header=None, names=COLUMN_NAMES) return df def load_test(data_dir: str, subset: str) -> pd.DataFrame: """Load a test_FDxxx.txt file (trajectories truncated before failure).""" path = Path(data_dir) / f"test_{subset}.txt" df = pd.read_csv(path, sep=r"\s+", header=None, names=COLUMN_NAMES) return df def load_test_rul(data_dir: str, subset: str) -> pd.Series: """ Load RUL_FDxxx.txt: one value per test engine, giving the remaining cycles AFTER the last observed cycle in test_FDxxx.txt. The file order matches the engine order in the test file (engine_id 1, 2, 3, ...). """ path = Path(data_dir) / f"RUL_{subset}.txt" rul = pd.read_csv(path, sep=r"\s+", header=None, names=["final_rul"]) return rul["final_rul"] def basic_sanity_checks(train_df: pd.DataFrame, test_df: pd.DataFrame, rul: pd.Series) -> dict: """ Cheap structural checks used in the EDA section (Task 1). Returns a dictionary so the notebook can print/display it as a small table. """ checks = {} checks["n_train_engines"] = train_df["engine_id"].nunique() checks["n_test_engines"] = test_df["engine_id"].nunique() checks["n_rul_values"] = len(rul) checks["rul_count_matches_test_engines"] = ( len(rul) == test_df["engine_id"].nunique() ) checks["train_missing_values"] = int(train_df.isna().sum().sum()) checks["test_missing_values"] = int(test_df.isna().sum().sum()) checks["train_non_finite_values"] = int( ( ~train_df.drop(columns=["engine_id"]) .apply(pd.to_numeric) .apply(lambda s: s.between(-1e15, 1e15)) ) .sum() .sum() ) checks["train_duplicate_engine_cycle_keys"] = int( train_df.duplicated(subset=["engine_id", "cycle"]).sum() ) checks["test_duplicate_engine_cycle_keys"] = int( test_df.duplicated(subset=["engine_id", "cycle"]).sum() ) # Cycle continuity: for every engine, cycle should go 1,2,3,...,max with no gaps. def _has_gaps(df): bad = 0 for _, g in df.groupby("engine_id"): cycles = g["cycle"].sort_values().to_numpy() expected = range(1, len(cycles) + 1) if not all(c == e for c, e in zip(cycles, expected)): bad += 1 return bad checks["train_engines_with_cycle_gaps"] = _has_gaps(train_df) checks["test_engines_with_cycle_gaps"] = _has_gaps(test_df) return checks