File size: 3,301 Bytes
872138b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """
utils/data_loading.py
Everything needed to read the raw NASA C-MAPSS text files into tidy
pandas DataFrames. The raw files are whitespace separated and have NO
header row, so we assign the column names ourselves (see project brief,
section 3.2).
"""
import pandas as pd
from pathlib import Path
# Column names for the 26 raw columns, in the official order.
COLUMN_NAMES = (
["engine_id", "cycle"]
+ [f"op_setting_{i}" for i in (1, 2, 3)]
+ [f"sensor_{i}" for i in range(1, 22)]
)
OP_SETTING_COLS = [f"op_setting_{i}" for i in (1, 2, 3)]
SENSOR_COLS = [f"sensor_{i}" for i in range(1, 22)]
def load_train(data_dir: str, subset: str) -> pd.DataFrame:
"""Load a train_FDxxx.txt file (run-to-failure trajectories)."""
path = Path(data_dir) / f"train_{subset}.txt"
df = pd.read_csv(path, sep=r"\s+", header=None, names=COLUMN_NAMES)
return df
def load_test(data_dir: str, subset: str) -> pd.DataFrame:
"""Load a test_FDxxx.txt file (trajectories truncated before failure)."""
path = Path(data_dir) / f"test_{subset}.txt"
df = pd.read_csv(path, sep=r"\s+", header=None, names=COLUMN_NAMES)
return df
def load_test_rul(data_dir: str, subset: str) -> pd.Series:
"""
Load RUL_FDxxx.txt: one value per test engine, giving the remaining
cycles AFTER the last observed cycle in test_FDxxx.txt. The file order
matches the engine order in the test file (engine_id 1, 2, 3, ...).
"""
path = Path(data_dir) / f"RUL_{subset}.txt"
rul = pd.read_csv(path, sep=r"\s+", header=None, names=["final_rul"])
return rul["final_rul"]
def basic_sanity_checks(train_df: pd.DataFrame, test_df: pd.DataFrame, rul: pd.Series) -> dict:
"""
Cheap structural checks used in the EDA section (Task 1). Returns a
dictionary so the notebook can print/display it as a small table.
"""
checks = {}
checks["n_train_engines"] = train_df["engine_id"].nunique()
checks["n_test_engines"] = test_df["engine_id"].nunique()
checks["n_rul_values"] = len(rul)
checks["rul_count_matches_test_engines"] = (
len(rul) == test_df["engine_id"].nunique()
)
checks["train_missing_values"] = int(train_df.isna().sum().sum())
checks["test_missing_values"] = int(test_df.isna().sum().sum())
checks["train_non_finite_values"] = int(
(
~train_df.drop(columns=["engine_id"])
.apply(pd.to_numeric)
.apply(lambda s: s.between(-1e15, 1e15))
)
.sum()
.sum()
)
checks["train_duplicate_engine_cycle_keys"] = int(
train_df.duplicated(subset=["engine_id", "cycle"]).sum()
)
checks["test_duplicate_engine_cycle_keys"] = int(
test_df.duplicated(subset=["engine_id", "cycle"]).sum()
)
# Cycle continuity: for every engine, cycle should go 1,2,3,...,max with no gaps.
def _has_gaps(df):
bad = 0
for _, g in df.groupby("engine_id"):
cycles = g["cycle"].sort_values().to_numpy()
expected = range(1, len(cycles) + 1)
if not all(c == e for c, e in zip(cycles, expected)):
bad += 1
return bad
checks["train_engines_with_cycle_gaps"] = _has_gaps(train_df)
checks["test_engines_with_cycle_gaps"] = _has_gaps(test_df)
return checks
|