import math import json import uuid from datetime import datetime from pathlib import Path import numpy as np import pandas as pd import torch from torch import nn import gradio as gr # ============================= # Fixed settings # ============================= SEQ_LEN = 4 TARGET_COLS = [ "OD", "CarbonContents", "Ethanol", "Acetoin", "BDO", "Succinate", "Lactate", "Formate", "Acetate", ] # 9 targets (including OD) DYN_EXTRA = ["rpm", "Feed", "delta_t"] ALL_DYN = TARGET_COLS + DYN_EXTRA # 12 # static embedding columns are FIXED by name, not by column position EMBED_COLS = [ "AR", "DR", "PPAR", "acoA", "acoABC", "acoO", "blaA", "crr", "dar", "dhaD", "gldA", "ldhA", "oqxB", "pflB", "ptsG", "Glucose", "Xylose", "Glycerol", "원당", "가수분해원당", "당밀", "당밀 산가수분해액", "목질계당화액", "바이오슈가", "액당", "K2HPO4", "KH2PO4", "(NH4)2SO4", "YE", "MgSO4_7H2O", "FeSO4_7H2O", "ZnSO4_7H2O", "MnSO4_H2O", "CaCl2_2H2O", "HCl", "Peptone", "BE", "NaCl", "CSL", "BDO", "Succinic acid", "Acetoin", "Formic acid", "Ethanol", "Acetic acid", "IPTG", "CSP", "CDS" ] DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") BDO_IDX = TARGET_COLS.index("BDO") RPM_IDX = ALL_DYN.index("rpm") SESSION_ROOT = Path("saved_sessions") SESSION_ROOT.mkdir(exist_ok=True) # ============================= # Robust CSV reader # ============================= def _read_csv_any_encoding(path: str, encodings=("utf-8-sig", "utf-8", "cp949", "euc-kr", "latin1")): last_err = None for enc in encodings: try: return pd.read_csv(path, encoding=enc, low_memory=False) except Exception as e: last_err = e raise last_err # ============================= # Helpers # ============================= def _safe_float(x, default=0.0): try: if x is None or x == "": return float(default) return float(x) except Exception: return float(default) def _make_session_dir(session_id: str | None = None) -> Path: if not session_id: ts = datetime.now().strftime("%Y%m%d_%H%M%S") session_id = f"session_{ts}_{uuid.uuid4().hex[:8]}" session_dir = SESSION_ROOT / session_id session_dir.mkdir(parents=True, exist_ok=True) return session_dir def _session_paths(session_dir: Path): return { "dynamic_csv": session_dir / "dynamic_data_current.csv", "static_csv": session_dir / "static_data_original.csv", "embed_npz": session_dir / "embed_state.npz", "meta_json": session_dir / "session_meta.json", } def save_session_files(session_state: dict): """ Save current session to disk. """ if not session_state or "session_dir" not in session_state: return None session_dir = Path(session_state["session_dir"]) paths = _session_paths(session_dir) df_dyn = pd.DataFrame(session_state["df_dyn"]) df_dyn.to_csv(paths["dynamic_csv"], index=False, encoding="utf-8-sig") embed_vec = np.array(session_state["embed_vec"], dtype=np.float32) embed_cols = np.array(session_state["embed_cols"], dtype=object) np.savez(paths["embed_npz"], embed_vec=embed_vec, embed_cols=embed_cols) meta = { "session_id": session_state["session_id"], "session_dir": str(session_dir), "created_at": session_state.get("created_at"), "updated_at": datetime.now().isoformat(timespec="seconds"), "last_delta": session_state.get("last_delta"), "seq_len": SEQ_LEN, "n_dynamic_rows": len(df_dyn), } with open(paths["meta_json"], "w", encoding="utf-8") as f: json.dump(meta, f, ensure_ascii=False, indent=2) return str(paths["dynamic_csv"]) def load_saved_session(session_dir_str: str): session_dir = Path(session_dir_str) paths = _session_paths(session_dir) if not paths["dynamic_csv"].exists(): raise ValueError(f"Saved session not found: {paths['dynamic_csv']}") df_dyn = _read_csv_any_encoding(str(paths["dynamic_csv"])) if not paths["embed_npz"].exists(): raise ValueError(f"Saved embed state not found: {paths['embed_npz']}") npz = np.load(paths["embed_npz"], allow_pickle=True) embed_vec = npz["embed_vec"].astype(np.float32) embed_cols = list(npz["embed_cols"]) meta = {} if paths["meta_json"].exists(): with open(paths["meta_json"], "r", encoding="utf-8") as f: meta = json.load(f) session_state = { "session_id": meta.get("session_id", session_dir.name), "session_dir": str(session_dir), "created_at": meta.get("created_at", datetime.now().isoformat(timespec="seconds")), "last_delta": int(meta.get("last_delta", 2)), "df_dyn": df_dyn.to_dict(orient="records"), "embed_vec": embed_vec.tolist(), "embed_cols": list(embed_cols), } save_path = save_session_files(session_state) return session_state, save_path # ============================= # delta_t builder # ============================= def add_delta_t(df_dyn: pd.DataFrame, last_delta: int) -> pd.DataFrame: """ Sort by Time and compute: delta_t[i] = Time[i+1] - Time[i] delta_t[last] = last_delta """ if "Time" not in df_dyn.columns: raise ValueError("dynamic_data CSV must contain 'Time' column.") df = df_dyn.copy() df.columns = df.columns.map(str.strip) df["Time"] = pd.to_numeric(df["Time"], errors="coerce") df = df.dropna(subset=["Time"]).sort_values("Time").reset_index(drop=True) if len(df) == 0: raise ValueError("dynamic_data CSV has no valid Time rows.") t = df["Time"].to_numpy(dtype=float) dt = np.empty(len(df), dtype=float) if len(df) >= 2: dt[:-1] = t[1:] - t[:-1] dt[-1] = float(last_delta) df["delta_t"] = dt return df # ============================= # Column normalization # ============================= def normalize_dynamic_columns(df_dyn: pd.DataFrame) -> pd.DataFrame: """ Make dynamic CSV robust to column order. Required columns are identified by NAME, not by position. """ df = df_dyn.copy() df.columns = df.columns.map(lambda x: str(x).strip()) required_dyn = ["Time"] + TARGET_COLS + ["rpm", "Feed"] missing = [c for c in required_dyn if c not in df.columns] if missing: raise ValueError(f"dynamic_data missing required columns: {missing}") ordered = ["Time"] + TARGET_COLS + ["rpm", "Feed"] df = df.loc[:, ordered].copy() for c in ordered: df[c] = pd.to_numeric(df[c], errors="coerce") df[ordered] = df[ordered].ffill().bfill().fillna(0.0) return df def normalize_static_embedding(df_stat: pd.DataFrame): """ Make static CSV robust to column order. Reorder columns to match fixed EMBED_COLS. """ df = df_stat.copy() df.columns = df.columns.map(lambda x: str(x).strip()) missing = [c for c in EMBED_COLS if c not in df.columns] if missing: raise ValueError( "static_data missing required embedding columns: " f"{missing}" ) df_emb = df.loc[:, EMBED_COLS].copy() df_emb = df_emb.apply(pd.to_numeric, errors="coerce").fillna(0.0) if len(df_emb) == 0: raise ValueError("static_data CSV has no rows.") embed_vec = df_emb.iloc[0].to_numpy(dtype=np.float32) return embed_vec, EMBED_COLS def validate_dynamic_columns(df_dyn: pd.DataFrame) -> pd.DataFrame: """ UI/저장용: - 업로드한 원본 컬럼 순서를 유지 - 필요한 컬럼 존재 여부만 검사 - 숫자형 변환만 수행 """ df = df_dyn.copy() df.columns = df.columns.map(lambda x: str(x).strip()) required_dyn = ["Time"] + TARGET_COLS + ["rpm", "Feed"] missing = [c for c in required_dyn if c not in df.columns] if missing: raise ValueError(f"dynamic_data missing required columns: {missing}") for c in required_dyn: df[c] = pd.to_numeric(df[c], errors="coerce") df[required_dyn] = df[required_dyn].ffill().bfill().fillna(0.0) return df def make_dynamic_model_input(df_dyn: pd.DataFrame, last_delta: int) -> pd.DataFrame: """ 예측용: - 여기서만 모델 입력 컬럼 순서로 재정렬 - delta_t 생성 """ df = validate_dynamic_columns(df_dyn) df = add_delta_t(df, last_delta=int(last_delta)) df = df.sort_values("Time").reset_index(drop=True) ordered = ["Time"] + TARGET_COLS + ["rpm", "Feed", "delta_t"] missing = [c for c in ordered if c not in df.columns] if missing: raise ValueError(f"dynamic_data missing required columns after preprocessing: {missing}") return df.loc[:, ordered].copy() # ============================= # Load uploaded files # ============================= def load_two_files(dynamic_csv_path: str, static_csv_path: str, last_delta: int): df_dyn = _read_csv_any_encoding(dynamic_csv_path) df_stat = _read_csv_any_encoding(static_csv_path) # dynamic: 원본 컬럼 순서 유지 df_dyn = validate_dynamic_columns(df_dyn) # static: 예측용 embedding만 이름 기준 고정 정렬 embed_vec, embed_cols = normalize_static_embedding(df_stat) return df_dyn, embed_vec, embed_cols, df_stat # ============================= # Model definition # ============================= class PositionalEncoding(nn.Module): def __init__(self, d_model: int, max_len: int = 2048): super().__init__() pos = torch.arange(max_len, dtype=torch.float32).unsqueeze(1) div = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)) pe = torch.zeros(max_len, d_model) pe[:, 0::2] = torch.sin(pos * div) pe[:, 1::2] = torch.cos(pos * div) self.register_buffer("pe", pe) def forward(self, x): if x.dim() != 3: raise ValueError("Input must be 3-D tensor") if x.shape[1] <= x.shape[0]: l = x.shape[1] pe = self.pe[:l].unsqueeze(0) else: l = x.shape[0] pe = self.pe[:l].unsqueeze(1) return x + pe class TransformerRegressor(nn.Module): def __init__( self, input_dim: int, *, d_model: int = 64, nhead: int = 8, num_layers: int = 2, dropout: float = 0.2, max_len: int = 2048, out_activation: str = "relu", ): super().__init__() self.input_proj = nn.Linear(input_dim, d_model) try: layer = nn.TransformerEncoderLayer( d_model=d_model, nhead=nhead, dim_feedforward=4 * d_model, dropout=dropout, batch_first=True, ) self.batch_first = True except TypeError: layer = nn.TransformerEncoderLayer( d_model=d_model, nhead=nhead, dim_feedforward=4 * d_model, dropout=dropout, ) self.batch_first = False self.encoder = nn.TransformerEncoder(layer, num_layers=num_layers) self.pos = PositionalEncoding(d_model, max_len=max_len) self.norm = nn.LayerNorm(d_model) self.head = nn.Linear(d_model, len(TARGET_COLS)) if out_activation == "relu": self.out_act = nn.ReLU() elif out_activation == "softplus": self.out_act = nn.Softplus(beta=1.0, threshold=20.0) elif out_activation == "none": self.out_act = nn.Identity() else: raise ValueError("out_activation must be 'relu', 'softplus', or 'none'") def forward(self, x): h = self.pos(self.input_proj(x)) if self.batch_first: h = self.encoder(h) last = h[:, -1] else: h = self.encoder(h.permute(1, 0, 2)) last = h[-1] out = self.head(self.norm(last)) return self.out_act(out) def build_model_from_ckpt(input_dim: int, ckpt_path: Path) -> nn.Module: model = TransformerRegressor( input_dim=input_dim, d_model=64, nhead=8, num_layers=2, dropout=0.2, max_len=2048, out_activation="relu", ).to(DEVICE) state = torch.load(str(ckpt_path), map_location=DEVICE) model.load_state_dict(state, strict=False) model.eval() return model # ============================= # Prediction / RPM optimization # ============================= def predict_from_window(win_dyn: np.ndarray, embed_vec: np.ndarray, model: nn.Module, device: torch.device): L = win_dyn.shape[0] static = np.repeat(embed_vec[None], L, axis=0).astype(np.float32) xb = torch.from_numpy(np.hstack([win_dyn.astype(np.float32), static]))[None].to(device) with torch.no_grad(): y_pred = model(xb).squeeze(0).cpu().numpy().astype(np.float32) return y_pred def choose_rpm_realtime( win_dyn: np.ndarray, embed_vec: np.ndarray, model: torch.nn.Module, device: torch.device, deltas=(-200, -150, -100, -50, 0, 50, 100, 150, 200), rpm_min=150, rpm_max=650, ): """ Always search RPM in real time. Returns the RPM that maximizes predicted next BDO. """ base_pred = predict_from_window(win_dyn, embed_vec, model, device) curr_bdo = float(win_dyn[-1, BDO_IDX]) base_bdo = float(base_pred[BDO_IDX]) base_rpm = float(win_dyn[-1, RPM_IDX]) best = { "rpm_best": float(base_rpm), "bdo_best": float(base_bdo), "bdo_baseline": float(base_bdo), "rpm_baseline": float(base_rpm), "bdo_current": float(curr_bdo), "rpm_changed": False, "rpm_delta": 0.0, } for d in deltas: new_rpm = base_rpm + d if new_rpm < rpm_min or new_rpm > rpm_max: continue tmp = win_dyn.copy() tmp[-1, RPM_IDX] = new_rpm pred = predict_from_window(tmp, embed_vec, model, device) bdo = float(pred[BDO_IDX]) if bdo > best["bdo_best"]: best["rpm_best"] = float(new_rpm) best["bdo_best"] = float(bdo) best["rpm_delta"] = float(d) best["rpm_changed"] = (best["rpm_best"] != best["rpm_baseline"]) return best def run_model_on_df(df_dyn: pd.DataFrame, embed_vec: np.ndarray, embed_cols, last_delta: int): ckpt_path = Path("ckpt.pt") if not ckpt_path.exists(): raise ValueError("repo 루트에 ckpt.pt가 없습니다. Spaces에 ckpt.pt를 업로드/커밋해주세요.") # 예측용으로만 재정렬 df_model = make_dynamic_model_input(df_dyn, last_delta=int(last_delta)) if len(df_model) < SEQ_LEN: raise ValueError(f"dynamic_data row 수({len(df_model)})가 SEQ_LEN({SEQ_LEN})보다 작습니다.") input_dim = len(ALL_DYN) + len(embed_cols) model = build_model_from_ckpt(input_dim=input_dim, ckpt_path=ckpt_path) dyn_mat = df_model[ALL_DYN].to_numpy(dtype=np.float32) win = dyn_mat[-SEQ_LEN:] y_pred = predict_from_window(win, embed_vec, model, DEVICE) pred_bdo = float(y_pred[BDO_IDX]) curr_bdo = float(win[-1, BDO_IDX]) rec = choose_rpm_realtime(win, embed_vec, model, DEVICE) out = ( f"- Current BDO (last row): {curr_bdo:.6f}\n" f"- Predicted next BDO (baseline): {pred_bdo:.6f}\n" f"\n[Real-time RPM optimization]\n" f"- Current RPM: {rec['rpm_baseline']:.0f}\n" f"- Recommended RPM: {rec['rpm_best']:.0f}\n" f"- RPM delta: {rec['rpm_delta']:+.0f}\n" f"- Predicted BDO @ current RPM: {rec['bdo_baseline']:.6f}\n" f"- Predicted BDO @ recommended RPM: {rec['bdo_best']:.6f}\n" ) if rec["rpm_changed"]: out += "- Action: RPM change recommended.\n" else: out += "- Action: Keep current RPM.\n" # 반환은 화면 표시용 df_dyn 그대로 return out, df_dyn # ============================= # Gradio handlers # ============================= def initialize_session(dynamic_file, static_file, last_delta: int): if dynamic_file is None or static_file is None: return None, "❌ dynamic/static CSV 두 파일을 모두 업로드해주세요.", None, None, None if not (1 <= int(last_delta) <= 6): return None, "❌ last_delta는 1~6 사이의 정수여야 합니다.", None, None, None try: df_dyn, embed_vec, embed_cols, df_stat = load_two_files( dynamic_file.name, static_file.name, last_delta=int(last_delta) ) result_text, df_dyn = run_model_on_df(df_dyn, embed_vec, embed_cols, last_delta=int(last_delta)) session_dir = _make_session_dir() session_state = { "session_id": session_dir.name, "session_dir": str(session_dir), "created_at": datetime.now().isoformat(timespec="seconds"), "last_delta": int(last_delta), "df_dyn": df_dyn.to_dict(orient="records"), "embed_vec": embed_vec.tolist(), "embed_cols": list(embed_cols), } # save originals / session state paths = _session_paths(session_dir) df_stat.to_csv(paths["static_csv"], index=False, encoding="utf-8-sig") save_path = save_session_files(session_state) preview = prepare_editable_tail(session_state) session_info = ( f"✅ Session initialized\n" f"- session_id: {session_state['session_id']}\n" f"- saved_dir: {session_state['session_dir']}\n" f"- dynamic rows: {len(df_dyn)}" ) return session_state, result_text, preview, save_path, session_info except Exception as e: return None, f"❌ {e}", None, None, None def prepare_editable_tail(session_state): """ 업로드한 dynamic_data의 원본 컬럼 순서를 유지해서 tail 표시. 맨 아래에 빈 row 1개를 추가. """ if not session_state: return None df_dyn = pd.DataFrame(session_state["df_dyn"]).copy() # 화면에는 delta_t를 굳이 안 보여주고 싶으면 제외 display_cols = [c for c in df_dyn.columns if c != "delta_t"] df_show = df_dyn.loc[:, display_cols].tail(8).copy() empty_row = {c: np.nan for c in display_cols} df_show = pd.concat([df_show, pd.DataFrame([empty_row])], ignore_index=True) return df_show def apply_edited_dynamic_table(session_state, edited_df, last_delta): if not session_state: return session_state, "❌ 먼저 CSV 업로드 후 session을 초기화해주세요.", None, None, None try: df_full = pd.DataFrame(session_state["df_dyn"]).copy() df_full = df_full.drop(columns=["delta_t"], errors="ignore") if edited_df is None: return session_state, "❌ 편집된 데이터가 없습니다.", None, None, None if not isinstance(edited_df, pd.DataFrame): edited_df = pd.DataFrame(edited_df) edited_df = edited_df.copy() edited_df.columns = [str(c).strip() for c in edited_df.columns] required_cols = ["Time"] + TARGET_COLS + ["rpm", "Feed"] missing = [c for c in required_cols if c not in edited_df.columns] if missing: return session_state, f"❌ edited table missing columns: {missing}", None, None, None # 숫자형 변환은 필요한 컬럼만 for c in required_cols: edited_df[c] = pd.to_numeric(edited_df[c], errors="coerce") edited_df = edited_df.dropna(how="all").copy() edited_df = edited_df.dropna(subset=["Time"]).copy() if len(edited_df) == 0: return session_state, "❌ 입력된 row가 없습니다.", None, None, None existing_times = set(pd.to_numeric(df_full["Time"], errors="coerce").dropna().astype(float).tolist()) for _, row in edited_df.iterrows(): t = float(row["Time"]) if t in existing_times: mask = pd.to_numeric(df_full["Time"], errors="coerce").astype(float) == t for c in edited_df.columns: if c in df_full.columns: df_full.loc[mask, c] = row[c] else: new_row = {} for c in df_full.columns: new_row[c] = row[c] if c in edited_df.columns else np.nan df_full = pd.concat([df_full, pd.DataFrame([new_row])], ignore_index=True) # 여기서는 원본 컬럼 순서 유지 df_full = validate_dynamic_columns(df_full) embed_vec = np.array(session_state["embed_vec"], dtype=np.float32) embed_cols = list(session_state["embed_cols"]) result_text, _ = run_model_on_df(df_full, embed_vec, embed_cols, last_delta=int(last_delta)) session_state["df_dyn"] = df_full.to_dict(orient="records") session_state["last_delta"] = int(last_delta) save_path = save_session_files(session_state) preview = prepare_editable_tail(session_state) session_info = ( f"✅ Session updated\n" f"- session_id: {session_state['session_id']}\n" f"- saved_dir: {session_state['session_dir']}\n" f"- dynamic rows: {len(df_full)}" ) return session_state, result_text, preview, save_path, session_info except Exception as e: return session_state, f"❌ {e}", None, None, None def load_existing_session(saved_session_dir: str): if saved_session_dir is None or str(saved_session_dir).strip() == "": return None, "❌ 불러올 session 경로를 입력해주세요.", None, None, None try: session_state, save_path = load_saved_session(saved_session_dir.strip()) df_dyn = pd.DataFrame(session_state["df_dyn"]) embed_vec = np.array(session_state["embed_vec"], dtype=np.float32) embed_cols = list(session_state["embed_cols"]) result_text, df_dyn = run_model_on_df( df_dyn, embed_vec, embed_cols, last_delta=int(session_state["last_delta"]) ) session_state["df_dyn"] = df_dyn.to_dict(orient="records") save_path = save_session_files(session_state) preview = prepare_editable_tail(session_state) session_info = ( f"✅ Session loaded\n" f"- session_id: {session_state['session_id']}\n" f"- saved_dir: {session_state['session_dir']}\n" f"- dynamic rows: {len(df_dyn)}" ) return session_state, result_text, preview, save_path, session_info except Exception as e: return None, f"❌ {e}", None, None, None # ============================= # Gradio UI # ============================= with gr.Blocks(title="BDO Transformer (Realtime RPM + Session)") as demo: session_state = gr.State(value=None) gr.Markdown( """ # BDO Transformer Inference ## Features - Upload **dynamic_data CSV** + **static_data CSV** - static CSV는 **컬럼 순서가 달라도**, embedding 컬럼 이름 기준으로 자동 정렬 - dynamic CSV는 **업로드한 원본 컬럼 순서 그대로 표시** - 예측할 때만 모델 입력 순서로 내부 정렬 - **마지막 window (SEQ_LEN=4)** 로 다음 target 예측 - RPM 최적화는 **항상(real-time)** 수행 - Latest dynamic data 표에서 **직접 row를 수정/추가** - 누적될 때마다 session 파일 저장 + updated dynamic CSV 다운로드 가능 """.strip() ) with gr.Tab("1) Upload CSV and initialize"): with gr.Row(): dynamic_file = gr.File(label="dynamic_data.csv") static_file = gr.File(label="static_data.csv") last_delta = gr.Slider( label="Last row delta_t (1~6)", minimum=1, maximum=6, step=1, value=2 ) init_btn = gr.Button("Initialize Session", variant="primary") with gr.Tab("2) Restore saved session"): saved_session_dir = gr.Textbox( label="Saved session directory path", placeholder="예: saved_sessions/session_20260327_123456_ab12cd34" ) load_btn = gr.Button("Load Saved Session") gr.Markdown("## Outputs") preview_df = gr.Dataframe( label="Latest dynamic data (tail) - editable", interactive=True ) apply_edit_btn = gr.Button("Apply Edited Table", variant="primary") result_box = gr.Textbox(label="Result", lines=16) download_file = gr.File(label="Download updated dynamic CSV") session_info_box = gr.Textbox(label="Session Info", lines=4) init_btn.click( fn=initialize_session, inputs=[dynamic_file, static_file, last_delta], outputs=[session_state, result_box, preview_df, download_file, session_info_box] ) apply_edit_btn.click( fn=apply_edited_dynamic_table, inputs=[session_state, preview_df, last_delta], outputs=[session_state, result_box, preview_df, download_file, session_info_box] ) load_btn.click( fn=load_existing_session, inputs=[saved_session_dir], outputs=[session_state, result_box, preview_df, download_file, session_info_box] ) if __name__ == "__main__": demo.launch(ssr_mode=False)