hpce-dev / scripts /sim_offpersona.py
μ΄λ™ν˜„
[TEST] off-persona 강건성 검증 슀크립트
52204de
Raw
History Blame Contribute Delete
5.41 kB
from __future__ import annotations
"""
κ²°ν•©(bundle-v3) off-persona 강건성 검증.
페λ₯΄μ†Œλ‚˜μ— 맀이지 μ•Šμ€ λ¬΄μž‘μœ„/μž„μ˜ μ„€λ¬Έ μ‘λ‹΅μ—μ„œλ„
A. 초기 뢄포가 응닡(ν”„λ‘œν•„)을 ν•©λ¦¬μ μœΌλ‘œ λ°˜μ˜ν•˜λŠ”κ°€
B. 각 행동이 κ·Έ ν–‰λ™μ˜ κ΄€λ ¨ intentλ₯Ό λŒμ–΄μ˜¬λ¦¬λŠ”κ°€ (μ˜λ„ λ³€ν™”μ˜ 해석가λŠ₯μ„±)
λ₯Ό μ •λŸ‰Β·μ •μ„±μœΌλ‘œ ν™•μΈν•œλ‹€.
μ‹€ν–‰: python scripts/sim_offpersona.py
"""
import random
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from core.engines import config, get_engine # noqa: E402
from core.extractor import get_extractor # noqa: E402
from core.inference import infer_batch, infer_with_behavior # noqa: E402
from scripts.sim_bundle import _behavior_map, _intent_names, _rank_of, SID # noqa: E402
def _random_answers(rng: random.Random) -> dict[str, str]:
"""λͺ¨λ“  문항을 μ˜΅μ…˜ 쀑 κ· λ“± λ¬΄μž‘μœ„λ‘œ (페λ₯΄μ†Œλ‚˜ 무관)."""
survey = config.get_survey(SID)
out = {}
for q in survey["questions"]:
out[q["id"]] = rng.choice([o["code"] for o in q["options"]])
return out
def _rank(scores, iid: str) -> int:
return _rank_of(scores, iid)
# ── B. 행동 응닡성: λ¬΄μž‘μœ„ ν”„λ‘œν•„ Γ— λ¬΄μž‘μœ„ 단일 행동 ─────────────
def behavior_responsiveness(k: int = 300, seed: int = 11) -> None:
rng = random.Random(seed)
bmap = _behavior_map()
sig = config.get_behavior_signals(SID)
# μ‹ ν˜Έλ₯Ό κ°€μ§„ entity만 (back/exit μ œμ™Έ)
actable = [(bid, et, en) for bid, (et, en) in bmap.items()
if sig.get(en) and et not in ("navigate_back", "app_exit")]
ext = get_extractor()
rose, in_top5, total = 0, 0, 0
for i in range(k):
ans = _random_answers(rng)
bid, et, en = rng.choice(actable)
targets = sig[en]
sess = f"__off_{i}"
ext.reset(sess)
ext.add_event(sess, et, en)
_, scores = infer_with_behavior(ans, sess, SID)
ext.reset(sess)
smap = {s.intent_id: s for s in scores}
for t in targets:
s = smap.get(t)
if s is None:
continue
total += 1
if s.rank_change > 0: # baseline λŒ€λΉ„ μˆœμœ„ μƒμŠΉ
rose += 1
if s.rank <= 5:
in_top5 += 1
print("── B. 행동 응닡성 (λ¬΄μž‘μœ„ ν”„λ‘œν•„ Γ— λ¬΄μž‘μœ„ 행동, k=%d) ──" % k)
print(f" ν΄λ¦­ν•œ ν–‰λ™μ˜ κ΄€λ ¨ intentκ°€ baseline λŒ€λΉ„ μˆœμœ„ μƒμŠΉ: {rose/total*100:.1f}%")
print(f" 클릭 ν›„ κ·Έ intentκ°€ Top-5 μ§„μž…: {in_top5/total*100:.1f}%")
# ── A. 초기 λΆ„ν¬μ˜ ν”„λ‘œν•„ 반영 (λŒ€μ‘° ν”„λ‘œν•„) ────────────────────
def initial_reflects_profile() -> None:
names = _intent_names()
eng = get_engine(SID)
# μ˜λ„μ μœΌλ‘œ λŒ€μ‘°λ˜λŠ” μž„μ˜ ν”„λ‘œν•„ (페λ₯΄μ†Œλ‚˜ μ •μ˜ μ•„λ‹˜)
profiles = {
"μ•½μ •λ§Œλ£Œ+ν’ˆμ§ˆλΆˆλ§Œ(μ΄νƒˆμ§•ν›„)": {"Q1":"A","Q2":"D","Q3":"C","Q4":"A","Q5":"C","Q6":"A","Q7":"C","Q8":"C","Q9":"D","Q10":"A","Q11":"C","Q12":"A"},
"κ²°ν•©λ―Έλ³΄μœ +κ°€μ‘±ε€š(ν™•μž₯μ—¬μ§€)": {"Q1":"B","Q2":"B","Q3":"C","Q4":"C","Q5":"A","Q6":"C","Q7":"A","Q8":"A","Q9":"B","Q10":"B","Q11":"F","Q12":"C"},
"ν˜œνƒμ• ν˜Έ+프리미엄(VIP)": {"Q1":"A","Q2":"C","Q3":"D","Q4":"B","Q5":"C","Q6":"A","Q7":"C","Q8":"B","Q9":"D","Q10":"C","Q11":"F","Q12":"C"},
"μ €κ°€+λΉ„μš©λ―Όκ°": {"Q1":"B","Q2":"B","Q3":"A","Q4":"A","Q5":"A","Q6":"A","Q7":"B","Q8":"A","Q9":"A","Q10":"A","Q11":"A","Q12":"D"},
}
print("\n── A. 초기 λΆ„ν¬μ˜ ν”„λ‘œν•„ 반영 (μž„μ˜ λŒ€μ‘° ν”„λ‘œν•„) ──")
for label, ans in profiles.items():
_, scores = infer_batch(ans, SID)
top = sorted(scores, key=lambda s: s.final_score, reverse=True)[:5]
print(f" [{label}] Top-5: " + ", ".join(f"{s.intent_id} {names[s.intent_id][:12]}" for s in top))
# ── B2. 단일 μž„μ˜ ν”„λ‘œν•„μ˜ 행동별 λ³€ν™” walkthrough ──────────────
def walkthrough() -> None:
names = _intent_names()
bmap = _behavior_map()
sig = config.get_behavior_signals(SID)
ext = get_extractor()
ans = {"Q1":"A","Q2":"C","Q3":"C","Q4":"B","Q5":"B","Q6":"A","Q7":"B","Q8":"B","Q9":"C","Q10":"B","Q11":"F","Q12":"C"} # 쀑립적 μž„μ˜
seq = ["1-B", "2-B1", "1-C", "2-C2", "1-D", "2-D3"] # κ²°ν•©μ‘°νšŒβ†’κ°€μ‘±κ²°ν•©β†’ν™ˆνƒμƒ‰β†’IPTVβ†’μ•½μ •β†’μž¬μ•½μ •ν˜œνƒ
print("\n── B2. μž„μ˜ ν”„λ‘œν•„ 행동 walkthrough (ν–‰λ™λ§ˆλ‹€ μƒμŠΉ intent) ──")
_, base = infer_batch(ans, SID)
base_rank = {s.intent_id: s.rank for s in base}
sess = "__walk"; ext.reset(sess)
for bid in seq:
et, en = bmap.get(bid, (None, None))
if not et:
continue
ext.add_event(sess, et, en)
_, sc = infer_with_behavior(ans, sess, SID)
risers = sorted(sc, key=lambda s: s.rank_change, reverse=True)[:3]
tgt = sig.get(en, [])
print(f" click {bid}({en}) β†’ μ‹ ν˜Έ={tgt}")
print(" μ΅œλŒ€ μƒμŠΉ: " + ", ".join(f"{s.intent_id} {names[s.intent_id][:10]}(↑{s.rank_change},#{s.rank})" for s in risers))
ext.reset(sess)
if __name__ == "__main__":
behavior_responsiveness()
initial_reflects_profile()
walkthrough()