File size: 9,180 Bytes
216004d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
# -*- coding: utf-8 -*-
"""OaK 的 Model→Planning 层在 KeyDoor 上的基准 —— 指标 = **look-ahead 操作数**。

论文(Sutton et al. 2022)用它自己的指标衡量这套机制:一个"让规划更省"的
抽象,只有在**规划代价**上才看得见收益(§9.2:"Use the source's own metric
for the mechanism")。本项目已在 gridworld 上复现了论文的双向结果:

    两房间 (36 状态)   primitive 1400 / shortest 1750 / **RR 875**
    四房间 (80 状态)   primitive 48872 / shortest 53710 / **RR 26200**

本文件把同一件事搬到 **KeyDoorMDP**(时序结构已证:A ∈ [−1,+1], 33% 显著)。

## 消融矩阵(相邻两臂只差**一个**变量)

  arm            c(cumulant)  z(stopping value)        作用
  ─────────────  ───────────  ────────────────────────  ──────────────────
  primitive      —            —                         下界基线(无 option)
  rr-exact       R            V_main + (w̄−w_i)x_i        **机制本体**
  rr-zeroV       R            0      + (w̄−0 )x_i        ← 削掉**价值函数**
  rr-nobonus     R            V_main + (0 )x_i   ≡ V_main ← 削掉停止奖励(已知退化)
  rr-nocum       0            V_main + (w̄−w_i)x_i        ← 削掉**沿途主任务奖励**
  rr-nofeat      R            V_main + (w̄−w_i)x_uniform  ← 削掉**特征语义**
  bottleneck     −1           0 @ 瓶颈                   **已知弱(论文点名)**
  random         R            V_main + (w̄−w_i)x_random   **随机目标(负对照)**

★ 判据**在跑之前写死**(写进本文件, 见 `INTERPRETATION_RULES`), 免得数字
  出来以后往想要的方向读。
"""
import json
import os
import sys

import numpy as np

sys.path.insert(0, ".")
from hibs_lnn.rr_options import (                            # noqa: E402
    KeyDoorTabular, build_bottleneck_library, build_random_goal_library,
    build_rr_library, feature_vectors, main_value,
)
from hibs_lnn.subtask_options import (                       # noqa: E402
    build_option_model, ops_to_tolerance,
)

TOL = 0.01
MAX_OPS = 400000

# ── 判据写死在此:跑完**先读这里再读数字** ────────────────────────────
INTERPRETATION_RULES = [
    "① 内部哨兵: primitive 臂必须复现历史值(两房间 1400 量级的同型行为)。"
    "若 primitive 臂与历史不符 -> 本 harness 自身有 bug, 其余数字全部作废。",
    "② 已知负向: bottleneck 臂**不得**优于 primitive。若它赢了 -> harness 有 bug"
    "(论文明确指出瓶颈子目标是已知弱基线)。",
    "③ 机制判据: rr-exact 优于 primitive 才叫'机制有效'。"
    "相等等不于'无效', 而是'这个 regime 上无可抽象的结构', 必须换 regime 再说。",
    "④ 价值函数假设(用户的猜测): 比较 rr-exact vs rr-zeroV。"
    "两者**逐位相同** = 该 regime 下价值函数不影响规划 -> 假设**未被这个试验台检验**"
    "(不是'被证伪');两者不同且 rr-exact 更省 = 价值函数是收益来源。",
    "⑤ 只在 tol=0.01 与 max_ops=400000 下读数:'打满上限'的臂记为不收敛, 不当作有限值比较。",
    "⑥ 单 regime 的结果只是**一个点**; 结论要在多 regime 上一致才成立。",
]


def arm_libraries(mdl, V_main):
    """构造矩阵里每个臂的 option 库。"""
    arms = {}
    arms["rr-exact"] = build_rr_library(mdl, V_main, bonus_weight=1.0)
    arms["rr-zeroV"] = build_rr_library(mdl, V_main, bonus_weight=1.0,
                                        z_mode="zeroV")
    arms["rr-nobonus"] = build_rr_library(mdl, V_main, bonus_weight=1.0,
                                          z_mode="nobonus")
    arms["rr-nocum"] = build_rr_library(mdl, V_main, bonus_weight=1.0,
                                        cumulant="zero")
    # 削掉特征语义: 特征用一个与任务无关的均匀特征(半数为 1)
    n = mdl.n_states
    x_unif = np.array([1.0 if (s % 2 == 0) else 0.0 for s in range(n)])
    from hibs_lnn.subtask_options import reward_respecting_subtask
    st_u = reward_respecting_subtask(mdl, x_unif, V_main, bonus_weight=1.0)
    st_u.name = "rr[nofeat]"
    st_u.feature = x_unif
    st_u.solve(mdl)
    arms["rr-nofeat"] = [st_u]
    arms["bottleneck"] = build_bottleneck_library(mdl)
    arms["random"] = build_random_goal_library(mdl, V_main, seed=0)
    return arms


def eval_arm(mdl, lib, v_star, s0=0):
    models = {}
    ks = []
    for st in lib:
        name = st.name
        if name in models:                      # 重名 -> 加后缀
            name = f"{name}#{len(models)}"
        M = build_option_model(mdl, st)
        models[name] = M
        # ★ 时长仪表: ops 指标可能被"option 有多长"主导而不是被"目标选得好不好"
        #   主导 —— 一个 k 步的 option 只花 1 次 backup 就传播 k 步的价值。
        #   记录每个 option 的期望终止步数, 才能把这个 confound 读出来。
        for s in range(mdl.n_states):
            for ent in M[s]:
                ks.append(ent[3])
    ops, errs, ops_curve, err_curve = ops_to_tolerance(
        mdl, models, v_star, s0, tol=TOL, max_ops=MAX_OPS)
    return {"ops": int(ops), "err": float(errs), "converged": bool(errs < TOL),
            "n_options": len(models), "n_sweeps": len(ops_curve),
            "ops_per_sweep": (float(ops) / max(1, len(ops_curve))),
            "mean_k": float(np.mean(ks)) if ks else 1.0,
            "max_k": float(np.max(ks)) if ks else 1.0,
            "beta_total": int(sum(int(np.sum(st.beta)) for st in lib))}


def main():
    regimes = [
        (2, 5, 7, "A: key=2 door=5 goal=7 (需全链)"),
        (1, 4, 7, "B: key=1 door=4 goal=7 (需全链)"),
        (3, 6, 7, "C: key=3 door=6 goal=7 (需全链)"),
        (2, 5, 3, "D: goal<door (免钥匙, 门无关)"),
    ]
    out = {"tol": TOL, "max_ops": MAX_OPS, "rules": INTERPRETATION_RULES,
           "regimes": []}

    print("=" * 78)
    print("KeyDoor 规划基准  |  指标 = look-ahead 操作数到 |V(s0)-V*| < %.2f" % TOL)
    print("=" * 78)

    for kp, dp, gp, label in regimes:
        from hibs_lnn.skill_mdp import KeyDoorMDP
        mdp = KeyDoorMDP(n_pos=8, key_pos=kp, door_pos=dp, goal_pos=gp, horizon=30)
        mdp.set_regime(kp, dp, gp)
        mdl = KeyDoorTabular(mdp, gamma=0.95)
        V_main = main_value(mdl)
        v_star = float(V_main[0])

        # ── ① 内部哨兵: primitive ────────────────────────────────────
        base = eval_arm(mdl, [], v_star)
        row = {"regime": (kp, dp, gp), "label": label,
               "v_star": v_star, "primitive": base, "arms": {}}

        print(f"\n{label}   V*(s0)={v_star:.6f}   |goal|={int(mdl.is_goal.sum())}")
        print(f"  {'arm':12s} {'ops':>7s} {'×prim':>6s} {'conv':>5s} "
              f"{'mean_k':>7s} {'max_k':>6s} {'n_opt':>5s} {'|β|':>5s} {'ops/扫':>8s}")
        print(f"  {'primitive':12s} {base['ops']:7d} {1.0:6.3f} "
              f"{str(base['converged']):>5s} {1.0:7.2f} {1.0:6.2f} {'-':>5s} "
              f"{'-':>5s} {base['ops_per_sweep']:8.1f}")

        for name, lib in arm_libraries(mdl, V_main).items():
            r = eval_arm(mdl, lib, v_star)
            r["ratio_vs_primitive"] = r["ops"] / max(1, base["ops"])
            row["arms"][name] = r
            print(f"  {name:12s} {r['ops']:7d} {r['ratio_vs_primitive']:6.3f} "
                  f"{str(r['converged']):>5s} {r['mean_k']:7.2f} {r['max_k']:6.2f} "
                  f"{r['n_options']:5d} {r['beta_total']:5d} {r['ops_per_sweep']:8.1f}")
        out["regimes"].append(row)

    # ── w̄ 扫描(dose-response: 中间值应当最好, 见配方 §9.3 末) ────────
    print("\n" + "=" * 78)
    print("w̄ 扫描(regime A)—— 论文: 中间值最好; w̄→大 退化成 shortest-path")
    print("=" * 78)
    kp, dp, gp = 2, 5, 7
    from hibs_lnn.skill_mdp import KeyDoorMDP
    mdp = KeyDoorMDP(n_pos=8, key_pos=kp, door_pos=dp, goal_pos=gp, horizon=30)
    mdp.set_regime(kp, dp, gp)
    mdl = KeyDoorTabular(mdp)
    V_main = main_value(mdl)
    base = eval_arm(mdl, [], float(V_main[0]))
    wbar = {}
    print(f"  primitive: ops={base['ops']}")
    for w in (0.0, 0.1, 0.3, 0.5, 0.78, 1.0, 3.0, 10.0, 100.0):
        lib = build_rr_library(mdl, V_main, bonus_weight=w)
        r = eval_arm(mdl, lib, float(V_main[0]))
        wbar[str(w)] = r
        print(f"  w̄={w:<6} ops={r['ops']:8d}  err={r['err']:.2e} "
              f"conv={r['converged']}  ×prim={r['ops']/max(1,base['ops']):.3f}  "
              f"|β|={r['beta_total']}")
    out["wbar_sweep_regimeA"] = {"primitive_ops": base["ops"], "arms": wbar}

    os.makedirs("results", exist_ok=True)
    with open("results/rr_planning_keydoor.json", "w") as f:
        json.dump(out, f, indent=2, ensure_ascii=False)
    print("\n→ results/rr_planning_keydoor.json")


if __name__ == "__main__":
    main()