File size: 5,772 Bytes
e48a2d8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | // egress_backoff_test.go 出口级失败归因(借鉴 ardeyouxipianyi/workbuddy2api-hub 的
// wb_egress.py / issue #271)。
//
// 钉住两件事:
// 1. 直连路径的传输层失败记在**出口**上,不记在账号上——一次链路抖动不该把
// 恰好被轮转到的号逐个降权(hub 现场:4 小时 196 次 URLError 降权 13 个号)。
// 2. 阶梯是秒级且封顶 15s;连续失败到 egressGiveUpStreak 后不再换号重试。
package server
import (
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/misakano7545/workbuddy2api-panel/internal/auth"
"github.com/misakano7545/workbuddy2api-panel/internal/upstream"
)
// TestEgressBackoffLadder 阶梯:1s 起、逐次翻倍、封顶 15s、±25% 抖动;
// base<=0(测试)恒 0,streak<=0 不等待。
func TestEgressBackoffLadder(t *testing.T) {
base := time.Second
cases := []struct {
streak int
want time.Duration
}{
{0, 0},
{1, 1 * time.Second},
{2, 2 * time.Second},
{3, 4 * time.Second},
{4, 8 * time.Second},
{5, 15 * time.Second},
{9, 15 * time.Second}, // 封顶:不做 2^n 溢出
}
for _, c := range cases {
got := egressBackoff(c.streak, base)
if c.want == 0 {
if got != 0 {
t.Errorf("streak=%d got=%v want 0", c.streak, got)
}
continue
}
lo := time.Duration(float64(c.want) * 0.75)
hi := time.Duration(float64(c.want) * 1.25)
if got < lo || got > hi {
t.Errorf("streak=%d got=%v want 落在 [%v,%v]", c.streak, got, lo, hi)
}
}
if got := egressBackoff(3, 0); got != 0 {
t.Errorf("base=0(测试模式)必须恒 0,got=%v", got)
}
}
// egressTransportErrUpstream 所有请求都在传输层失败(模拟直连链路断掉)。
func egressTransportErrUpstream(calls *int) *upstream.Client {
return &upstream.Client{
HTTP: &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
*calls++
return nil, errors.New("dial tcp: connection refused")
})},
ChatBaseCN: "https://fake.example",
BillingBaseCN: "https://fake.example",
}
}
// TestTransportFailureChargedToEgressNotAccount ⭐ 核心:三个号、链路全断,
// 一个请求轮转打满 MaxRotate。断言**没有**任何账号背上连败/降权,而出口计数
// 恰好等于实际上游调用次数——这就是「归因正确」的定义。
//
// 旧行为:每次传输层失败都喂 NoteFailures(该轮被选中的号)。链路抖动几秒、
// 请求连发几轮,整池号会一个个被降到出池,而链路本身早已恢复。
func TestTransportFailureChargedToEgressNotAccount(t *testing.T) {
var calls int
p := testPoolWith(
&auth.Auth{UID: "e1", AccessToken: "at-e1", ExpiresAt: 9999999999},
&auth.Auth{UID: "e2", AccessToken: "at-e2", ExpiresAt: 9999999999},
&auth.Auth{UID: "e3", AccessToken: "at-e3", ExpiresAt: 9999999999},
)
h := NewHandler(Config{Pool: p, Upstream: egressTransportErrUpstream(&calls)})
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
strings.NewReader(`{"model":"glm-5.2","messages":[{"role":"user","content":"hi"}]}`)))
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("code=%d body=%s want 503", rec.Code, rec.Body)
}
if calls == 0 {
t.Fatal("没有任何上游调用:轮转没跑起来,本用例失去意义")
}
for _, uid := range []string{"e1", "e2", "e3"} {
st, _ := p.Status(uid)
if st.ConsecutiveFails != 0 || !st.DegradeUntil.IsZero() {
t.Errorf("账号 %s 不该为链路抖动背惩罚: consecutiveFails=%d degradeUntil=%v",
uid, st.ConsecutiveFails, st.DegradeUntil)
}
}
if got := p.EgressStreak(egressKey(&auth.Auth{UID: "e1"}), time.Now()); got != calls {
t.Errorf("出口计数=%d want %d(等于本次请求的上游调用数)", got, calls)
}
}
// TestEgressGiveUpStopsRotating 连续失败到 egressGiveUpStreak 之后,下一个请求
// 只打一次上游就 503:同一出口换号换不到别的链路,轮转只是把同样的等待与失败
// 重复 MaxRotate 遍。
func TestEgressGiveUpStopsRotating(t *testing.T) {
var calls int
p := testPoolWith(
&auth.Auth{UID: "g1", AccessToken: "at-g1", ExpiresAt: 9999999999},
&auth.Auth{UID: "g2", AccessToken: "at-g2", ExpiresAt: 9999999999},
&auth.Auth{UID: "g3", AccessToken: "at-g3", ExpiresAt: 9999999999},
)
h := NewHandler(Config{Pool: p, Upstream: egressTransportErrUpstream(&calls)})
send := func() {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
strings.NewReader(`{"model":"glm-5.2","messages":[{"role":"user","content":"hi"}]}`)))
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("code=%d body=%s want 503", rec.Code, rec.Body)
}
}
send()
first := calls
if first < egressGiveUpStreak-1 {
t.Fatalf("首个请求只打了 %d 次上游,没到 egressGiveUpStreak=%d,用例前提不成立",
first, egressGiveUpStreak)
}
send()
if second := calls - first; second != 1 {
t.Fatalf("链路已判定为断:第二个请求应只打 1 次上游,实际 %d 次(轮转被放行)", second)
}
}
// TestEgressStreakSeparatedByRealm cn 与 global 是不同的上游环境:cn 抖动不得
// 让 global 的请求提前放弃轮转。
func TestEgressStreakSeparatedByRealm(t *testing.T) {
if got := egressKey(&auth.Auth{UID: "x", AccessToken: "a"}); got != "direct:cn" {
t.Fatalf("无 realm 时应归 cn: %q", got)
}
auth.SetGlobalEnabled(true)
ga := &auth.Auth{UID: "x", AccessToken: "a", Domain: "www.workbuddy.ai"}
if got := egressKey(ga); got != "direct:global" {
t.Fatalf("realm=global: %q", got)
}
if got := egressKey(nil); got != "direct:cn" {
t.Fatalf("nil 账号应归 cn: %q", got)
}
}
|