File size: 5,772 Bytes
e48a2d8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
// egress_backoff_test.go 出口级失败归因(借鉴 ardeyouxipianyi/workbuddy2api-hub 的
// wb_egress.py / issue #271)。
//
// 钉住两件事:
//   1. 直连路径的传输层失败记在**出口**上,不记在账号上——一次链路抖动不该把
//      恰好被轮转到的号逐个降权(hub 现场:4 小时 196 次 URLError 降权 13 个号)。
//   2. 阶梯是秒级且封顶 15s;连续失败到 egressGiveUpStreak 后不再换号重试。
package server

import (
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/misakano7545/workbuddy2api-panel/internal/auth"
	"github.com/misakano7545/workbuddy2api-panel/internal/upstream"
)

// TestEgressBackoffLadder 阶梯:1s 起、逐次翻倍、封顶 15s、±25% 抖动;
// base<=0(测试)恒 0,streak<=0 不等待。
func TestEgressBackoffLadder(t *testing.T) {
	base := time.Second
	cases := []struct {
		streak int
		want   time.Duration
	}{
		{0, 0},
		{1, 1 * time.Second},
		{2, 2 * time.Second},
		{3, 4 * time.Second},
		{4, 8 * time.Second},
		{5, 15 * time.Second},
		{9, 15 * time.Second}, // 封顶:不做 2^n 溢出
	}
	for _, c := range cases {
		got := egressBackoff(c.streak, base)
		if c.want == 0 {
			if got != 0 {
				t.Errorf("streak=%d got=%v want 0", c.streak, got)
			}
			continue
		}
		lo := time.Duration(float64(c.want) * 0.75)
		hi := time.Duration(float64(c.want) * 1.25)
		if got < lo || got > hi {
			t.Errorf("streak=%d got=%v want 落在 [%v,%v]", c.streak, got, lo, hi)
		}
	}
	if got := egressBackoff(3, 0); got != 0 {
		t.Errorf("base=0(测试模式)必须恒 0,got=%v", got)
	}
}

// egressTransportErrUpstream 所有请求都在传输层失败(模拟直连链路断掉)。
func egressTransportErrUpstream(calls *int) *upstream.Client {
	return &upstream.Client{
		HTTP: &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
			*calls++
			return nil, errors.New("dial tcp: connection refused")
		})},
		ChatBaseCN:    "https://fake.example",
		BillingBaseCN: "https://fake.example",
	}
}

// TestTransportFailureChargedToEgressNotAccount ⭐ 核心:三个号、链路全断,
// 一个请求轮转打满 MaxRotate。断言**没有**任何账号背上连败/降权,而出口计数
// 恰好等于实际上游调用次数——这就是「归因正确」的定义。
//
// 旧行为:每次传输层失败都喂 NoteFailures(该轮被选中的号)。链路抖动几秒、
// 请求连发几轮,整池号会一个个被降到出池,而链路本身早已恢复。
func TestTransportFailureChargedToEgressNotAccount(t *testing.T) {
	var calls int
	p := testPoolWith(
		&auth.Auth{UID: "e1", AccessToken: "at-e1", ExpiresAt: 9999999999},
		&auth.Auth{UID: "e2", AccessToken: "at-e2", ExpiresAt: 9999999999},
		&auth.Auth{UID: "e3", AccessToken: "at-e3", ExpiresAt: 9999999999},
	)
	h := NewHandler(Config{Pool: p, Upstream: egressTransportErrUpstream(&calls)})
	rec := httptest.NewRecorder()
	h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
		strings.NewReader(`{"model":"glm-5.2","messages":[{"role":"user","content":"hi"}]}`)))
	if rec.Code != http.StatusServiceUnavailable {
		t.Fatalf("code=%d body=%s want 503", rec.Code, rec.Body)
	}
	if calls == 0 {
		t.Fatal("没有任何上游调用:轮转没跑起来,本用例失去意义")
	}
	for _, uid := range []string{"e1", "e2", "e3"} {
		st, _ := p.Status(uid)
		if st.ConsecutiveFails != 0 || !st.DegradeUntil.IsZero() {
			t.Errorf("账号 %s 不该为链路抖动背惩罚: consecutiveFails=%d degradeUntil=%v",
				uid, st.ConsecutiveFails, st.DegradeUntil)
		}
	}
	if got := p.EgressStreak(egressKey(&auth.Auth{UID: "e1"}), time.Now()); got != calls {
		t.Errorf("出口计数=%d want %d(等于本次请求的上游调用数)", got, calls)
	}
}

// TestEgressGiveUpStopsRotating 连续失败到 egressGiveUpStreak 之后,下一个请求
// 只打一次上游就 503:同一出口换号换不到别的链路,轮转只是把同样的等待与失败
// 重复 MaxRotate 遍。
func TestEgressGiveUpStopsRotating(t *testing.T) {
	var calls int
	p := testPoolWith(
		&auth.Auth{UID: "g1", AccessToken: "at-g1", ExpiresAt: 9999999999},
		&auth.Auth{UID: "g2", AccessToken: "at-g2", ExpiresAt: 9999999999},
		&auth.Auth{UID: "g3", AccessToken: "at-g3", ExpiresAt: 9999999999},
	)
	h := NewHandler(Config{Pool: p, Upstream: egressTransportErrUpstream(&calls)})
	send := func() {
		rec := httptest.NewRecorder()
		h.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
			strings.NewReader(`{"model":"glm-5.2","messages":[{"role":"user","content":"hi"}]}`)))
		if rec.Code != http.StatusServiceUnavailable {
			t.Fatalf("code=%d body=%s want 503", rec.Code, rec.Body)
		}
	}
	send()
	first := calls
	if first < egressGiveUpStreak-1 {
		t.Fatalf("首个请求只打了 %d 次上游,没到 egressGiveUpStreak=%d,用例前提不成立",
			first, egressGiveUpStreak)
	}
	send()
	if second := calls - first; second != 1 {
		t.Fatalf("链路已判定为断:第二个请求应只打 1 次上游,实际 %d 次(轮转被放行)", second)
	}
}

// TestEgressStreakSeparatedByRealm cn 与 global 是不同的上游环境:cn 抖动不得
// 让 global 的请求提前放弃轮转。
func TestEgressStreakSeparatedByRealm(t *testing.T) {
	if got := egressKey(&auth.Auth{UID: "x", AccessToken: "a"}); got != "direct:cn" {
		t.Fatalf("无 realm 时应归 cn: %q", got)
	}
	auth.SetGlobalEnabled(true)
	ga := &auth.Auth{UID: "x", AccessToken: "a", Domain: "www.workbuddy.ai"}
	if got := egressKey(ga); got != "direct:global" {
		t.Fatalf("realm=global: %q", got)
	}
	if got := egressKey(nil); got != "direct:cn" {
		t.Fatalf("nil 账号应归 cn: %q", got)
	}
}