File size: 9,206 Bytes
e48a2d8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
// Package alert 账号池可用性阈值告警。Enabled=false 时 Run 直接返回。
package alert

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	defaultInterval     = 30 * time.Second
	defaultTimeout      = 5 * time.Second
	defaultForTicks     = 2
	defaultClearTicks   = 2
	defaultStartupGrace = 30 * time.Second
	defaultRecover      = 1
	defaultServiceName  = "workbuddy2api"
)

type Health struct {
	Total, Healthy, Cooling, Disabled, Breaker, Degraded, InFlight int
}

type Source interface {
	Health(realm string) Health
	WAFActive() bool
}

type Config struct {
	Enabled          bool
	WebhookURL       string
	Secret           string
	Interval         time.Duration
	Timeout          time.Duration
	StartupGrace     time.Duration
	MinHealthyCN     int
	MinHealthyGlobal int
	RecoverHealthy   int
	BreakerThreshold int
	ForTicks         int
	ClearTicks       int
	SendResolve      bool
	ServiceName      string
}

func (c *Config) normalize() {
	if c.Interval <= 0 {
		c.Interval = defaultInterval
	}
	if c.Timeout <= 0 {
		c.Timeout = defaultTimeout
	}
	if c.StartupGrace <= 0 {
		c.StartupGrace = defaultStartupGrace
	}
	if c.ForTicks <= 0 {
		c.ForTicks = defaultForTicks
	}
	if c.ClearTicks <= 0 {
		c.ClearTicks = defaultClearTicks
	}
	if c.RecoverHealthy <= 0 {
		c.RecoverHealthy = defaultRecover
	}
	if c.ServiceName == "" {
		c.ServiceName = defaultServiceName
	}
}

type ruleState int

const (
	stateOK ruleState = iota
	statePending
	stateFiring
)

type rule struct {
	id, severity, realm string
	sample              func(Source) (value float64, bad, recovered bool, threshold float64, msg string)
	state               ruleState
	badTicks            int
	clearCount          int
	since               time.Time
}

type event int

const (
	eventNone event = iota
	eventAlert
	eventResolve
)

func (r *rule) step(now time.Time, src Source, forTicks, clearTicks int) (event, float64, float64, string) {
	v, bad, recovered, th, msg := r.sample(src)
	switch r.state {
	case stateOK:
		if !bad {
			return eventNone, 0, 0, ""
		}
		r.state = statePending
		r.badTicks = 0
		fallthrough
	case statePending:
		if !bad {
			r.state = stateOK
			r.badTicks = 0
			return eventNone, 0, 0, ""
		}
		r.badTicks++
		if r.badTicks >= forTicks {
			r.state = stateFiring
			r.since = now
			r.clearCount = 0
			return eventAlert, v, th, msg
		}
	case stateFiring:
		if recovered {
			r.clearCount++
			if r.clearCount >= clearTicks {
				r.state = stateOK
				r.badTicks = 0
				r.clearCount = 0
				return eventResolve, v, th, msg
			}
		} else {
			r.clearCount = 0
		}
	}
	return eventNone, 0, 0, ""
}

func newRules(cfg Config) []*rule {
	var rs []*rule
	for _, rc := range []struct {
		realm string
		min   int
	}{{"cn", cfg.MinHealthyCN}, {"global", cfg.MinHealthyGlobal}} {
		if rc.min <= 0 {
			continue
		}
		realm, min := rc.realm, rc.min
		rs = append(rs, &rule{
			id: "realm_healthy_low", severity: "critical", realm: realm,
			sample: func(src Source) (float64, bool, bool, float64, string) {
				v := float64(src.Health(realm).Healthy)
				bad := v < float64(min)
				return v, bad, v >= float64(cfg.RecoverHealthy), float64(min),
					fmt.Sprintf("%s realm healthy accounts (%d) below threshold (%d)", realm, int(v), min)
			},
		})
	}
	if cfg.BreakerThreshold > 0 {
		th := cfg.BreakerThreshold
		rs = append(rs, &rule{
			id: "breaker_high", severity: "warning",
			sample: func(src Source) (float64, bool, bool, float64, string) {
				v := float64(src.Health("cn").Breaker + src.Health("global").Breaker)
				return v, v >= float64(th), v < float64(th), float64(th),
					fmt.Sprintf("accounts in circuit-breaker (%d) reached threshold (%d)", int(v), th)
			},
		})
	}
	rs = append(rs, &rule{
		id: "waf_ip_block", severity: "warning",
		sample: func(src Source) (float64, bool, bool, float64, string) {
			active := src.WAFActive()
			v := 0.0
			if active {
				v = 1
			}
			return v, active, !active, 1, "IP-level WAF block is active"
		},
	})
	return rs
}

type Snapshot struct {
	CN        RealmSnapshot `json:"cn"`
	Global    RealmSnapshot `json:"global"`
	WAFActive bool          `json:"waf_active"`
}

type RealmSnapshot struct {
	Total    int `json:"total"`
	Healthy  int `json:"healthy"`
	Cooling  int `json:"cooling"`
	Disabled int `json:"disabled"`
	Breaker  int `json:"breaker"`
	// Degraded 连败降权中的账号数。alert.Health 一直有这个观测,但快照转换时
	// 被丢弃——webhook 接收方永远看不到「有一批号在连败降权」,只能等它恶化成
	// healthy 不足才被 realm_healthy_low 告警。
	Degraded int `json:"degraded"`
	InFlight int `json:"in_flight"`
}

type Payload struct {
	Version   int       `json:"version"`
	Event     string    `json:"event"`
	Service   string    `json:"service"`
	Rule      string    `json:"rule"`
	Severity  string    `json:"severity"`
	Realm     string    `json:"realm,omitempty"`
	Value     float64   `json:"value"`
	Threshold float64   `json:"threshold"`
	Message   string    `json:"message"`
	Since     time.Time `json:"since"`
	FiredAt   time.Time `json:"fired_at"`
	Snapshot  Snapshot  `json:"snapshot"`
}

type Monitor struct {
	cfg       Config
	src       Source
	rules     []*rule
	client    *http.Client
	startedAt time.Time
	mu        sync.Mutex
	stop      chan struct{}
	stopOnce  sync.Once
}

func New(cfg Config, src Source) *Monitor {
	cfg.normalize()
	return &Monitor{
		cfg: cfg, src: src, rules: newRules(cfg), startedAt: time.Now(),
		client: &http.Client{
			Timeout: cfg.Timeout,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				if len(via) >= 2 {
					return fmt.Errorf("too many redirects")
				}
				if req.URL.Host != via[0].URL.Host {
					return fmt.Errorf("cross-host redirect refused")
				}
				return nil
			},
		},
	}
}

func (m *Monitor) Run(ctx context.Context) {
	if !m.cfg.Enabled || m.src == nil {
		return
	}
	m.mu.Lock()
	if m.stop != nil {
		m.mu.Unlock()
		return
	}
	stop := make(chan struct{})
	m.stop = stop
	m.mu.Unlock()
	t := time.NewTicker(m.cfg.Interval)
	defer t.Stop()
	m.Tick(time.Now())
	for {
		select {
		case <-ctx.Done():
			return
		case <-stop:
			return
		case <-t.C:
			m.Tick(time.Now())
		}
	}
}

func (m *Monitor) Stop() {
	m.stopOnce.Do(func() {
		m.mu.Lock()
		defer m.mu.Unlock()
		if m.stop != nil {
			close(m.stop)
			m.stop = nil
		}
	})
}

func (m *Monitor) Tick(now time.Time) {
	if !m.cfg.Enabled || m.src == nil {
		return
	}
	if now.Sub(m.startedAt) < m.cfg.StartupGrace {
		return
	}
	for _, r := range m.rules {
		ev, value, threshold, msg := r.step(now, m.src, m.cfg.ForTicks, m.cfg.ClearTicks)
		switch ev {
		case eventAlert:
			m.send("alert", r, now, value, threshold, msg)
		case eventResolve:
			if m.cfg.SendResolve {
				m.send("resolve", r, now, value, threshold, msg)
			}
		}
	}
}

func (m *Monitor) snapshot() Snapshot {
	to := func(h Health) RealmSnapshot {
		return RealmSnapshot{Total: h.Total, Healthy: h.Healthy, Cooling: h.Cooling, Disabled: h.Disabled, Breaker: h.Breaker, Degraded: h.Degraded, InFlight: h.InFlight}
	}
	return Snapshot{CN: to(m.src.Health("cn")), Global: to(m.src.Health("global")), WAFActive: m.src.WAFActive()}
}

func (m *Monitor) send(eventName string, r *rule, now time.Time, value, threshold float64, msg string) {
	m.post(Payload{
		Version: 1, Event: eventName, Service: m.cfg.ServiceName, Rule: r.id, Severity: r.severity,
		Realm: r.realm, Value: value, Threshold: threshold, Message: msg, Since: r.since, FiredAt: now,
		Snapshot: m.snapshot(),
	}, r.id)
}

// Notify 推送一条通知(无阈值语义的事件型消息,如排程轮次结果汇总)。
// 未启用或无 webhook 时静默——通知是旁路,不该影响调用方主流程。
func (m *Monitor) Notify(message string) {
	if !m.cfg.Enabled || m.cfg.WebhookURL == "" {
		return
	}
	m.post(Payload{
		Version: 1, Event: "notify", Service: m.cfg.ServiceName, Severity: "info",
		Message: message, FiredAt: time.Now(), Snapshot: m.snapshot(),
	}, "notify")
}

// post 发 payload 到 webhook(HMAC 签名同规则事件;失败只记日志,不重试)。
func (m *Monitor) post(p Payload, tag string) {
	body, err := json.Marshal(p)
	if err != nil {
		log.Printf("WARN: [alert] marshal payload rule=%s: %v", tag, err)
		return
	}
	req, err := http.NewRequest(http.MethodPost, m.cfg.WebhookURL, bytes.NewReader(body))
	if err != nil {
		log.Printf("WARN: [alert] build request rule=%s: %v", tag, err)
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "workbuddy2api-alert/1")
	if m.cfg.Secret != "" {
		mac := hmac.New(sha256.New, []byte(m.cfg.Secret))
		mac.Write(body)
		req.Header.Set("X-WB2A-Signature", "sha256="+hex.EncodeToString(mac.Sum(nil)))
	}
	resp, err := m.client.Do(req)
	if err != nil {
		log.Printf("WARN: [alert] webhook post failed rule=%s: %v", tag, err)
		return
	}
	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
	_ = resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		log.Printf("WARN: [alert] webhook returned %d rule=%s", resp.StatusCode, tag)
	}
}