ds2api / internal /reqlog /store.go
luckfun233
feat(logs): 请求日志页面与只读日志 API Key;修复禁言账号导致的 503
7e312e3
Raw History Blame Contribute Delete
10.2 kB
package reqlog
import (
"archive/zip"
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
// Entry 是一条请求日志。只记录元数据,不记录请求/响应正文,避免泄露用户内容。
type Entry struct {
ID string `json:"id"`
Time string `json:"time"`
Level string `json:"level"`
Method string `json:"method"`
Path string `json:"path"`
Status int `json:"status"`
LatencyMS int64 `json:"latency_ms"`
ClientIP string `json:"client_ip,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
APIKey string `json:"api_key,omitempty"`
APIKeyName string `json:"api_key_name,omitempty"`
Account string `json:"account,omitempty"`
Model string `json:"model,omitempty"`
Stream bool `json:"stream,omitempty"`
ReqBytes int64 `json:"req_bytes,omitempty"`
RespBytes int64 `json:"resp_bytes,omitempty"`
ErrorCode string `json:"error_code,omitempty"`
Error string `json:"error,omitempty"`
Upstream string `json:"upstream,omitempty"`
Dropped bool `json:"-"`
}
// Options 控制内存窗口与文件轮转。
type Options struct {
Dir string
MemoryLimit int // 内存保留条数
MaxFileBytes int64 // 单个文件大小上限
MaxFiles int // 保留文件数
DisableFile bool
}
func DefaultOptions(dir string) Options {
return Options{
Dir: dir,
MemoryLimit: 2000,
MaxFileBytes: 8 << 20,
MaxFiles: 5,
}
}
// Store 同时提供内存环形窗口与按大小轮转的文件日志。
type Store struct {
mu sync.RWMutex
opts Options
entries []Entry
next int
full bool
seq uint64
file *os.File
fileSize int64
fileName string
fileErr error
}
func New(opts Options) *Store {
if opts.MemoryLimit <= 0 {
opts.MemoryLimit = 2000
}
if opts.MaxFileBytes <= 0 {
opts.MaxFileBytes = 8 << 20
}
if opts.MaxFiles <= 0 {
opts.MaxFiles = 5
}
s := &Store{opts: opts, entries: make([]Entry, opts.MemoryLimit)}
if !opts.DisableFile && strings.TrimSpace(opts.Dir) != "" {
if err := os.MkdirAll(opts.Dir, 0o750); err != nil {
s.fileErr = err
}
}
return s
}
func (s *Store) Options() Options { return s.opts }
func (s *Store) Add(e Entry) {
if e.Time == "" {
e.Time = time.Now().Format("2006-01-02T15:04:05.000Z07:00")
}
s.mu.Lock()
s.seq++
if e.ID == "" {
e.ID = fmt.Sprintf("%d-%06d", time.Now().UnixMilli(), s.seq%1000000)
}
s.entries[s.next] = e
s.next = (s.next + 1) % len(s.entries)
if s.next == 0 {
s.full = true
}
s.writeFileLocked(e)
s.mu.Unlock()
}
func (s *Store) entriesLocked() []Entry {
if !s.full {
return append([]Entry(nil), s.entries[:s.next]...)
}
out := make([]Entry, 0, len(s.entries))
out = append(out, s.entries[s.next:]...)
out = append(out, s.entries[:s.next]...)
return out
}
// Query 过滤内存中的日志,按时间倒序返回。
type Query struct {
Limit int
Level string
Status int
Keyword string
SinceUnix int64
UntilUnix int64
}
func (s *Store) Query(q Query) []Entry {
s.mu.RLock()
all := s.entriesLocked()
s.mu.RUnlock()
limit := q.Limit
if limit <= 0 || limit > len(all) {
limit = len(all)
}
keyword := strings.ToLower(strings.TrimSpace(q.Keyword))
out := make([]Entry, 0, limit)
for i := len(all) - 1; i >= 0 && len(out) < limit; i-- {
e := all[i]
if q.Level != "" && !strings.EqualFold(e.Level, q.Level) {
continue
}
if q.Status != 0 && e.Status != q.Status {
continue
}
if q.SinceUnix > 0 && entryUnix(e) < q.SinceUnix {
continue
}
if q.UntilUnix > 0 && entryUnix(e) > q.UntilUnix {
continue
}
if keyword != "" && !entryMatches(e, keyword) {
continue
}
out = append(out, e)
}
return out
}
func entryUnix(e Entry) int64 {
t, err := time.Parse(time.RFC3339Nano, e.Time)
if err != nil {
return 0
}
return t.Unix()
}
func entryMatches(e Entry, keyword string) bool {
fields := []string{e.Path, e.Account, e.Model, e.ErrorCode, e.Error, e.Upstream, e.APIKeyName, e.APIKey, e.ClientIP}
for _, f := range fields {
if f != "" && strings.Contains(strings.ToLower(f), keyword) {
return true
}
}
return strings.Contains(strings.ToLower(e.Method), keyword) || strings.Contains(strings.ToLower(e.UserAgent), keyword)
}
// Stats 返回内存窗口的聚合信息。
func (s *Store) Stats() map[string]any {
s.mu.RLock()
all := s.entriesLocked()
fileErr := s.fileErr
s.mu.RUnlock()
total := len(all)
errors := 0
warns := 0
lastHour := 0
cutoff := time.Now().Add(-time.Hour).Unix()
for _, e := range all {
if e.Status >= 500 || e.Level == LevelError {
errors++
}
if e.Status >= 400 && e.Status < 500 {
warns++
}
if entryUnix(e) >= cutoff {
lastHour++
}
}
out := map[string]any{
"total": total,
"errors": errors,
"client_errors": warns,
"last_hour": lastHour,
"memory_limit": len(s.entries),
}
if fileErr != nil {
out["file_error"] = fileErr.Error()
}
return out
}
func (s *Store) ensureFileLocked() error {
if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" {
return nil
}
if s.file != nil {
return nil
}
if err := os.MkdirAll(s.opts.Dir, 0o750); err != nil {
s.fileErr = err
return err
}
name := "requests-" + time.Now().Format("20060102-150405") + ".jsonl"
f, err := os.OpenFile(filepath.Join(s.opts.Dir, name), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
if err != nil {
s.fileErr = err
return err
}
st, _ := f.Stat()
s.file = f
s.fileName = name
if st != nil {
s.fileSize = st.Size()
} else {
s.fileSize = 0
}
s.fileErr = nil
return nil
}
func (s *Store) writeFileLocked(e Entry) {
if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" {
return
}
if err := s.ensureFileLocked(); err != nil {
return
}
if s.file == nil {
return
}
if s.fileSize >= s.opts.MaxFileBytes {
if err := s.rotateLocked(); err != nil {
s.fileErr = err
return
}
s.trimFilesLocked()
}
b, err := json.Marshal(e)
if err != nil {
return
}
b = append(b, '\n')
n, err := s.file.Write(b)
s.fileSize += int64(n)
if err != nil {
s.fileErr = err
return
}
s.fileErr = nil
}
func (s *Store) rotateLocked() error {
if s.file != nil {
if err := s.file.Close(); err != nil {
return err
}
s.file = nil
}
return s.ensureFileLocked()
}
func (s *Store) trimFilesLocked() {
files, err := s.listFilesLocked()
if err != nil || len(files) <= s.opts.MaxFiles {
return
}
for _, name := range files[s.opts.MaxFiles:] {
if name == s.fileName {
continue
}
_ = os.Remove(filepath.Join(s.opts.Dir, name))
}
}
func (s *Store) listFilesLocked() ([]string, error) {
entries, err := os.ReadDir(s.opts.Dir)
if err != nil {
return nil, err
}
names := make([]string, 0, len(entries))
for _, de := range entries {
if de.IsDir() || !strings.HasSuffix(de.Name(), ".jsonl") {
continue
}
names = append(names, de.Name())
}
sort.Sort(sort.Reverse(sort.StringSlice(names)))
return names, nil
}
// Files 返回轮转文件(新到旧)。
func (s *Store) Files() []map[string]any {
s.mu.RLock()
defer s.mu.RUnlock()
if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" {
return nil
}
names, err := s.listFilesLocked()
if err != nil {
return nil
}
out := make([]map[string]any, 0, len(names))
for _, name := range names {
info, statErr := os.Stat(filepath.Join(s.opts.Dir, name))
if statErr != nil {
continue
}
out = append(out, map[string]any{
"name": name,
"size": info.Size(),
"modified_at": info.ModTime().Format(time.RFC3339),
})
}
return out
}
var ErrInvalidFileName = errors.New("invalid log file name")
// ResolveFile 返回安全的绝对路径,拒绝路径穿越。
func (s *Store) ResolveFile(name string) (string, error) {
name = strings.TrimSpace(name)
if name == "" || name != filepath.Base(name) || strings.Contains(name, "..") {
return "", ErrInvalidFileName
}
if !strings.HasSuffix(name, ".jsonl") {
return "", ErrInvalidFileName
}
path := filepath.Join(s.opts.Dir, name)
if st, err := os.Stat(path); err != nil || st.IsDir() {
return "", os.ErrNotExist
}
return path, nil
}
// Tail 读取单个文件最后 limit 行(limit<=0 表示全部)。
func (s *Store) Tail(name string, limit int) ([]Entry, error) {
path, err := s.ResolveFile(name)
if err != nil {
return nil, err
}
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 64*1024), 1<<20)
entries := make([]Entry, 0, 256)
for scanner.Scan() {
var e Entry
if err := json.Unmarshal(scanner.Bytes(), &e); err != nil {
continue
}
entries = append(entries, e)
}
if err := scanner.Err(); err != nil {
return nil, err
}
if limit > 0 && len(entries) > limit {
entries = entries[len(entries)-limit:]
}
return entries, nil
}
func (s *Store) listFiles() ([]string, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.listFilesLocked()
}
// WriteZip 把指定文件(为空表示全部)打包输出。
func (s *Store) WriteZip(w io.Writer, names []string) error {
if len(names) == 0 {
files, err := s.listFiles()
if err != nil {
return err
}
names = files
}
zw := zip.NewWriter(w)
for _, name := range names {
path, err := s.ResolveFile(name)
if err != nil {
continue
}
src, err := os.Open(path)
if err != nil {
continue
}
info, statErr := src.Stat()
if statErr != nil {
_ = src.Close()
continue
}
header, err := zip.FileInfoHeader(info)
if err != nil {
_ = src.Close()
continue
}
header.Name = name
header.Method = zip.Deflate
dst, err := zw.CreateHeader(header)
if err != nil {
_ = src.Close()
return err
}
if _, err := io.Copy(dst, src); err != nil {
_ = src.Close()
return err
}
_ = src.Close()
}
return zw.Close()
}
// Close 关闭当前文件句柄。
func (s *Store) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil {
return nil
}
err := s.file.Close()
s.file = nil
return err
}
const (
LevelInfo = "info"
LevelWarn = "warn"
LevelError = "error"
)