Download internal/reqlog/store.go from a3216/ds2api: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/spaces/a3216/ds2api/resolve/main/internal/reqlog/store.go
- Command line
-
hf download hf://spaces/a3216/ds2api/internal/reqlog/store.go
-
curl -L -o store.go https://huggingface.co/spaces/a3216/ds2api/resolve/main/internal/reqlog/store.go
10.2 kB
| package reqlog | |
| import ( | |
| "archive/zip" | |
| "bufio" | |
| "encoding/json" | |
| "errors" | |
| "fmt" | |
| "io" | |
| "os" | |
| "path/filepath" | |
| "sort" | |
| "strings" | |
| "sync" | |
| "time" | |
| ) | |
| // Entry 是一条请求日志。只记录元数据,不记录请求/响应正文,避免泄露用户内容。 | |
| type Entry struct { | |
| ID string `json:"id"` | |
| Time string `json:"time"` | |
| Level string `json:"level"` | |
| Method string `json:"method"` | |
| Path string `json:"path"` | |
| Status int `json:"status"` | |
| LatencyMS int64 `json:"latency_ms"` | |
| ClientIP string `json:"client_ip,omitempty"` | |
| UserAgent string `json:"user_agent,omitempty"` | |
| APIKey string `json:"api_key,omitempty"` | |
| APIKeyName string `json:"api_key_name,omitempty"` | |
| Account string `json:"account,omitempty"` | |
| Model string `json:"model,omitempty"` | |
| Stream bool `json:"stream,omitempty"` | |
| ReqBytes int64 `json:"req_bytes,omitempty"` | |
| RespBytes int64 `json:"resp_bytes,omitempty"` | |
| ErrorCode string `json:"error_code,omitempty"` | |
| Error string `json:"error,omitempty"` | |
| Upstream string `json:"upstream,omitempty"` | |
| Dropped bool `json:"-"` | |
| } | |
| // Options 控制内存窗口与文件轮转。 | |
| type Options struct { | |
| Dir string | |
| MemoryLimit int // 内存保留条数 | |
| MaxFileBytes int64 // 单个文件大小上限 | |
| MaxFiles int // 保留文件数 | |
| DisableFile bool | |
| } | |
| func DefaultOptions(dir string) Options { | |
| return Options{ | |
| Dir: dir, | |
| MemoryLimit: 2000, | |
| MaxFileBytes: 8 << 20, | |
| MaxFiles: 5, | |
| } | |
| } | |
| // Store 同时提供内存环形窗口与按大小轮转的文件日志。 | |
| type Store struct { | |
| mu sync.RWMutex | |
| opts Options | |
| entries []Entry | |
| next int | |
| full bool | |
| seq uint64 | |
| file *os.File | |
| fileSize int64 | |
| fileName string | |
| fileErr error | |
| } | |
| func New(opts Options) *Store { | |
| if opts.MemoryLimit <= 0 { | |
| opts.MemoryLimit = 2000 | |
| } | |
| if opts.MaxFileBytes <= 0 { | |
| opts.MaxFileBytes = 8 << 20 | |
| } | |
| if opts.MaxFiles <= 0 { | |
| opts.MaxFiles = 5 | |
| } | |
| s := &Store{opts: opts, entries: make([]Entry, opts.MemoryLimit)} | |
| if !opts.DisableFile && strings.TrimSpace(opts.Dir) != "" { | |
| if err := os.MkdirAll(opts.Dir, 0o750); err != nil { | |
| s.fileErr = err | |
| } | |
| } | |
| return s | |
| } | |
| func (s *Store) Options() Options { return s.opts } | |
| func (s *Store) Add(e Entry) { | |
| if e.Time == "" { | |
| e.Time = time.Now().Format("2006-01-02T15:04:05.000Z07:00") | |
| } | |
| s.mu.Lock() | |
| s.seq++ | |
| if e.ID == "" { | |
| e.ID = fmt.Sprintf("%d-%06d", time.Now().UnixMilli(), s.seq%1000000) | |
| } | |
| s.entries[s.next] = e | |
| s.next = (s.next + 1) % len(s.entries) | |
| if s.next == 0 { | |
| s.full = true | |
| } | |
| s.writeFileLocked(e) | |
| s.mu.Unlock() | |
| } | |
| func (s *Store) entriesLocked() []Entry { | |
| if !s.full { | |
| return append([]Entry(nil), s.entries[:s.next]...) | |
| } | |
| out := make([]Entry, 0, len(s.entries)) | |
| out = append(out, s.entries[s.next:]...) | |
| out = append(out, s.entries[:s.next]...) | |
| return out | |
| } | |
| // Query 过滤内存中的日志,按时间倒序返回。 | |
| type Query struct { | |
| Limit int | |
| Level string | |
| Status int | |
| Keyword string | |
| SinceUnix int64 | |
| UntilUnix int64 | |
| } | |
| func (s *Store) Query(q Query) []Entry { | |
| s.mu.RLock() | |
| all := s.entriesLocked() | |
| s.mu.RUnlock() | |
| limit := q.Limit | |
| if limit <= 0 || limit > len(all) { | |
| limit = len(all) | |
| } | |
| keyword := strings.ToLower(strings.TrimSpace(q.Keyword)) | |
| out := make([]Entry, 0, limit) | |
| for i := len(all) - 1; i >= 0 && len(out) < limit; i-- { | |
| e := all[i] | |
| if q.Level != "" && !strings.EqualFold(e.Level, q.Level) { | |
| continue | |
| } | |
| if q.Status != 0 && e.Status != q.Status { | |
| continue | |
| } | |
| if q.SinceUnix > 0 && entryUnix(e) < q.SinceUnix { | |
| continue | |
| } | |
| if q.UntilUnix > 0 && entryUnix(e) > q.UntilUnix { | |
| continue | |
| } | |
| if keyword != "" && !entryMatches(e, keyword) { | |
| continue | |
| } | |
| out = append(out, e) | |
| } | |
| return out | |
| } | |
| func entryUnix(e Entry) int64 { | |
| t, err := time.Parse(time.RFC3339Nano, e.Time) | |
| if err != nil { | |
| return 0 | |
| } | |
| return t.Unix() | |
| } | |
| func entryMatches(e Entry, keyword string) bool { | |
| fields := []string{e.Path, e.Account, e.Model, e.ErrorCode, e.Error, e.Upstream, e.APIKeyName, e.APIKey, e.ClientIP} | |
| for _, f := range fields { | |
| if f != "" && strings.Contains(strings.ToLower(f), keyword) { | |
| return true | |
| } | |
| } | |
| return strings.Contains(strings.ToLower(e.Method), keyword) || strings.Contains(strings.ToLower(e.UserAgent), keyword) | |
| } | |
| // Stats 返回内存窗口的聚合信息。 | |
| func (s *Store) Stats() map[string]any { | |
| s.mu.RLock() | |
| all := s.entriesLocked() | |
| fileErr := s.fileErr | |
| s.mu.RUnlock() | |
| total := len(all) | |
| errors := 0 | |
| warns := 0 | |
| lastHour := 0 | |
| cutoff := time.Now().Add(-time.Hour).Unix() | |
| for _, e := range all { | |
| if e.Status >= 500 || e.Level == LevelError { | |
| errors++ | |
| } | |
| if e.Status >= 400 && e.Status < 500 { | |
| warns++ | |
| } | |
| if entryUnix(e) >= cutoff { | |
| lastHour++ | |
| } | |
| } | |
| out := map[string]any{ | |
| "total": total, | |
| "errors": errors, | |
| "client_errors": warns, | |
| "last_hour": lastHour, | |
| "memory_limit": len(s.entries), | |
| } | |
| if fileErr != nil { | |
| out["file_error"] = fileErr.Error() | |
| } | |
| return out | |
| } | |
| func (s *Store) ensureFileLocked() error { | |
| if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" { | |
| return nil | |
| } | |
| if s.file != nil { | |
| return nil | |
| } | |
| if err := os.MkdirAll(s.opts.Dir, 0o750); err != nil { | |
| s.fileErr = err | |
| return err | |
| } | |
| name := "requests-" + time.Now().Format("20060102-150405") + ".jsonl" | |
| f, err := os.OpenFile(filepath.Join(s.opts.Dir, name), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640) | |
| if err != nil { | |
| s.fileErr = err | |
| return err | |
| } | |
| st, _ := f.Stat() | |
| s.file = f | |
| s.fileName = name | |
| if st != nil { | |
| s.fileSize = st.Size() | |
| } else { | |
| s.fileSize = 0 | |
| } | |
| s.fileErr = nil | |
| return nil | |
| } | |
| func (s *Store) writeFileLocked(e Entry) { | |
| if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" { | |
| return | |
| } | |
| if err := s.ensureFileLocked(); err != nil { | |
| return | |
| } | |
| if s.file == nil { | |
| return | |
| } | |
| if s.fileSize >= s.opts.MaxFileBytes { | |
| if err := s.rotateLocked(); err != nil { | |
| s.fileErr = err | |
| return | |
| } | |
| s.trimFilesLocked() | |
| } | |
| b, err := json.Marshal(e) | |
| if err != nil { | |
| return | |
| } | |
| b = append(b, '\n') | |
| n, err := s.file.Write(b) | |
| s.fileSize += int64(n) | |
| if err != nil { | |
| s.fileErr = err | |
| return | |
| } | |
| s.fileErr = nil | |
| } | |
| func (s *Store) rotateLocked() error { | |
| if s.file != nil { | |
| if err := s.file.Close(); err != nil { | |
| return err | |
| } | |
| s.file = nil | |
| } | |
| return s.ensureFileLocked() | |
| } | |
| func (s *Store) trimFilesLocked() { | |
| files, err := s.listFilesLocked() | |
| if err != nil || len(files) <= s.opts.MaxFiles { | |
| return | |
| } | |
| for _, name := range files[s.opts.MaxFiles:] { | |
| if name == s.fileName { | |
| continue | |
| } | |
| _ = os.Remove(filepath.Join(s.opts.Dir, name)) | |
| } | |
| } | |
| func (s *Store) listFilesLocked() ([]string, error) { | |
| entries, err := os.ReadDir(s.opts.Dir) | |
| if err != nil { | |
| return nil, err | |
| } | |
| names := make([]string, 0, len(entries)) | |
| for _, de := range entries { | |
| if de.IsDir() || !strings.HasSuffix(de.Name(), ".jsonl") { | |
| continue | |
| } | |
| names = append(names, de.Name()) | |
| } | |
| sort.Sort(sort.Reverse(sort.StringSlice(names))) | |
| return names, nil | |
| } | |
| // Files 返回轮转文件(新到旧)。 | |
| func (s *Store) Files() []map[string]any { | |
| s.mu.RLock() | |
| defer s.mu.RUnlock() | |
| if s.opts.DisableFile || strings.TrimSpace(s.opts.Dir) == "" { | |
| return nil | |
| } | |
| names, err := s.listFilesLocked() | |
| if err != nil { | |
| return nil | |
| } | |
| out := make([]map[string]any, 0, len(names)) | |
| for _, name := range names { | |
| info, statErr := os.Stat(filepath.Join(s.opts.Dir, name)) | |
| if statErr != nil { | |
| continue | |
| } | |
| out = append(out, map[string]any{ | |
| "name": name, | |
| "size": info.Size(), | |
| "modified_at": info.ModTime().Format(time.RFC3339), | |
| }) | |
| } | |
| return out | |
| } | |
| var ErrInvalidFileName = errors.New("invalid log file name") | |
| // ResolveFile 返回安全的绝对路径,拒绝路径穿越。 | |
| func (s *Store) ResolveFile(name string) (string, error) { | |
| name = strings.TrimSpace(name) | |
| if name == "" || name != filepath.Base(name) || strings.Contains(name, "..") { | |
| return "", ErrInvalidFileName | |
| } | |
| if !strings.HasSuffix(name, ".jsonl") { | |
| return "", ErrInvalidFileName | |
| } | |
| path := filepath.Join(s.opts.Dir, name) | |
| if st, err := os.Stat(path); err != nil || st.IsDir() { | |
| return "", os.ErrNotExist | |
| } | |
| return path, nil | |
| } | |
| // Tail 读取单个文件最后 limit 行(limit<=0 表示全部)。 | |
| func (s *Store) Tail(name string, limit int) ([]Entry, error) { | |
| path, err := s.ResolveFile(name) | |
| if err != nil { | |
| return nil, err | |
| } | |
| f, err := os.Open(path) | |
| if err != nil { | |
| return nil, err | |
| } | |
| defer func() { _ = f.Close() }() | |
| scanner := bufio.NewScanner(f) | |
| scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) | |
| entries := make([]Entry, 0, 256) | |
| for scanner.Scan() { | |
| var e Entry | |
| if err := json.Unmarshal(scanner.Bytes(), &e); err != nil { | |
| continue | |
| } | |
| entries = append(entries, e) | |
| } | |
| if err := scanner.Err(); err != nil { | |
| return nil, err | |
| } | |
| if limit > 0 && len(entries) > limit { | |
| entries = entries[len(entries)-limit:] | |
| } | |
| return entries, nil | |
| } | |
| func (s *Store) listFiles() ([]string, error) { | |
| s.mu.RLock() | |
| defer s.mu.RUnlock() | |
| return s.listFilesLocked() | |
| } | |
| // WriteZip 把指定文件(为空表示全部)打包输出。 | |
| func (s *Store) WriteZip(w io.Writer, names []string) error { | |
| if len(names) == 0 { | |
| files, err := s.listFiles() | |
| if err != nil { | |
| return err | |
| } | |
| names = files | |
| } | |
| zw := zip.NewWriter(w) | |
| for _, name := range names { | |
| path, err := s.ResolveFile(name) | |
| if err != nil { | |
| continue | |
| } | |
| src, err := os.Open(path) | |
| if err != nil { | |
| continue | |
| } | |
| info, statErr := src.Stat() | |
| if statErr != nil { | |
| _ = src.Close() | |
| continue | |
| } | |
| header, err := zip.FileInfoHeader(info) | |
| if err != nil { | |
| _ = src.Close() | |
| continue | |
| } | |
| header.Name = name | |
| header.Method = zip.Deflate | |
| dst, err := zw.CreateHeader(header) | |
| if err != nil { | |
| _ = src.Close() | |
| return err | |
| } | |
| if _, err := io.Copy(dst, src); err != nil { | |
| _ = src.Close() | |
| return err | |
| } | |
| _ = src.Close() | |
| } | |
| return zw.Close() | |
| } | |
| // Close 关闭当前文件句柄。 | |
| func (s *Store) Close() error { | |
| s.mu.Lock() | |
| defer s.mu.Unlock() | |
| if s.file == nil { | |
| return nil | |
| } | |
| err := s.file.Close() | |
| s.file = nil | |
| return err | |
| } | |
| const ( | |
| LevelInfo = "info" | |
| LevelWarn = "warn" | |
| LevelError = "error" | |
| ) | |