|
Download plans/config-refactoring.md from shimen/API: direct link, hf CLI and curl.
- Browser
- Download file 17.3 kB
-
https://huggingface.co/spaces/shimen/API/resolve/main/plans/config-refactoring.md
- Command line
-
hf download hf://spaces/shimen/API/plans/config-refactoring.md
-
curl -L -o config-refactoring.md https://huggingface.co/spaces/shimen/API/resolve/main/plans/config-refactoring.md
17.3 kB
| # Configuration Management Refactoring Plan | |
| ## Current State Analysis | |
| **File**: `internal/config/config.go` (1,767 lines) | |
| **Issues**: | |
| - Single file with all config structures | |
| - Mix of loading, validation, and migration logic | |
| - Complex nested structs with 326+ lines of YAML example | |
| - No clear separation between config sources (file, env, DB) | |
| - Migration logic scattered throughout | |
| - No structured validation (manual checks) | |
| ## Target Architecture | |
| ``` | |
| internal/config/ | |
| βββ domain/ | |
| β βββ models.go # Core config structs (no logic) | |
| β βββ validation.go # Validation rules | |
| βββ loader/ | |
| β βββ file.go # YAML/JSON file loading | |
| β βββ env.go # Environment variable loading | |
| β βββ merge.go # Config merging logic | |
| βββ provider/ | |
| β βββ interface.go # ConfigProvider interface | |
| β βββ file.go # File-based provider | |
| β βββ postgres.go # PostgreSQL provider | |
| β βββ git.go # Git-based provider | |
| β βββ objectstore.go # S3/MinIO provider | |
| βββ migrations/ | |
| β βββ registry.go # Migration registry | |
| β βββ v1_to_v2.go # Specific migrations | |
| β βββ runner.go # Migration runner | |
| βββ watcher/ | |
| β βββ file.go # File watching for hot reload | |
| βββ config.go # Public API (facade) | |
| ``` | |
| ## Phase 1: Domain Models Extraction (Week 1) | |
| ### 1.1 Core Models | |
| **File**: `internal/config/domain/models.go` | |
| ```go | |
| package domain | |
| import ( | |
| "time" | |
| "github.com/echyai/cliproxyapi/internal/domain/auth" | |
| ) | |
| // Config is the root configuration | |
| type Config struct { | |
| Version string `yaml:"version" json:"version" validate:"required,semver"` | |
| Server Server `yaml:"server" json:"server" validate:"required"` | |
| Providers Providers `yaml:"providers" json:"providers"` | |
| Storage Storage `yaml:"storage" json:"storage" validate:"required"` | |
| Logging Logging `yaml:"logging" json:"logging"` | |
| Metrics Metrics `yaml:"metrics" json:"metrics"` | |
| } | |
| func (c *Config) Validate() error { | |
| validate := validator.New() | |
| validate.RegisterValidation("semver", validateSemver) | |
| return validate.Struct(c) | |
| } | |
| // Server configuration | |
| type Server struct { | |
| Host string `yaml:"host" json:"host" validate:"hostname|ip"` | |
| Port int `yaml:"port" json:"port" validate:"required,min=1,max=65535"` | |
| ReadTimeout time.Duration `yaml:"read_timeout" json:"read_timeout" validate:"min=1s"` | |
| WriteTimeout time.Duration `yaml:"write_timeout" json:"write_timeout" validate:"min=1s"` | |
| MaxHeaderBytes int `yaml:"max_header_bytes" json:"max_header_bytes" validate:"min=1024"` | |
| TLS *TLSConfig `yaml:"tls,omitempty" json:"tls,omitempty"` | |
| } | |
| // TLS configuration | |
| type TLSConfig struct { | |
| CertFile string `yaml:"cert_file" json:"cert_file" validate:"required,file"` | |
| KeyFile string `yaml:"key_file" json:"key_file" validate:"required,file"` | |
| } | |
| // Providers configuration | |
| type Providers struct { | |
| Gemini *GeminiConfig `yaml:"gemini,omitempty" json:"gemini,omitempty"` | |
| Claude *ClaudeConfig `yaml:"claude,omitempty" json:"claude,omitempty"` | |
| Codex *CodexConfig `yaml:"codex,omitempty" json:"codex,omitempty"` | |
| Antigravity *AntigravityConfig `yaml:"antigravity,omitempty" json:"antigravity,omitempty"` | |
| Qwen *QwenConfig `yaml:"qwen,omitempty" json:"qwen,omitempty"` | |
| Vertex *VertexConfig `yaml:"vertex,omitempty" json:"vertex,omitempty"` | |
| } | |
| // Gemini configuration | |
| type GeminiConfig struct { | |
| Enabled bool `yaml:"enabled" json:"enabled"` | |
| APIKey string `yaml:"api_key,omitempty" json:"api_key,omitempty" validate:"omitempty,min=10"` | |
| OAuth *OAuthConfig `yaml:"oauth,omitempty" json:"oauth,omitempty"` | |
| RateLimit RateLimitConfig `yaml:"rate_limit" json:"rate_limit"` | |
| Models []ModelConfig `yaml:"models" json:"models" validate:"dive"` | |
| } | |
| // OAuth configuration (shared) | |
| type OAuthConfig struct { | |
| ClientID string `yaml:"client_id" validate:"required"` | |
| ClientSecret string `yaml:"client_secret" validate:"required"` | |
| RedirectURL string `yaml:"redirect_url" validate:"required,url"` | |
| Scopes []string `yaml:"scopes" validate:"min=1"` | |
| } | |
| // Rate limit configuration | |
| type RateLimitConfig struct { | |
| RequestsPerSecond float64 `yaml:"requests_per_second" validate:"min=0.1"` | |
| Burst int `yaml:"burst" validate:"min=1"` | |
| Cooldown time.Duration `yaml:"cooldown" validate:"min=0"` | |
| } | |
| // Model configuration | |
| type ModelConfig struct { | |
| Name string `yaml:"name" validate:"required"` | |
| DisplayName string `yaml:"display_name"` | |
| MaxTokens int `yaml:"max_tokens" validate:"min=1"` | |
| Enabled bool `yaml:"enabled"` | |
| } | |
| // Storage configuration | |
| type Storage struct { | |
| Type StorageType `yaml:"type" validate:"required,oneof=file postgres git objectstore"` | |
| File *FileStorage `yaml:"file,omitempty"` | |
| Postgres *PostgresStorage `yaml:"postgres,omitempty"` | |
| Git *GitStorage `yaml:"git,omitempty"` | |
| Object *ObjectStorage `yaml:"object,omitempty"` | |
| } | |
| type StorageType string | |
| const ( | |
| StorageTypeFile StorageType = "file" | |
| StorageTypePostgres StorageType = "postgres" | |
| StorageTypeGit StorageType = "git" | |
| StorageTypeObject StorageType = "objectstore" | |
| ) | |
| // FileStorage configuration | |
| type FileStorage struct { | |
| Path string `yaml:"path" validate:"required,dirpath"` | |
| } | |
| // PostgresStorage configuration | |
| type PostgresStorage struct { | |
| Host string `yaml:"host" validate:"required"` | |
| Port int `yaml:"port" validate:"min=1,max=65535"` | |
| Database string `yaml:"database" validate:"required"` | |
| Username string `yaml:"username" validate:"required"` | |
| Password string `yaml:"password" validate:"required"` | |
| SSLMode string `yaml:"ssl_mode" validate:"oneof=disable require verify-ca verify-full"` | |
| } | |
| // Logging configuration | |
| type Logging struct { | |
| Level string `yaml:"level" validate:"oneof=debug info warn error fatal panic"` | |
| Format string `yaml:"format" validate:"oneof=json text"` | |
| Output string `yaml:"output" validate:"oneof=stdout stderr file"` | |
| File string `yaml:"file,omitempty"` | |
| } | |
| // Metrics configuration | |
| type Metrics struct { | |
| Enabled bool `yaml:"enabled"` | |
| Path string `yaml:"path" validate:"startswith=/"` | |
| Port int `yaml:"port" validate:"min=1,max=65535"` | |
| } | |
| // Provider-specific configs (Claude, Codex, etc. follow similar pattern) | |
| type ClaudeConfig struct { | |
| Enabled bool `yaml:"enabled"` | |
| APIKey string `yaml:"api_key,omitempty"` | |
| OAuth *OAuthConfig `yaml:"oauth,omitempty"` | |
| RateLimit RateLimitConfig `yaml:"rate_limit"` | |
| } | |
| // ... other provider configs | |
| ``` | |
| ### 1.2 Validation Rules | |
| **File**: `internal/config/domain/validation.go` | |
| ```go | |
| package domain | |
| import ( | |
| "github.com/go-playground/validator/v10" | |
| "regexp" | |
| ) | |
| var semverRegex = regexp.MustCompile(`^v?(\d+)\.(\d+)\.(\d+)(?:-([\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*))?(?:\+([\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*))?$`) | |
| func validateSemver(fl validator.FieldLevel) bool { | |
| return semverRegex.MatchString(fl.Field().String()) | |
| } | |
| // ConfigValidator handles validation with custom rules | |
| type ConfigValidator struct { | |
| validate *validator.Validate | |
| } | |
| func NewValidator() *ConfigValidator { | |
| v := validator.New() | |
| v.RegisterValidation("semver", validateSemver) | |
| return &ConfigValidator{validate: v} | |
| } | |
| func (cv *ConfigValidator) Validate(cfg *Config) error { | |
| if err := cv.validate.Struct(cfg); err != nil { | |
| return NewValidationError(err) | |
| } | |
| // Cross-field validations | |
| if err := cv.validateStorage(cfg.Storage); err != nil { | |
| return err | |
| } | |
| if err := cv.validateProviders(cfg.Providers); err != nil { | |
| return err | |
| } | |
| return nil | |
| } | |
| func (cv *ConfigValidator) validateStorage(s Storage) error { | |
| switch s.Type { | |
| case StorageTypeFile: | |
| if s.File == nil { | |
| return ValidationError{Field: "storage.file", Message: "required when type is file"} | |
| } | |
| case StorageTypePostgres: | |
| if s.Postgres == nil { | |
| return ValidationError{Field: "storage.postgres", Message: "required when type is postgres"} | |
| } | |
| // ... other cases | |
| } | |
| return nil | |
| } | |
| func (cv *ConfigValidator) validateProviders(p Providers) error { | |
| // At least one provider should be enabled | |
| hasEnabled := false | |
| // Check each provider... | |
| if !hasEnabled { | |
| return ValidationError{Message: "at least one provider must be enabled"} | |
| } | |
| return nil | |
| } | |
| ``` | |
| ## Phase 2: Provider Interface (Week 2) | |
| ### 2.1 Provider Interface | |
| **File**: `internal/config/provider/interface.go` | |
| ```go | |
| package provider | |
| import ( | |
| "context" | |
| "github.com/echyai/cliproxyapi/internal/config/domain" | |
| ) | |
| // Provider loads and saves configuration | |
| type Provider interface { | |
| // Load retrieves the current configuration | |
| Load(ctx context.Context) (*domain.Config, error) | |
| // Save persists the configuration | |
| Save(ctx context.Context, cfg *domain.Config) error | |
| // Watch returns a channel that receives config updates | |
| Watch(ctx context.Context) (<-chan domain.Config, error) | |
| // Close cleans up resources | |
| Close() error | |
| } | |
| // ProviderFactory creates providers based on type | |
| type ProviderFactory struct { | |
| // dependencies | |
| } | |
| func (f *ProviderFactory) Create(storage domain.Storage) (Provider, error) { | |
| switch storage.Type { | |
| case domain.StorageTypeFile: | |
| return NewFileProvider(storage.File) | |
| case domain.StorageTypePostgres: | |
| return NewPostgresProvider(storage.Postgres) | |
| case domain.StorageTypeGit: | |
| return NewGitProvider(storage.Git) | |
| case domain.StorageTypeObject: | |
| return NewObjectProvider(storage.Object) | |
| default: | |
| return nil, fmt.Errorf("unknown storage type: %s", storage.Type) | |
| } | |
| } | |
| ``` | |
| ### 2.2 File Provider | |
| **File**: `internal/config/provider/file.go` | |
| ```go | |
| package provider | |
| import ( | |
| "context" | |
| "fmt" | |
| "os" | |
| "path/filepath" | |
| "github.com/fsnotify/fsnotify" | |
| "gopkg.in/yaml.v3" | |
| "github.com/echyai/cliproxyapi/internal/config/domain" | |
| ) | |
| type FileProvider struct { | |
| path string | |
| watcher *fsnotify.Watcher | |
| validate *domain.ConfigValidator | |
| } | |
| func NewFileProvider(cfg *domain.FileStorage) (*FileProvider, error) { | |
| absPath, err := filepath.Abs(cfg.Path) | |
| if err != nil { | |
| return nil, fmt.Errorf("invalid path: %w", err) | |
| } | |
| return &FileProvider{ | |
| path: absPath, | |
| validate: domain.NewValidator(), | |
| }, nil | |
| } | |
| func (p *FileProvider) Load(ctx context.Context) (*domain.Config, error) { | |
| data, err := os.ReadFile(p.path) | |
| if err != nil { | |
| if os.IsNotExist(err) { | |
| return nil, domain.ErrConfigNotFound | |
| } | |
| return nil, fmt.Errorf("read config: %w", err) | |
| } | |
| var cfg domain.Config | |
| if err := yaml.Unmarshal(data, &cfg); err != nil { | |
| return nil, fmt.Errorf("parse config: %w", err) | |
| } | |
| if err := p.validate.Validate(&cfg); err != nil { | |
| return nil, fmt.Errorf("validate config: %w", err) | |
| } | |
| return &cfg, nil | |
| } | |
| func (p *FileProvider) Save(ctx context.Context, cfg *domain.Config) error { | |
| if err := p.validate.Validate(cfg); err != nil { | |
| return err | |
| } | |
| data, err := yaml.Marshal(cfg) | |
| if err != nil { | |
| return fmt.Errorf("marshal config: %w", err) | |
| } | |
| // Write atomically | |
| tmpPath := p.path + ".tmp" | |
| if err := os.WriteFile(tmpPath, data, 0644); err != nil { | |
| return fmt.Errorf("write temp file: %w", err) | |
| } | |
| return os.Rename(tmpPath, p.path) | |
| } | |
| func (p *FileProvider) Watch(ctx context.Context) (<-chan domain.Config, error) { | |
| watcher, err := fsnotify.NewWatcher() | |
| if err != nil { | |
| return nil, err | |
| } | |
| if err := watcher.Add(p.path); err != nil { | |
| return nil, err | |
| } | |
| p.watcher = watcher | |
| updates := make(chan domain.Config) | |
| go func() { | |
| defer close(updates) | |
| defer watcher.Close() | |
| for { | |
| select { | |
| case event, ok := <-watcher.Events: | |
| if !ok { | |
| return | |
| } | |
| if event.Op&fsnotify.Write == fsnotify.Write { | |
| cfg, err := p.Load(ctx) | |
| if err == nil { | |
| updates <- *cfg | |
| } | |
| } | |
| case err, ok := <-watcher.Errors: | |
| if !ok { | |
| return | |
| } | |
| // Log error | |
| _ = err | |
| case <-ctx.Done(): | |
| return | |
| } | |
| } | |
| }() | |
| return updates, nil | |
| } | |
| func (p *FileProvider) Close() error { | |
| if p.watcher != nil { | |
| return p.watcher.Close() | |
| } | |
| return nil | |
| } | |
| ``` | |
| ## Phase 3: Migration System (Week 2) | |
| ### 3.1 Migration Framework | |
| **File**: `internal/config/migrations/registry.go` | |
| ```go | |
| package migrations | |
| import ( | |
| "fmt" | |
| "github.com/echyai/cliproxyapi/internal/config/domain" | |
| ) | |
| // Migration transforms config from one version to another | |
| type Migration interface { | |
| FromVersion() string | |
| ToVersion() string | |
| Migrate(cfg map[string]interface{}) (map[string]interface{}, error) | |
| } | |
| // Registry holds all migrations | |
| type Registry struct { | |
| migrations map[string]Migration | |
| } | |
| func NewRegistry() *Registry { | |
| r := &Registry{migrations: make(map[string]Migration)} | |
| r.registerDefaults() | |
| return r | |
| } | |
| func (r *Registry) register(m Migration) { | |
| key := fmt.Sprintf("%s->%s", m.FromVersion(), m.ToVersion()) | |
| r.migrations[key] = m | |
| } | |
| func (r *Registry) registerDefaults() { | |
| r.register(&V1ToV2Migration{}) | |
| r.register(&V2ToV3Migration{}) | |
| // ... more migrations | |
| } | |
| // Runner executes migrations | |
| type Runner struct { | |
| registry *Registry | |
| } | |
| func (r *Runner) Migrate(cfg map[string]interface{}, targetVersion string) (map[string]interface{}, error) { | |
| currentVersion, _ := cfg["version"].(string) | |
| if currentVersion == "" { | |
| currentVersion = "1.0.0" | |
| } | |
| for currentVersion != targetVersion { | |
| key := fmt.Sprintf("%s->%s", currentVersion, targetVersion) | |
| migration, ok := r.registry.migrations[key] | |
| if !ok { | |
| // Try to find intermediate step | |
| nextVersion := findNextVersion(currentVersion) | |
| if nextVersion == "" { | |
| return nil, fmt.Errorf("no migration path from %s to %s", currentVersion, targetVersion) | |
| } | |
| key = fmt.Sprintf("%s->%s", currentVersion, nextVersion) | |
| migration = r.registry.migrations[key] | |
| } | |
| var err error | |
| cfg, err = migration.Migrate(cfg) | |
| if err != nil { | |
| return nil, fmt.Errorf("migrate %s: %w", key, err) | |
| } | |
| currentVersion = migration.ToVersion() | |
| } | |
| return cfg, nil | |
| } | |
| ``` | |
| ## Phase 4: Public API Facade (Week 3) | |
| **File**: `internal/config/config.go` (simplified facade) | |
| ```go | |
| package config | |
| import ( | |
| "context" | |
| "sync" | |
| "github.com/echyai/cliproxyapi/internal/config/domain" | |
| "github.com/echyai/cliproxyapi/internal/config/migrations" | |
| "github.com/echyai/cliproxyapi/internal/config/provider" | |
| ) | |
| // Manager is the public API for configuration | |
| type Manager struct { | |
| provider provider.Provider | |
| current *domain.Config | |
| mu sync.RWMutex | |
| validate *domain.ConfigValidator | |
| } | |
| func NewManager(storage domain.Storage) (*Manager, error) { | |
| p, err := provider.NewFactory().Create(storage) | |
| if err != nil { | |
| return nil, err | |
| } | |
| return &Manager{ | |
| provider: p, | |
| validate: domain.NewValidator(), | |
| }, nil | |
| } | |
| func (m *Manager) Load(ctx context.Context) error { | |
| cfg, err := m.provider.Load(ctx) | |
| if err != nil { | |
| return err | |
| } | |
| m.mu.Lock() | |
| m.current = cfg | |
| m.mu.Unlock() | |
| return nil | |
| } | |
| func (m *Manager) Get() *domain.Config { | |
| m.mu.RLock() | |
| defer m.mu.RUnlock() | |
| return m.current | |
| } | |
| func (m *Manager) Update(ctx context.Context, updater func(*domain.Config) error) error { | |
| m.mu.Lock() | |
| defer m.mu.Unlock() | |
| cfg := *m.current // Copy | |
| if err := updater(&cfg); err != nil { | |
| return err | |
| } | |
| if err := m.validate.Validate(&cfg); err != nil { | |
| return err | |
| } | |
| if err := m.provider.Save(ctx, &cfg); err != nil { | |
| return err | |
| } | |
| m.current = &cfg | |
| return nil | |
| } | |
| func (m *Manager) Watch(ctx context.Context) (<-chan domain.Config, error) { | |
| return m.provider.Watch(ctx) | |
| } | |
| func (m *Manager) Close() error { | |
| return m.provider.Close() | |
| } | |
| ``` | |
| ## Migration Steps | |
| 1. **Week 1**: Create new structure alongside existing code | |
| 2. **Week 2**: Implement providers and migrations | |
| 3. **Week 3**: Switch to new config system, keep old as backup | |
| 4. **Week 4**: Remove old config code after testing | |
| ## Success Metrics | |
| - [ ] Config file split into logical packages | |
| - [ ] Validation using struct tags | |
| - [ ] All providers implement same interface | |
| - [ ] Hot reload works for all storage types | |
| - [ ] Migrations tested and documented | |