File size: 17,349 Bytes
2e6b65c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 | # Configuration Management Refactoring Plan
## Current State Analysis
**File**: `internal/config/config.go` (1,767 lines)
**Issues**:
- Single file with all config structures
- Mix of loading, validation, and migration logic
- Complex nested structs with 326+ lines of YAML example
- No clear separation between config sources (file, env, DB)
- Migration logic scattered throughout
- No structured validation (manual checks)
## Target Architecture
```
internal/config/
βββ domain/
β βββ models.go # Core config structs (no logic)
β βββ validation.go # Validation rules
βββ loader/
β βββ file.go # YAML/JSON file loading
β βββ env.go # Environment variable loading
β βββ merge.go # Config merging logic
βββ provider/
β βββ interface.go # ConfigProvider interface
β βββ file.go # File-based provider
β βββ postgres.go # PostgreSQL provider
β βββ git.go # Git-based provider
β βββ objectstore.go # S3/MinIO provider
βββ migrations/
β βββ registry.go # Migration registry
β βββ v1_to_v2.go # Specific migrations
β βββ runner.go # Migration runner
βββ watcher/
β βββ file.go # File watching for hot reload
βββ config.go # Public API (facade)
```
## Phase 1: Domain Models Extraction (Week 1)
### 1.1 Core Models
**File**: `internal/config/domain/models.go`
```go
package domain
import (
"time"
"github.com/echyai/cliproxyapi/internal/domain/auth"
)
// Config is the root configuration
type Config struct {
Version string `yaml:"version" json:"version" validate:"required,semver"`
Server Server `yaml:"server" json:"server" validate:"required"`
Providers Providers `yaml:"providers" json:"providers"`
Storage Storage `yaml:"storage" json:"storage" validate:"required"`
Logging Logging `yaml:"logging" json:"logging"`
Metrics Metrics `yaml:"metrics" json:"metrics"`
}
func (c *Config) Validate() error {
validate := validator.New()
validate.RegisterValidation("semver", validateSemver)
return validate.Struct(c)
}
// Server configuration
type Server struct {
Host string `yaml:"host" json:"host" validate:"hostname|ip"`
Port int `yaml:"port" json:"port" validate:"required,min=1,max=65535"`
ReadTimeout time.Duration `yaml:"read_timeout" json:"read_timeout" validate:"min=1s"`
WriteTimeout time.Duration `yaml:"write_timeout" json:"write_timeout" validate:"min=1s"`
MaxHeaderBytes int `yaml:"max_header_bytes" json:"max_header_bytes" validate:"min=1024"`
TLS *TLSConfig `yaml:"tls,omitempty" json:"tls,omitempty"`
}
// TLS configuration
type TLSConfig struct {
CertFile string `yaml:"cert_file" json:"cert_file" validate:"required,file"`
KeyFile string `yaml:"key_file" json:"key_file" validate:"required,file"`
}
// Providers configuration
type Providers struct {
Gemini *GeminiConfig `yaml:"gemini,omitempty" json:"gemini,omitempty"`
Claude *ClaudeConfig `yaml:"claude,omitempty" json:"claude,omitempty"`
Codex *CodexConfig `yaml:"codex,omitempty" json:"codex,omitempty"`
Antigravity *AntigravityConfig `yaml:"antigravity,omitempty" json:"antigravity,omitempty"`
Qwen *QwenConfig `yaml:"qwen,omitempty" json:"qwen,omitempty"`
Vertex *VertexConfig `yaml:"vertex,omitempty" json:"vertex,omitempty"`
}
// Gemini configuration
type GeminiConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
APIKey string `yaml:"api_key,omitempty" json:"api_key,omitempty" validate:"omitempty,min=10"`
OAuth *OAuthConfig `yaml:"oauth,omitempty" json:"oauth,omitempty"`
RateLimit RateLimitConfig `yaml:"rate_limit" json:"rate_limit"`
Models []ModelConfig `yaml:"models" json:"models" validate:"dive"`
}
// OAuth configuration (shared)
type OAuthConfig struct {
ClientID string `yaml:"client_id" validate:"required"`
ClientSecret string `yaml:"client_secret" validate:"required"`
RedirectURL string `yaml:"redirect_url" validate:"required,url"`
Scopes []string `yaml:"scopes" validate:"min=1"`
}
// Rate limit configuration
type RateLimitConfig struct {
RequestsPerSecond float64 `yaml:"requests_per_second" validate:"min=0.1"`
Burst int `yaml:"burst" validate:"min=1"`
Cooldown time.Duration `yaml:"cooldown" validate:"min=0"`
}
// Model configuration
type ModelConfig struct {
Name string `yaml:"name" validate:"required"`
DisplayName string `yaml:"display_name"`
MaxTokens int `yaml:"max_tokens" validate:"min=1"`
Enabled bool `yaml:"enabled"`
}
// Storage configuration
type Storage struct {
Type StorageType `yaml:"type" validate:"required,oneof=file postgres git objectstore"`
File *FileStorage `yaml:"file,omitempty"`
Postgres *PostgresStorage `yaml:"postgres,omitempty"`
Git *GitStorage `yaml:"git,omitempty"`
Object *ObjectStorage `yaml:"object,omitempty"`
}
type StorageType string
const (
StorageTypeFile StorageType = "file"
StorageTypePostgres StorageType = "postgres"
StorageTypeGit StorageType = "git"
StorageTypeObject StorageType = "objectstore"
)
// FileStorage configuration
type FileStorage struct {
Path string `yaml:"path" validate:"required,dirpath"`
}
// PostgresStorage configuration
type PostgresStorage struct {
Host string `yaml:"host" validate:"required"`
Port int `yaml:"port" validate:"min=1,max=65535"`
Database string `yaml:"database" validate:"required"`
Username string `yaml:"username" validate:"required"`
Password string `yaml:"password" validate:"required"`
SSLMode string `yaml:"ssl_mode" validate:"oneof=disable require verify-ca verify-full"`
}
// Logging configuration
type Logging struct {
Level string `yaml:"level" validate:"oneof=debug info warn error fatal panic"`
Format string `yaml:"format" validate:"oneof=json text"`
Output string `yaml:"output" validate:"oneof=stdout stderr file"`
File string `yaml:"file,omitempty"`
}
// Metrics configuration
type Metrics struct {
Enabled bool `yaml:"enabled"`
Path string `yaml:"path" validate:"startswith=/"`
Port int `yaml:"port" validate:"min=1,max=65535"`
}
// Provider-specific configs (Claude, Codex, etc. follow similar pattern)
type ClaudeConfig struct {
Enabled bool `yaml:"enabled"`
APIKey string `yaml:"api_key,omitempty"`
OAuth *OAuthConfig `yaml:"oauth,omitempty"`
RateLimit RateLimitConfig `yaml:"rate_limit"`
}
// ... other provider configs
```
### 1.2 Validation Rules
**File**: `internal/config/domain/validation.go`
```go
package domain
import (
"github.com/go-playground/validator/v10"
"regexp"
)
var semverRegex = regexp.MustCompile(`^v?(\d+)\.(\d+)\.(\d+)(?:-([\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*))?(?:\+([\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*))?$`)
func validateSemver(fl validator.FieldLevel) bool {
return semverRegex.MatchString(fl.Field().String())
}
// ConfigValidator handles validation with custom rules
type ConfigValidator struct {
validate *validator.Validate
}
func NewValidator() *ConfigValidator {
v := validator.New()
v.RegisterValidation("semver", validateSemver)
return &ConfigValidator{validate: v}
}
func (cv *ConfigValidator) Validate(cfg *Config) error {
if err := cv.validate.Struct(cfg); err != nil {
return NewValidationError(err)
}
// Cross-field validations
if err := cv.validateStorage(cfg.Storage); err != nil {
return err
}
if err := cv.validateProviders(cfg.Providers); err != nil {
return err
}
return nil
}
func (cv *ConfigValidator) validateStorage(s Storage) error {
switch s.Type {
case StorageTypeFile:
if s.File == nil {
return ValidationError{Field: "storage.file", Message: "required when type is file"}
}
case StorageTypePostgres:
if s.Postgres == nil {
return ValidationError{Field: "storage.postgres", Message: "required when type is postgres"}
}
// ... other cases
}
return nil
}
func (cv *ConfigValidator) validateProviders(p Providers) error {
// At least one provider should be enabled
hasEnabled := false
// Check each provider...
if !hasEnabled {
return ValidationError{Message: "at least one provider must be enabled"}
}
return nil
}
```
## Phase 2: Provider Interface (Week 2)
### 2.1 Provider Interface
**File**: `internal/config/provider/interface.go`
```go
package provider
import (
"context"
"github.com/echyai/cliproxyapi/internal/config/domain"
)
// Provider loads and saves configuration
type Provider interface {
// Load retrieves the current configuration
Load(ctx context.Context) (*domain.Config, error)
// Save persists the configuration
Save(ctx context.Context, cfg *domain.Config) error
// Watch returns a channel that receives config updates
Watch(ctx context.Context) (<-chan domain.Config, error)
// Close cleans up resources
Close() error
}
// ProviderFactory creates providers based on type
type ProviderFactory struct {
// dependencies
}
func (f *ProviderFactory) Create(storage domain.Storage) (Provider, error) {
switch storage.Type {
case domain.StorageTypeFile:
return NewFileProvider(storage.File)
case domain.StorageTypePostgres:
return NewPostgresProvider(storage.Postgres)
case domain.StorageTypeGit:
return NewGitProvider(storage.Git)
case domain.StorageTypeObject:
return NewObjectProvider(storage.Object)
default:
return nil, fmt.Errorf("unknown storage type: %s", storage.Type)
}
}
```
### 2.2 File Provider
**File**: `internal/config/provider/file.go`
```go
package provider
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/fsnotify/fsnotify"
"gopkg.in/yaml.v3"
"github.com/echyai/cliproxyapi/internal/config/domain"
)
type FileProvider struct {
path string
watcher *fsnotify.Watcher
validate *domain.ConfigValidator
}
func NewFileProvider(cfg *domain.FileStorage) (*FileProvider, error) {
absPath, err := filepath.Abs(cfg.Path)
if err != nil {
return nil, fmt.Errorf("invalid path: %w", err)
}
return &FileProvider{
path: absPath,
validate: domain.NewValidator(),
}, nil
}
func (p *FileProvider) Load(ctx context.Context) (*domain.Config, error) {
data, err := os.ReadFile(p.path)
if err != nil {
if os.IsNotExist(err) {
return nil, domain.ErrConfigNotFound
}
return nil, fmt.Errorf("read config: %w", err)
}
var cfg domain.Config
if err := yaml.Unmarshal(data, &cfg); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
if err := p.validate.Validate(&cfg); err != nil {
return nil, fmt.Errorf("validate config: %w", err)
}
return &cfg, nil
}
func (p *FileProvider) Save(ctx context.Context, cfg *domain.Config) error {
if err := p.validate.Validate(cfg); err != nil {
return err
}
data, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshal config: %w", err)
}
// Write atomically
tmpPath := p.path + ".tmp"
if err := os.WriteFile(tmpPath, data, 0644); err != nil {
return fmt.Errorf("write temp file: %w", err)
}
return os.Rename(tmpPath, p.path)
}
func (p *FileProvider) Watch(ctx context.Context) (<-chan domain.Config, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
if err := watcher.Add(p.path); err != nil {
return nil, err
}
p.watcher = watcher
updates := make(chan domain.Config)
go func() {
defer close(updates)
defer watcher.Close()
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
if event.Op&fsnotify.Write == fsnotify.Write {
cfg, err := p.Load(ctx)
if err == nil {
updates <- *cfg
}
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
// Log error
_ = err
case <-ctx.Done():
return
}
}
}()
return updates, nil
}
func (p *FileProvider) Close() error {
if p.watcher != nil {
return p.watcher.Close()
}
return nil
}
```
## Phase 3: Migration System (Week 2)
### 3.1 Migration Framework
**File**: `internal/config/migrations/registry.go`
```go
package migrations
import (
"fmt"
"github.com/echyai/cliproxyapi/internal/config/domain"
)
// Migration transforms config from one version to another
type Migration interface {
FromVersion() string
ToVersion() string
Migrate(cfg map[string]interface{}) (map[string]interface{}, error)
}
// Registry holds all migrations
type Registry struct {
migrations map[string]Migration
}
func NewRegistry() *Registry {
r := &Registry{migrations: make(map[string]Migration)}
r.registerDefaults()
return r
}
func (r *Registry) register(m Migration) {
key := fmt.Sprintf("%s->%s", m.FromVersion(), m.ToVersion())
r.migrations[key] = m
}
func (r *Registry) registerDefaults() {
r.register(&V1ToV2Migration{})
r.register(&V2ToV3Migration{})
// ... more migrations
}
// Runner executes migrations
type Runner struct {
registry *Registry
}
func (r *Runner) Migrate(cfg map[string]interface{}, targetVersion string) (map[string]interface{}, error) {
currentVersion, _ := cfg["version"].(string)
if currentVersion == "" {
currentVersion = "1.0.0"
}
for currentVersion != targetVersion {
key := fmt.Sprintf("%s->%s", currentVersion, targetVersion)
migration, ok := r.registry.migrations[key]
if !ok {
// Try to find intermediate step
nextVersion := findNextVersion(currentVersion)
if nextVersion == "" {
return nil, fmt.Errorf("no migration path from %s to %s", currentVersion, targetVersion)
}
key = fmt.Sprintf("%s->%s", currentVersion, nextVersion)
migration = r.registry.migrations[key]
}
var err error
cfg, err = migration.Migrate(cfg)
if err != nil {
return nil, fmt.Errorf("migrate %s: %w", key, err)
}
currentVersion = migration.ToVersion()
}
return cfg, nil
}
```
## Phase 4: Public API Facade (Week 3)
**File**: `internal/config/config.go` (simplified facade)
```go
package config
import (
"context"
"sync"
"github.com/echyai/cliproxyapi/internal/config/domain"
"github.com/echyai/cliproxyapi/internal/config/migrations"
"github.com/echyai/cliproxyapi/internal/config/provider"
)
// Manager is the public API for configuration
type Manager struct {
provider provider.Provider
current *domain.Config
mu sync.RWMutex
validate *domain.ConfigValidator
}
func NewManager(storage domain.Storage) (*Manager, error) {
p, err := provider.NewFactory().Create(storage)
if err != nil {
return nil, err
}
return &Manager{
provider: p,
validate: domain.NewValidator(),
}, nil
}
func (m *Manager) Load(ctx context.Context) error {
cfg, err := m.provider.Load(ctx)
if err != nil {
return err
}
m.mu.Lock()
m.current = cfg
m.mu.Unlock()
return nil
}
func (m *Manager) Get() *domain.Config {
m.mu.RLock()
defer m.mu.RUnlock()
return m.current
}
func (m *Manager) Update(ctx context.Context, updater func(*domain.Config) error) error {
m.mu.Lock()
defer m.mu.Unlock()
cfg := *m.current // Copy
if err := updater(&cfg); err != nil {
return err
}
if err := m.validate.Validate(&cfg); err != nil {
return err
}
if err := m.provider.Save(ctx, &cfg); err != nil {
return err
}
m.current = &cfg
return nil
}
func (m *Manager) Watch(ctx context.Context) (<-chan domain.Config, error) {
return m.provider.Watch(ctx)
}
func (m *Manager) Close() error {
return m.provider.Close()
}
```
## Migration Steps
1. **Week 1**: Create new structure alongside existing code
2. **Week 2**: Implement providers and migrations
3. **Week 3**: Switch to new config system, keep old as backup
4. **Week 4**: Remove old config code after testing
## Success Metrics
- [ ] Config file split into logical packages
- [ ] Validation using struct tags
- [ ] All providers implement same interface
- [ ] Hot reload works for all storage types
- [ ] Migrations tested and documented
|