File size: 8,435 Bytes
9284ad7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
// Node 20 includes native fetch support

// AI Client for Groq with Failover Support + Circuit Breaker
// Fallback shims for Probot runtime
/* eslint-disable no-console */
import {
  ChatGPTAPI as AIClient,
  ChatGPTError as AIError,
  ChatMessage,
  SendMessageOptions
} from 'chatgpt'
import pRetry from 'p-retry'
import CircuitBreaker from 'opossum'
import {AIOptions, Options} from './options'
import {TokenLimits} from './limits'
import pino from 'pino'

const logger = pino({ level: process.env.LOG_LEVEL || 'info' })

// Circuit breaker configuration
const CIRCUIT_BREAKER_OPTIONS = {
  timeout: 60000, // 60 seconds
  errorThresholdPercentage: 50,
  resetTimeout: 30000,
  volumeThreshold: 5,
}

const logInfo = (msg: string) => logger.info(msg)
const logWarning = (msg: string) => logger.warn(msg)
const logError = (msg: string) => logger.error(msg)
const setFailed = (msg: string) => {
  logger.error(msg)
  process.exit(1)
}

export interface Ids {
  parentMessageId?: string
  conversationId?: string
}

export class Bot {
  private api: AIClient | null = null
  private readonly options: Options
  private readonly aiOptions: AIOptions
  private currentModel: string
  private fallbackModels: string[]
  private circuitBreaker: CircuitBreaker | null = null

  constructor(options: Options, aiOptions: AIOptions) {
    this.options = options
    this.aiOptions = aiOptions
    this.currentModel = aiOptions.model

    // Setup fallback chain based on tier
    if (Options.HEAVY_MODELS.includes(this.currentModel)) {
      this.fallbackModels = [...Options.HEAVY_MODELS]
    } else {
      this.fallbackModels = [...Options.LIGHT_MODELS]
    }

    // Remove current model from fallbacks to avoid immediate repeat
    this.fallbackModels = this.fallbackModels.filter(
      m => m !== this.currentModel
    )

    this.initClient()
    this.initCircuitBreaker()
  }

  private initCircuitBreaker(): void {
    if (!this.api) return

    // Wrap the sendMessage method with circuit breaker
    this.circuitBreaker = new CircuitBreaker(
      async ({ message, opts }: { message: string; opts: SendMessageOptions }) => {
        return this.api!.sendMessage(message, opts)
      },
      CIRCUIT_BREAKER_OPTIONS
    )

    // Circuit breaker event handlers
    this.circuitBreaker.on('open', () => {
      logWarning(`Circuit breaker opened for model: ${this.currentModel}`)
    })

    this.circuitBreaker.on('halfOpen', () => {
      logInfo(`Circuit breaker half-open for model: ${this.currentModel}`)
    })

    this.circuitBreaker.on('close', () => {
      logInfo(`Circuit breaker closed for model: ${this.currentModel}`)
    })

    this.circuitBreaker.on('fallback', (_result: unknown) => {
      logWarning(`Circuit breaker fallback executed for model: ${this.currentModel}`)
    })
  }

  private initClient(): void {
    const apiKey = process.env.GROQ_API_KEY || process.env.AI_API_KEY
    if (apiKey) {
      const currentDate = new Date().toISOString().split('T')[0]
      const limits = new TokenLimits(this.currentModel)
      const systemMessage = `${this.options.systemMessage} 

Knowledge cutoff: ${limits.knowledgeCutOff}

Current date: ${currentDate}

IMPORTANT: Entire response must be in ISO code: ${this.options.language}

`

      this.api = new AIClient({
        apiBaseUrl: this.options.apiBaseUrl,
        systemMessage,
        apiKey,
        apiOrg: process.env.AI_API_ORG || undefined,
        debug: this.options.debug,
        maxModelTokens: limits.maxTokens,
        maxResponseTokens: limits.responseTokens,
        completionParams: {
          temperature: this.options.modelTemperature,
          model: this.currentModel
        }
      })
      logInfo(`Initialized AI client with model: ${this.currentModel}`)
    } else {
      throw new Error("Missing 'GROQ_API_KEY' or 'AI_API_KEY'")
    }
  }

  chat = async (message: string, ids: Ids): Promise<[string, Ids]> => {
    let res: [string, Ids] = ['', {}]
    try {
      res = await this.chat_(message, ids)
      return res
    } catch (e: unknown) {
      if (e instanceof AIError) {
        logWarning(`AI communication error: ${e.message}`)
      }
      return res
    }
  }

  private readonly chat_ = async (
    message: string,
    ids: Ids
  ): Promise<[string, Ids]> => {
    const start = Date.now()
    if (!message) return ['', {}]

    let response: ChatMessage | undefined

    if (this.api != null) {
      const opts: SendMessageOptions = {
        timeoutMs: this.options.timeoutMS
      }
      if (ids.parentMessageId) {
        opts.parentMessageId = ids.parentMessageId
      }

      try {
        // Check if circuit breaker is open
        if (this.circuitBreaker?.opened) {
          logWarning(`Circuit breaker is OPEN for ${this.currentModel}, attempting fallback...`)
          if (this.fallbackModels.length > 0) {
            const nextModel = this.fallbackModels.shift()
            if (nextModel) {
              this.currentModel = nextModel
              this.initClient()
              this.initCircuitBreaker()
              // Recursive retry with new model
              return this.chat_(message, ids)
            }
          }
          throw new Error(`Circuit breaker open and no fallback models available for ${this.currentModel}`)
        }

        const result = await pRetry<ChatMessage>(
          async () => {
            if (!this.circuitBreaker) {
              throw new Error('Circuit breaker not initialized')
            }
            try {
              const messageResult = await this.circuitBreaker.fire({ message, opts })
              return messageResult as ChatMessage
            } catch (e: any) {
              // Check if circuit breaker opened
              if (this.circuitBreaker.opened && this.fallbackModels.length > 0) {
                const nextModel = this.fallbackModels.shift()
                if (nextModel) {
                  logWarning(
                    `Circuit breaker opened for ${this.currentModel}. Switching to fallback: ${nextModel}`
                  )
                  this.currentModel = nextModel
                  this.initClient()
                  this.initCircuitBreaker()
                  // Throw to trigger retry with new model
                  throw new Error(`Model switched to ${nextModel}, retrying...`)
                }
              }
              
              // Handle rate limiting / context size
              if ((e?.status === 429 || e?.status === 413) && this.fallbackModels.length > 0) {
                const nextModel = this.fallbackModels.shift()
                if (nextModel) {
                  logWarning(
                    `Rate limit or context size error (${e.status}) hit for ${this.currentModel}. Switching to fallback: ${nextModel}`
                  )
                  this.currentModel = nextModel
                  this.initClient()
                  this.initCircuitBreaker()
                  throw new Error(`Model switched to ${nextModel}, retrying...`)
                }
              }
              throw e
            }
          },
          {
            retries: this.options.retries,
            minTimeout: 20000, // 20s delay to allow TPM refresh
            onFailedAttempt: (err: any) => {
              logInfo(
                `Attempt ${err.attemptNumber} failed. ${err.retriesLeft} retries left. Error: ${err.message}`
              )
            }
          }
        )
        response = result
      } catch (e: unknown) {
        if (e instanceof AIError) {
          logWarning(
            `AI Error after retries: ${e.message} (Last model used: ${this.currentModel})`
          )
        } else if (e instanceof Error) {
          logError(`Failed to get AI response: ${e.message}`)
        }
      }

      const end = Date.now()
      logInfo(`AI response time: ${end - start} ms`)
    } else {
      setFailed('AI client is not initialized')
    }

    let responseText = ''
    if (response != null) {
      responseText = response.text
    }

    const newIds: Ids = {
      parentMessageId: response?.id,
      conversationId: response?.conversationId
    }
    return [responseText, newIds]
  }
}