Spaces:
Paused
Paused
File size: 1,124 Bytes
9284ad7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | export class TokenLimits {
maxTokens: number
requestTokens: number
responseTokens: number
knowledgeCutOff: string
constructor(model = 'llama-3.1-8b-instant') {
this.knowledgeCutOff = '2023-12-01'
// Adjust for Groq TPM/RPM limits on Free/On-Demand tier
if (model.includes('llama-4')) {
this.maxTokens = 29500
this.responseTokens = 4000
} else if (
model.includes('llama-3.1-8b') ||
model.includes('llama-3.2-1') ||
model.includes('llama-3.1-70b') ||
model.includes('llama-3.3-70b')
) {
// Groq limits are ~6k TPM for 8b and ~12k TPM for 70b
this.maxTokens = model.includes('8b') ? 5500 : 11500
this.responseTokens = 2000 // Reserve less for response to allow more context
} else {
this.maxTokens = 128000
this.responseTokens = 16000
}
// Provide some margin
this.requestTokens = this.maxTokens - this.responseTokens - 500
}
string(): string {
return `max_tokens=${this.maxTokens}, request_tokens=${this.requestTokens}, response_tokens=${this.responseTokens}`
}
}
|