Spaces:
Paused
Paused
File size: 1,343 Bytes
440cddf 9284ad7 440cddf 9284ad7 440cddf 9284ad7 440cddf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | "use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenLimits = void 0;
class TokenLimits {
maxTokens;
requestTokens;
responseTokens;
knowledgeCutOff;
constructor(model = 'llama-3.1-8b-instant') {
this.knowledgeCutOff = '2023-12-01';
// Adjust for Groq TPM/RPM limits on Free/On-Demand tier
if (model.includes('llama-4')) {
this.maxTokens = 29500;
this.responseTokens = 4000;
}
else if (model.includes('llama-3.1-8b') ||
model.includes('llama-3.2-1') ||
model.includes('llama-3.1-70b') ||
model.includes('llama-3.3-70b')) {
// Groq limits are ~6k TPM for 8b and ~12k TPM for 70b
this.maxTokens = model.includes('8b') ? 5500 : 11500;
this.responseTokens = 2000; // Reserve less for response to allow more context
}
else {
this.maxTokens = 128000;
this.responseTokens = 16000;
}
// Provide some margin
this.requestTokens = this.maxTokens - this.responseTokens - 500;
}
string() {
return `max_tokens=${this.maxTokens}, request_tokens=${this.requestTokens}, response_tokens=${this.responseTokens}`;
}
}
exports.TokenLimits = TokenLimits;
//# sourceMappingURL=limits.js.map |