| { | |
| "princeton-nlp/Sheared-LLaMA-1.3B": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "defog/sqlcoder-7b-2": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "m-a-p/OpenCodeInterpreter-DS-6.7B": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "ibm/labradorite-13b": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 8, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 8, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 8, | |
| "sequence_length": 4096, | |
| "num_cores": 8, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "gorilla-llm/gorilla-openfunctions-v2": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "m-a-p/ChatMusician": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "LargeWorldModel/LWM-Text-Chat-1M": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 2, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "01-ai/Yi-34B-200K": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 24, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 24, | |
| "auto_cast_type": "fp16" | |
| } | |
| ], | |
| "abacusai/Smaug-72B-v0.1": [ | |
| { | |
| "batch_size": 1, | |
| "sequence_length": 4096, | |
| "num_cores": 24, | |
| "auto_cast_type": "fp16" | |
| }, | |
| { | |
| "batch_size": 4, | |
| "sequence_length": 4096, | |
| "num_cores": 24, | |
| "auto_cast_type": "fp16" | |
| } | |
| ] | |
| } | |