| import time, asyncio |
| from openai import AsyncOpenAI |
| import uuid |
| import traceback |
|
|
|
|
| litellm_client = AsyncOpenAI( |
| api_key="sk-1234", |
| base_url="http://0.0.0.0:8000" |
| ) |
|
|
|
|
| async def litellm_completion(): |
| |
| try: |
| response = await litellm_client.chat.completions.create( |
| model="gpt-3.5-turbo", |
| messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], |
| ) |
| return response |
|
|
| except Exception as e: |
| |
| with open("error_log.txt", "a") as error_log: |
| error_log.write(f"Error during completion: {str(e)}\n") |
| pass |
| |
|
|
|
|
| async def main(): |
| start = time.time() |
| n = 1000 |
| tasks = [litellm_completion() for _ in range(n)] |
|
|
| chat_completions = await asyncio.gather(*tasks) |
|
|
| successful_completions = [c for c in chat_completions if c is not None] |
|
|
| |
| with open("error_log.txt", "a") as error_log: |
| for completion in chat_completions: |
| if isinstance(completion, str): |
| error_log.write(completion + "\n") |
|
|
| print(n, time.time() - start, len(successful_completions)) |
|
|
| if __name__ == "__main__": |
| |
| open("error_log.txt", "w").close() |
|
|
| asyncio.run(main()) |
|
|
