Log generation time and token count so the GPU duration can be measured, not guessed
Browse files
app.py
CHANGED
|
@@ -10,6 +10,7 @@ fenced ```sql block (extract_sql_from_completion), not the raw completion.
|
|
| 10 |
"""
|
| 11 |
|
| 12 |
import re
|
|
|
|
| 13 |
|
| 14 |
import spaces
|
| 15 |
import gradio as gr
|
|
@@ -132,12 +133,27 @@ def generate_sql(question: str) -> tuple[str, str]:
|
|
| 132 |
inputs = tokenizer(prompt_text, return_tensors="pt", add_special_tokens=False).to("cuda")
|
| 133 |
prompt_len = inputs["input_ids"].shape[1]
|
| 134 |
|
|
|
|
| 135 |
output_ids = model.generate(
|
| 136 |
**inputs,
|
| 137 |
max_new_tokens=MAX_NEW_TOKENS,
|
| 138 |
do_sample=False,
|
| 139 |
pad_token_id=tokenizer.eos_token_id,
|
| 140 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
completion = tokenizer.decode(
|
| 142 |
output_ids[0][prompt_len:], skip_special_tokens=True
|
| 143 |
).strip()
|
|
|
|
| 10 |
"""
|
| 11 |
|
| 12 |
import re
|
| 13 |
+
import time
|
| 14 |
|
| 15 |
import spaces
|
| 16 |
import gradio as gr
|
|
|
|
| 133 |
inputs = tokenizer(prompt_text, return_tensors="pt", add_special_tokens=False).to("cuda")
|
| 134 |
prompt_len = inputs["input_ids"].shape[1]
|
| 135 |
|
| 136 |
+
started = time.perf_counter()
|
| 137 |
output_ids = model.generate(
|
| 138 |
**inputs,
|
| 139 |
max_new_tokens=MAX_NEW_TOKENS,
|
| 140 |
do_sample=False,
|
| 141 |
pad_token_id=tokenizer.eos_token_id,
|
| 142 |
)
|
| 143 |
+
elapsed = time.perf_counter() - started
|
| 144 |
+
new_tokens = output_ids.shape[1] - prompt_len
|
| 145 |
+
|
| 146 |
+
# The decorator's `duration` above has to exceed this, or ZeroGPU kills
|
| 147 |
+
# the request mid-generation and the caller sees only "GPU task
|
| 148 |
+
# aborted". Logging it turns that budget into something measured
|
| 149 |
+
# rather than guessed: read these lines from the Space logs and set
|
| 150 |
+
# `duration` from the worst observed generation, plus headroom.
|
| 151 |
+
print(
|
| 152 |
+
f"[timing] generated {new_tokens} tokens in {elapsed:.1f}s "
|
| 153 |
+
f"({new_tokens / elapsed:.1f} tok/s), prompt {prompt_len} tokens",
|
| 154 |
+
flush=True,
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
completion = tokenizer.decode(
|
| 158 |
output_ids[0][prompt_len:], skip_special_tokens=True
|
| 159 |
).strip()
|