adelelsayed1991 commited on
Commit
f2d515b
·
verified ·
1 Parent(s): 7bbc0fc

Log generation time and token count so the GPU duration can be measured, not guessed

Browse files
Files changed (1) hide show
  1. app.py +16 -0
app.py CHANGED
@@ -10,6 +10,7 @@ fenced ```sql block (extract_sql_from_completion), not the raw completion.
10
  """
11
 
12
  import re
 
13
 
14
  import spaces
15
  import gradio as gr
@@ -132,12 +133,27 @@ def generate_sql(question: str) -> tuple[str, str]:
132
  inputs = tokenizer(prompt_text, return_tensors="pt", add_special_tokens=False).to("cuda")
133
  prompt_len = inputs["input_ids"].shape[1]
134
 
 
135
  output_ids = model.generate(
136
  **inputs,
137
  max_new_tokens=MAX_NEW_TOKENS,
138
  do_sample=False,
139
  pad_token_id=tokenizer.eos_token_id,
140
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
  completion = tokenizer.decode(
142
  output_ids[0][prompt_len:], skip_special_tokens=True
143
  ).strip()
 
10
  """
11
 
12
  import re
13
+ import time
14
 
15
  import spaces
16
  import gradio as gr
 
133
  inputs = tokenizer(prompt_text, return_tensors="pt", add_special_tokens=False).to("cuda")
134
  prompt_len = inputs["input_ids"].shape[1]
135
 
136
+ started = time.perf_counter()
137
  output_ids = model.generate(
138
  **inputs,
139
  max_new_tokens=MAX_NEW_TOKENS,
140
  do_sample=False,
141
  pad_token_id=tokenizer.eos_token_id,
142
  )
143
+ elapsed = time.perf_counter() - started
144
+ new_tokens = output_ids.shape[1] - prompt_len
145
+
146
+ # The decorator's `duration` above has to exceed this, or ZeroGPU kills
147
+ # the request mid-generation and the caller sees only "GPU task
148
+ # aborted". Logging it turns that budget into something measured
149
+ # rather than guessed: read these lines from the Space logs and set
150
+ # `duration` from the worst observed generation, plus headroom.
151
+ print(
152
+ f"[timing] generated {new_tokens} tokens in {elapsed:.1f}s "
153
+ f"({new_tokens / elapsed:.1f} tok/s), prompt {prompt_len} tokens",
154
+ flush=True,
155
+ )
156
+
157
  completion = tokenizer.decode(
158
  output_ids[0][prompt_len:], skip_special_tokens=True
159
  ).strip()