bash98 commited on
Commit
ba6319c
·
verified ·
1 Parent(s): 6ad9750

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +98 -76
app.py CHANGED
@@ -1,106 +1,128 @@
1
- """Document Q&A with Manual RAG - Hugging Face ZeroGPU."""
2
 
3
- # Import spaces BEFORE Gradio and any module importing torch/transformers.
4
- # ZeroGPU uses this import to install CUDA emulation during startup.
5
  import spaces
 
6
  import os
7
  import traceback
 
8
  import gradio as gr
9
 
10
  from rag_core import (
11
- load_retriever,
12
  load_generator,
 
13
  answer_question,
14
  display_evidence,
15
  )
16
 
17
 
18
- @spaces.GPU(duration=60)
19
- def respond(question, retrieval_only):
20
- """Retrieve evidence and optionally generate a grounded answer on ZeroGPU."""
21
- if not isinstance(question, str) or not question.strip():
22
- return "Please enter a question.", "", "Input error"
23
 
 
 
 
24
  try:
25
- result = answer_question(
26
- question.strip(), retrieval_only=bool(retrieval_only)
27
- )
28
- return result["answer"], display_evidence(result), result["status"]
 
29
 
30
- except (ValueError, RuntimeError) as error:
 
 
 
 
31
  traceback.print_exc()
32
- return str(error), "", "Input or setup error"
 
33
 
34
- except Exception:
 
 
 
 
 
35
  traceback.print_exc()
36
- return (
37
- "Dataset/model access or generation failed. Check the Space logs.",
38
- "",
39
- "Model error",
40
- )
 
 
 
 
41
 
42
 
43
  def make_demo():
44
- return gr.Interface(
45
- fn=respond,
46
- inputs=[
47
- gr.Textbox(
48
- label="Question",
49
- placeholder="Which company produces the iPod?",
50
- lines=2,
51
- ),
52
- gr.Checkbox(
53
- label="Retrieval only (skip answer generation)",
54
- value=False,
55
- ),
56
- ],
57
- outputs=[
58
- gr.Textbox(label="Answer - verify against the evidence", lines=5),
59
- gr.Textbox(label="Source passages and similarity scores", lines=15),
60
- gr.Textbox(label="Run status"),
61
- ],
62
- title="Document Q&A with Manual RAG",
63
- description=(
64
- "Ask questions about 30 SQuAD/Wikipedia passages loaded from "
65
- "Hugging Face. The app retrieves evidence and optionally generates "
66
- "an answer with FLAN-T5 on ZeroGPU. Verify answers against sources."
67
- ),
68
- examples=[
69
- ["Which company produces the iPod?", False],
70
- ["At what age did Frederic move to Paris?", False],
71
- ["What is Beyonce's private mobile phone number?", False],
72
- ],
73
- cache_examples=False,
74
- )
 
 
 
 
 
 
 
 
 
 
 
75
 
76
 
77
- def main():
78
- print("Initializing Document Q&A with Manual RAG (ZeroGPU)...")
79
- try:
80
- shape = load_retriever()
81
- print("Retriever initialized:", shape)
82
-
83
- # IMPORTANT: Preload the generation model and register its CUDA weights
84
- # before Gradio launches. The ZeroGPU runtime intercepts .to('cuda')
85
- # here, then transfers them to an actual GPU inside @spaces.GPU.
86
- print("Preparing FLAN-T5 for ZeroGPU...")
87
- model_id = load_generator()
88
- print("Generator ready:", model_id)
89
- except Exception:
90
- print("Failed to initialize RAG models.")
91
- traceback.print_exc()
92
- raise
93
 
94
- demo = make_demo()
95
- demo.queue(default_concurrency_limit=1, max_size=20)
 
 
96
  demo.launch(
97
  server_name="0.0.0.0",
98
- server_port=int(os.environ.get("PORT", "7860")),
99
- share=False,
100
  show_error=True,
 
101
  ssr_mode=False,
102
  )
103
-
104
-
105
- if __name__ == "__main__":
106
- main()
 
1
+ """Manual RAG: separate CPU retrieval, CPU fallback, and ZeroGPU generation."""
2
 
3
+ # MUST come before torch, Gradio, sentence-transformers, or transformers.
 
4
  import spaces
5
+
6
  import os
7
  import traceback
8
+ import torch
9
  import gradio as gr
10
 
11
  from rag_core import (
 
12
  load_generator,
13
+ load_retriever,
14
  answer_question,
15
  display_evidence,
16
  )
17
 
18
 
19
+ def render_result(result):
20
+ return result["answer"], display_evidence(result), result["status"]
 
 
 
21
 
22
+
23
+ def respond_retrieval(question):
24
+ """Retrieval-only processing on CPU; it never asks for a GPU."""
25
  try:
26
+ return render_result(answer_question(question, retrieval_only=True))
27
+ except Exception as error:
28
+ traceback.print_exc()
29
+ return f"Retrieval failed: {error}", "", "CPU retrieval error"
30
+
31
 
32
+ def respond_cpu(question):
33
+ """CPU answer-generation fallback when ZeroGPU is unavailable."""
34
+ try:
35
+ return render_result(answer_question(question, device="cpu"))
36
+ except Exception as error:
37
  traceback.print_exc()
38
+ return f"CPU generation failed: {error}", "", "CPU generation error"
39
+
40
 
41
+ @spaces.GPU(duration=60)
42
+ def respond_gpu(question):
43
+ """Answer with the model that ZeroGPU has packed for CUDA."""
44
+ try:
45
+ return render_result(answer_question(question, device="cuda"))
46
+ except Exception as error:
47
  traceback.print_exc()
48
+ return f"GPU generation failed: {error}", "", "GPU generation error"
49
+
50
+
51
+ @spaces.GPU(duration=10)
52
+ def check_zerogpu():
53
+ """Test GPU allocation without loading or invoking any RAG model."""
54
+ # If ZeroGPU worker_init fails, THIS LINE IS NEVER REACHED.
55
+ value = torch.ones(1, device="cuda").sum().item()
56
+ return f"ZeroGPU allocated successfully: {torch.cuda.get_device_name(0)}; test value = {value:.0f}"
57
 
58
 
59
  def make_demo():
60
+ with gr.Blocks(title="Document Q&A with Manual RAG") as demo:
61
+ gr.Markdown(
62
+ "# Document Q&A with Manual RAG\n"
63
+ "Ask questions about 30 SQuAD/Wikipedia passages. "
64
+ "Evidence is retrieved on CPU; you may generate an answer on "
65
+ "ZeroGPU or use the CPU fallback. Verify answers against sources."
66
+ )
67
+ question = gr.Textbox(
68
+ label="Question",
69
+ placeholder="Which company produces the iPod?",
70
+ lines=2,
71
+ )
72
+ with gr.Row():
73
+ retrieve_button = gr.Button("Retrieve evidence (CPU)")
74
+ gpu_button = gr.Button("Answer with ZeroGPU", variant="primary")
75
+ cpu_button = gr.Button("Answer with CPU fallback")
76
+ answer = gr.Textbox(label="Answer - verify against evidence", lines=5)
77
+ evidence = gr.Textbox(label="Source passages and similarity scores", lines=14)
78
+ status = gr.Textbox(label="Run status")
79
+ with gr.Accordion("ZeroGPU allocation diagnostic", open=False):
80
+ gr.Markdown(
81
+ "This runs one tiny CUDA operation. If it fails with "
82
+ "`No CUDA GPUs are available` in `worker_init`, the error "
83
+ "occurs before your RAG function runs."
84
+ )
85
+ test_button = gr.Button("Test ZeroGPU allocation")
86
+ gpu_test_result = gr.Textbox(label="GPU test result")
87
+
88
+ outputs = [answer, evidence, status]
89
+ retrieve_button.click(fn=respond_retrieval, inputs=question, outputs=outputs)
90
+ cpu_button.click(fn=respond_cpu, inputs=question, outputs=outputs)
91
+ gpu_button.click(fn=respond_gpu, inputs=question, outputs=outputs)
92
+ test_button.click(fn=check_zerogpu, inputs=[], outputs=gpu_test_result)
93
+ gr.Examples(
94
+ examples=[
95
+ ["Which company produces the iPod?"],
96
+ ["At what age did Frédéric move to Paris?"],
97
+ ["What is Beyoncé's private mobile phone number?"],
98
+ ],
99
+ inputs=[question],
100
+ )
101
+ return demo
102
 
103
 
104
+ # Run initialization on the MAIN THREAD at module import; no model loads in
105
+ # Gradio callbacks. In ZeroGPU's parent process, CUDA must remain uninitialized.
106
+ print("Starting Manual RAG (ZeroGPU diagnostics)", flush=True)
107
+ print(
108
+ f"ZeroGPU env: {os.getenv('SPACES_ZERO_GPU')} | "
109
+ f"torch={torch.__version__} | "
110
+ f"parent CUDA initialized={torch.cuda.is_initialized()}",
111
+ flush=True,
112
+ )
113
+ print("Preloading FLAN-T5 onto ZeroGPU-managed CUDA...", flush=True)
114
+ print("Generation model:", load_generator(), flush=True)
115
+ print("Loading CPU retriever:", load_retriever(), flush=True)
116
+ print("Parent CUDA initialized after load:", torch.cuda.is_initialized(), flush=True)
 
 
 
117
 
118
+ demo = make_demo()
119
+ demo.queue(default_concurrency_limit=1, max_size=20)
120
+
121
+ if __name__ == "__main__":
122
  demo.launch(
123
  server_name="0.0.0.0",
124
+ server_port=int(os.getenv("PORT", "7860")),
 
125
  show_error=True,
126
+ share=False,
127
  ssr_mode=False,
128
  )