Fix root cause: model was never loading on the Space (Gradio SDK never runs FastAPI lifespan), causing every request to crash. Module-level load + correct ZeroGPU device pattern + always-JSON outputs.

app.py

@@ -6,7 +6,6 @@ Inference endpoint + interactive demo.
 import os
 import sqlite3
 import json
-from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 import gradio as gr
@@ -15,105 +14,105 @@ from peft import AutoPeftModelForCausalLM
 import torch
 import spaces
 
-# Global model & tokenizer
-model = None
-tokenizer = None
+# Model is loaded once at IMPORT TIME, not inside a FastAPI lifespan hook.
+# HF Spaces' Gradio SDK serves the `demo` Blocks object directly and never
+# runs `app`'s ASGI lifespan -- confirmed via server logs that this left
+# model/tokenizer permanently None, so every request hit the "Model not
+# loaded" fallback and crashed the gr.JSON output component with it.
+#
+# Also: under HF's free ZeroGPU tier, no GPU is visible at import time --
+# it's only allocated for the duration of an @spaces.GPU-decorated call.
+# So load on CPU here, then move to CUDA inside gradio_interface() below.
+print("Loading fine-tuned model...")
+ADAPTER_PATH = "vishnuadupa/qwen-sql-lora"
+
+try:
+    print(f"Loading adapter from HF Hub: {ADAPTER_PATH}")
+    tokenizer = AutoTokenizer.from_pretrained(ADAPTER_PATH)
+    model = AutoPeftModelForCausalLM.from_pretrained(ADAPTER_PATH)
+    model = model.merge_and_unload()
+    print("Fine-tuned model loaded.")
+except Exception as e:
+    print(f"Could not load fine-tuned model ({e}); falling back to base model.")
+    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-1.5B-Instruct")
+    model = AutoModelForCausalLM.from_pretrained(
+        "Qwen/Qwen2.5-Coder-1.5B-Instruct", torch_dtype=torch.float16
+    )
+    print("Base model loaded.")
 
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    """Load model on startup, unload on shutdown."""
-    global model, tokenizer
-    print("Loading fine-tuned model...")
+model = model.to("cpu")
 
-    # Try to load from HF Hub (set this to your repo after training)
-    adapter_path = "vishnuadupa/qwen-sql-lora"
+app = FastAPI(title="SQL Copilot")
 
-    try:
-        print(f"Loading adapter from HF Hub: {adapter_path}")
-        tokenizer = AutoTokenizer.from_pretrained(adapter_path)
-        model = AutoPeftModelForCausalLM.from_pretrained(adapter_path, device_map="auto")
-        model = model.merge_and_unload()
-        print("✓ Fine-tuned model loaded!")
-    except Exception as e:
-        print(f"⚠ Could not load fine-tuned model: {e}")
-        print("Falling back to base model...")
-        tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-1.5B-Instruct")
-        model = AutoModelForCausalLM.from_pretrained(
-            "Qwen/Qwen2.5-Coder-1.5B-Instruct",
-            torch_dtype=torch.float16,
-            device_map="auto"
-        )
-        print("✓ Base model loaded")
-
-    yield
-    print("Unloading model...")
-    del model
-    torch.cuda.empty_cache()
-
-app = FastAPI(title="SQL Copilot", lifespan=lifespan)
 
 class SQLRequest(BaseModel):
     question: str
     schema: str
 
+
 class SQLResponse(BaseModel):
     sql: str
     error: str = None
 
-@app.post("/predict", response_model=SQLResponse)
-async def predict_sql(req: SQLRequest):
-    """Generate SQL from natural language."""
-    if model is None:
-        raise HTTPException(status_code=503, detail="Model not loaded")
 
+def build_prompt(question: str, schema: str) -> str:
     # Must match train_lora.py's training format exactly (wording + trailing
     # newline after "SQL:") -- this LoRA adapter is lightly trained and
     # very sensitive to prompt shape; a mismatched format was confirmed to
     # collapse accuracy from ~60% to ~2% in eval.py before this same fix.
-    prompt = f"""You are a SQL expert. Generate valid SQL.
+    return f"""You are a SQL expert. Generate valid SQL.
 
-Question: {req.question}
+Question: {question}
 
 Schema:
-{req.schema}
+{schema}
 
 SQL:
 """
 
-    try:
-        inputs = tokenizer(prompt, return_tensors="pt")
-        # Greedy decoding, not sampling: a SQL generator should return its
-        # single most-confident answer, not a random draw that can vary
-        # between identical requests.
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=256,
-            do_sample=False,
-            eos_token_id=tokenizer.eos_token_id,
-            pad_token_id=tokenizer.eos_token_id,
-        )
-        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        sql = response.split("SQL:")[-1].strip()
 
-        if "```" in sql:
-            sql = sql.split("```")[1].split("```")[0].strip()
+def generate_sql_text(question: str, schema: str, device: str = "cpu") -> str:
+    prompt = build_prompt(question, schema)
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
+    # Greedy decoding, not sampling: a SQL generator should return its
+    # single most-confident answer, not a random draw that can vary
+    # between identical requests.
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=256,
+        do_sample=False,
+        eos_token_id=tokenizer.eos_token_id,
+        pad_token_id=tokenizer.eos_token_id,
+    )
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    sql = response.split("SQL:")[-1].strip()
 
-        # Take first line only: the model's actual answer is always the
-        # first line, and it can drift into commentary (or even a second,
-        # unrelated code block) afterward on lightly-trained checkpoints.
-        sql = sql.split("\n")[0].strip()
+    if "```" in sql:
+        sql = sql.split("```")[1].split("```")[0].strip()
 
+    # Take first line only: the model's actual answer is always the first
+    # line, and it can drift into commentary (or even a second, unrelated
+    # code block) afterward on lightly-trained checkpoints.
+    return sql.split("\n")[0].strip()
+
+
+@app.post("/predict", response_model=SQLResponse)
+async def predict_sql(req: SQLRequest):
+    """Generate SQL from natural language."""
+    try:
+        sql = generate_sql_text(req.question, req.schema)
         return SQLResponse(sql=sql)
     except Exception as e:
         return SQLResponse(sql="", error=str(e))
 
+
 @app.get("/health")
 async def health():
     return {"status": "ok"}
 
-# Sample SQLite database for demo
+
 def create_sample_db():
-    """Create a sample DB for testing."""
+    """Create a sample DB for the demo."""
     conn = sqlite3.connect("sample.db")
     cursor = conn.cursor()
 
@@ -157,17 +156,25 @@ def create_sample_db():
     conn.commit()
     conn.close()
 
+
+# Run unconditionally at import time -- HF Spaces' Gradio SDK imports this
+# module rather than executing it as __main__, so anything gated behind
+# `if __name__ == "__main__":` never runs there.
+create_sample_db()
+
+
 def execute_sql(sql: str):
-    """Execute SQL and return results."""
+    """Execute SQL, returning (results_or_None, error_message_or_None)."""
     try:
         conn = sqlite3.connect("sample.db")
         cursor = conn.cursor()
         cursor.execute(sql)
         results = cursor.fetchall()
         conn.close()
-        return results
+        return results, None
     except Exception as e:
-        return f"Error: {str(e)}"
+        return None, str(e)
+
 
 @spaces.GPU
 def gradio_interface(question, schema):
@@ -176,43 +183,26 @@ def gradio_interface(question, schema):
     @spaces.GPU is required by HF Spaces' free ZeroGPU tier -- without it,
     the Space fails at startup with "No @spaces.GPU function detected"
     since ZeroGPU only allocates GPU time to functions explicitly marked
-    this way.
+    this way. GPU is only actually visible for the duration of this call,
+    so the model is moved to "cuda" here rather than at import time.
     """
-    if not model:
-        return "Model not loaded", "N/A"
-
-    req = SQLRequest(question=question, schema=schema)
-    response = execute_sql(req.question)
-
-    # Must match training format exactly -- see note in /predict above.
-    prompt = f"""You are a SQL expert. Generate valid SQL.
-
-Question: {req.question}
-
-Schema:
-{req.schema}
-
-SQL:
-"""
-
-    inputs = tokenizer(prompt, return_tensors="pt")
-    # Greedy decoding — see note in /predict above.
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=256,
-        do_sample=False,
-        eos_token_id=tokenizer.eos_token_id,
-        pad_token_id=tokenizer.eos_token_id,
-    )
-    sql = tokenizer.decode(outputs[0], skip_special_tokens=True).split("SQL:")[-1].strip()
-    # Take first line only — see note in /predict above.
-    sql = sql.split("\n")[0].strip()
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
 
     try:
-        results = execute_sql(sql)
-        return sql, json.dumps(results, indent=2)
+        sql = generate_sql_text(question, schema, device=device)
     except Exception as e:
-        return sql, f"Execution error: {str(e)}"
+        # result_output is a gr.JSON component: it always requires a
+        # JSON-serializable value, never a bare string, or it crashes with
+        # "Invalid JSON string" -- confirmed via server logs to be exactly
+        # what was happening on every request before this fix.
+        return "", json.dumps({"error": f"Generation failed: {e}"})
+
+    results, error = execute_sql(sql)
+    if error:
+        return sql, json.dumps({"error": error})
+    return sql, json.dumps(results, indent=2)
+
 
 # Gradio UI
 with gr.Blocks(title="SQL Copilot") as demo:
@@ -228,12 +218,14 @@ with gr.Blocks(title="SQL Copilot") as demo:
             )
             schema = gr.Textbox(
                 label="Database Schema",
-                placeholder="Paste your schema here",
+                placeholder="Paste your schema here (CREATE TABLE statements)",
                 lines=5,
-                value="""
-Table: customers (id, name, region, revenue)
-Table: orders (id, customer_id, amount, date)
-                """
+                # CREATE TABLE format, not a "Table: x (...)" summary --
+                # matches what the model was actually trained on
+                # (b-mc2/sql-create-context), so the default example
+                # actually demonstrates the model's real behavior.
+                value="""CREATE TABLE customers (id INTEGER, name TEXT, region TEXT, revenue REAL)
+CREATE TABLE orders (id INTEGER, customer_id INTEGER, amount REAL, date TEXT)"""
             )
             submit_btn = gr.Button("Generate SQL", variant="primary")
 
@@ -248,7 +240,4 @@ Table: orders (id, customer_id, amount, date)
     )
 
 if __name__ == "__main__":
-    create_sample_db()
-
-    # Launch Gradio
     demo.launch(server_name="0.0.0.0", server_port=7860, share=False)