File size: 10,008 Bytes
4fcd019 cc4d8d8 4fcd019 cc4d8d8 4fcd019 35212b7 4fcd019 35212b7 4fcd019 35212b7 4fcd019 35212b7 4fcd019 35212b7 4fcd019 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 | """
TalkToDoc Flask application.
Wires together speech-to-text, translation, natural language understanding,
and text-to-speech into the patient-provider communication flow described
in the research document.
Two-step flow:
1. Patient submits input (text or audio) in their chosen language.
The system transcribes it if needed, translates it to English, and
summarizes the likely symptoms or intent for the provider.
2. Provider submits a reply in English. The system translates it back
into the patient's language and generates spoken audio for it.
"""
import os
import uuid
from datetime import datetime, timezone
from dotenv import load_dotenv
from flask import Flask, request, jsonify, render_template, session
import database
import stt
import translation
import nlu
import tts
load_dotenv("env")
AUDIO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", "audio")
def create_app():
flask_app = Flask(__name__)
flask_app.secret_key = os.environ["SECRET_KEY"]
os.makedirs(AUDIO_DIR, exist_ok=True)
return flask_app
app = create_app()
# Runs whether the app is started directly (python app.py, local development)
# or imported by a production server like gunicorn (Render deployment).
# Initialization code inside "if __name__ == '__main__'" only runs on a
# direct start, gunicorn never triggers it, so anything the app needs to
# work at all has to happen here instead.
database.init_db()
tts.preload_models()
def get_current_user_id(language):
"""
Returns the user_id for the current browser session, creating a new
user and session record the first time this browser is seen. This is
what keeps a patient's messages tied to the same person instead of
creating a brand new anonymous patient on every single submission.
"""
if "user_id" not in session:
user_id = database.add_user(name="Patient", preferred_language=language, role="patient")
session_id = database.add_session(
user_id=user_id,
start_time=datetime.now(timezone.utc).isoformat(),
)
session["user_id"] = user_id
session["session_id"] = session_id
return session["user_id"]
@app.route("/")
def index():
return render_template("patient.html", languages=translation.SUPPORTED_LANGUAGES)
@app.route("/provider")
def provider_page():
return render_template("provider.html")
@app.route("/transcribe", methods=["POST"])
def transcribe_only():
"""
Transcribes an audio recording without saving anything or translating
it. Used by the transcript review step, so the patient can see and
edit what was heard before it becomes the message actually sent to
the provider. Nothing here touches the database.
"""
language = request.form.get("language")
if not language or language.lower() not in translation.SUPPORTED_LANGUAGES:
return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400
language = language.lower()
audio_file = request.files.get("audio")
if not audio_file:
return jsonify({"error": "provide an audio file"}), 400
temp_path = os.path.join(AUDIO_DIR, f"transcribe_{uuid.uuid4().hex}.wav")
audio_file.save(temp_path)
try:
patient_text = stt.transcribe_audio(temp_path, language)
finally:
os.remove(temp_path)
return jsonify({"patient_text": patient_text})
@app.route("/patient-input", methods=["POST"])
def patient_input():
language = request.form.get("language")
if not language or language.lower() not in translation.SUPPORTED_LANGUAGES:
return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400
language = language.lower()
text = request.form.get("text")
audio_file = request.files.get("audio")
if not text and not audio_file:
return jsonify({"error": "provide either text or audio"}), 400
if audio_file:
temp_path = os.path.join(AUDIO_DIR, f"upload_{uuid.uuid4().hex}.wav")
audio_file.save(temp_path)
try:
patient_text = stt.transcribe_audio(temp_path, language)
finally:
os.remove(temp_path)
else:
patient_text = text
english_text = translation.translate(patient_text, language, "english")
nlu_summary = nlu.interpret_query(english_text)
user_id = get_current_user_id(language)
interaction_id = database.add_interaction(
user_id=user_id,
input_text=patient_text,
detected_language=language,
translated_text=english_text,
nlu_summary=nlu_summary,
timestamp=datetime.now(timezone.utc).isoformat(),
)
return jsonify({
"interaction_id": interaction_id,
"patient_text": patient_text,
"translated_text": english_text,
"nlu_summary": nlu_summary,
})
@app.route("/preview-response", methods=["POST"])
def preview_response():
"""
Runs the exact same translate + synthesize pipeline as
/provider-response below, so the provider can read and hear precisely
what the patient will receive before committing to it. Unlike
/provider-response, this never touches the database - only "Send
reply" actually saves a response. Each preview click is a real
translation + speech synthesis call, same cost as an actual send.
"""
interaction_id = request.form.get("interaction_id")
response_text = request.form.get("response_text")
if not interaction_id or not response_text:
return jsonify({"error": "interaction_id and response_text are required"}), 400
interaction = database.get_interaction(interaction_id)
if not interaction:
return jsonify({"error": "interaction not found"}), 404
patient_language = interaction["detected_language"]
translated_response = translation.translate(response_text, "english", patient_language)
audio_filename = f"preview_{interaction_id}.wav"
audio_path = os.path.join(AUDIO_DIR, audio_filename)
tts.synthesize_speech(translated_response, patient_language, audio_path)
return jsonify({
"translated_response": translated_response,
"audio_url": f"/static/audio/{audio_filename}",
})
@app.route("/provider-response", methods=["POST"])
def provider_response():
interaction_id = request.form.get("interaction_id")
response_text = request.form.get("response_text")
# Optional cache-reuse fields: if the provider already generated a
# preview of this exact reply via /preview-response and didn't edit
# it afterward, the frontend passes the already-computed translation
# back here so this route can skip re-translating and re-synthesizing
# speech. Both are best-effort - if the preview audio isn't actually
# on disk, this quietly falls back to doing the full work itself,
# exactly as if these fields had never been sent.
cached_translation = request.form.get("translated_response")
reuse_audio = request.form.get("reuse_audio") == "true"
if not interaction_id or not response_text:
return jsonify({"error": "interaction_id and response_text are required"}), 400
interaction = database.get_interaction(interaction_id)
if not interaction:
return jsonify({"error": "interaction not found"}), 404
patient_language = interaction["detected_language"]
audio_filename = f"response_{interaction_id}.wav"
audio_path = os.path.join(AUDIO_DIR, audio_filename)
preview_path = os.path.join(AUDIO_DIR, f"preview_{interaction_id}.wav")
if cached_translation and reuse_audio and os.path.exists(preview_path):
translated_response = cached_translation
os.replace(preview_path, audio_path)
else:
translated_response = translation.translate(response_text, "english", patient_language)
tts.synthesize_speech(translated_response, patient_language, audio_path)
database.update_interaction_response(interaction_id, response_text, translated_response)
return jsonify({
"translated_response": translated_response,
"audio_url": f"/static/audio/{audio_filename}",
})
@app.route("/interaction/<interaction_id>")
def get_interaction(interaction_id):
interaction = database.get_interaction(interaction_id)
if not interaction:
return jsonify({"error": "interaction not found"}), 404
audio_filename = f"response_{interaction_id}.wav"
audio_path = os.path.join(AUDIO_DIR, audio_filename)
interaction["audio_url"] = f"/static/audio/{audio_filename}" if os.path.exists(audio_path) else None
return jsonify(interaction)
@app.route("/pending-interactions")
def pending_interactions():
return jsonify(database.get_pending_interactions())
@app.route("/completed-interactions")
def completed_interactions():
return jsonify(database.get_completed_interactions())
@app.route("/history")
def history():
if "user_id" not in session:
return jsonify([])
return jsonify(database.get_interactions_for_user(session["user_id"]))
@app.route("/patient-history/<user_id>")
def patient_history(user_id):
"""
All of one patient's interactions, for the provider workspace's
"conversation history" panel. Distinct from /history above: that route
always looks up the current browser's own session, which only works
for the patient viewing their own messages. The provider needs to look
up a specific patient's history by id instead.
"""
return jsonify(database.get_interactions_for_user(user_id))
@app.route("/end-session", methods=["POST"])
def end_session_route():
if "session_id" in session:
database.end_session(session["session_id"], datetime.now(timezone.utc).isoformat())
session.clear()
return jsonify({"status": "ended"})
if __name__ == "__main__":
is_production = os.environ.get("ENVIRONMENT") == "production"
app.run(debug=not is_production, host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))
|