| """ |
| TalkToDoc Flask application. |
| Wires together speech-to-text, translation, natural language understanding, |
| and text-to-speech into the patient-provider communication flow described |
| in the research document. |
| |
| Two-step flow: |
| 1. Patient submits input (text or audio) in their chosen language. |
| The system transcribes it if needed, translates it to English, and |
| summarizes the likely symptoms or intent for the provider. |
| 2. Provider submits a reply in English. The system translates it back |
| into the patient's language and generates spoken audio for it. |
| """ |
|
|
| import os |
| import uuid |
| from datetime import datetime, timezone |
|
|
| from dotenv import load_dotenv |
| from flask import Flask, request, jsonify, render_template, session |
|
|
| import database |
| import stt |
| import translation |
| import nlu |
| import tts |
|
|
| load_dotenv("env") |
|
|
| AUDIO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", "audio") |
|
|
|
|
| def create_app(): |
| flask_app = Flask(__name__) |
| flask_app.secret_key = os.environ["SECRET_KEY"] |
| os.makedirs(AUDIO_DIR, exist_ok=True) |
| return flask_app |
|
|
|
|
| app = create_app() |
|
|
| |
| |
| |
| |
| |
| database.init_db() |
| tts.preload_models() |
|
|
|
|
| def get_current_user_id(language): |
| """ |
| Returns the user_id for the current browser session, creating a new |
| user and session record the first time this browser is seen. This is |
| what keeps a patient's messages tied to the same person instead of |
| creating a brand new anonymous patient on every single submission. |
| """ |
| if "user_id" not in session: |
| user_id = database.add_user(name="Patient", preferred_language=language, role="patient") |
| session_id = database.add_session( |
| user_id=user_id, |
| start_time=datetime.now(timezone.utc).isoformat(), |
| ) |
| session["user_id"] = user_id |
| session["session_id"] = session_id |
|
|
| return session["user_id"] |
|
|
|
|
| @app.route("/") |
| def index(): |
| return render_template("patient.html", languages=translation.SUPPORTED_LANGUAGES) |
|
|
|
|
| @app.route("/provider") |
| def provider_page(): |
| return render_template("provider.html") |
|
|
|
|
| @app.route("/transcribe", methods=["POST"]) |
| def transcribe_only(): |
| """ |
| Transcribes an audio recording without saving anything or translating |
| it. Used by the transcript review step, so the patient can see and |
| edit what was heard before it becomes the message actually sent to |
| the provider. Nothing here touches the database. |
| """ |
| language = request.form.get("language") |
| if not language or language.lower() not in translation.SUPPORTED_LANGUAGES: |
| return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400 |
| language = language.lower() |
|
|
| audio_file = request.files.get("audio") |
| if not audio_file: |
| return jsonify({"error": "provide an audio file"}), 400 |
|
|
| temp_path = os.path.join(AUDIO_DIR, f"transcribe_{uuid.uuid4().hex}.wav") |
| audio_file.save(temp_path) |
| try: |
| patient_text = stt.transcribe_audio(temp_path, language) |
| finally: |
| os.remove(temp_path) |
|
|
| return jsonify({"patient_text": patient_text}) |
|
|
|
|
| @app.route("/patient-input", methods=["POST"]) |
| def patient_input(): |
| language = request.form.get("language") |
| if not language or language.lower() not in translation.SUPPORTED_LANGUAGES: |
| return jsonify({"error": f"language must be one of {translation.SUPPORTED_LANGUAGES}"}), 400 |
| language = language.lower() |
|
|
| text = request.form.get("text") |
| audio_file = request.files.get("audio") |
|
|
| if not text and not audio_file: |
| return jsonify({"error": "provide either text or audio"}), 400 |
|
|
| if audio_file: |
| temp_path = os.path.join(AUDIO_DIR, f"upload_{uuid.uuid4().hex}.wav") |
| audio_file.save(temp_path) |
| try: |
| patient_text = stt.transcribe_audio(temp_path, language) |
| finally: |
| os.remove(temp_path) |
| else: |
| patient_text = text |
|
|
| english_text = translation.translate(patient_text, language, "english") |
| nlu_summary = nlu.interpret_query(english_text) |
|
|
| user_id = get_current_user_id(language) |
| interaction_id = database.add_interaction( |
| user_id=user_id, |
| input_text=patient_text, |
| detected_language=language, |
| translated_text=english_text, |
| nlu_summary=nlu_summary, |
| timestamp=datetime.now(timezone.utc).isoformat(), |
| ) |
|
|
| return jsonify({ |
| "interaction_id": interaction_id, |
| "patient_text": patient_text, |
| "translated_text": english_text, |
| "nlu_summary": nlu_summary, |
| }) |
|
|
|
|
| @app.route("/preview-response", methods=["POST"]) |
| def preview_response(): |
| """ |
| Runs the exact same translate + synthesize pipeline as |
| /provider-response below, so the provider can read and hear precisely |
| what the patient will receive before committing to it. Unlike |
| /provider-response, this never touches the database - only "Send |
| reply" actually saves a response. Each preview click is a real |
| translation + speech synthesis call, same cost as an actual send. |
| """ |
| interaction_id = request.form.get("interaction_id") |
| response_text = request.form.get("response_text") |
|
|
| if not interaction_id or not response_text: |
| return jsonify({"error": "interaction_id and response_text are required"}), 400 |
|
|
| interaction = database.get_interaction(interaction_id) |
| if not interaction: |
| return jsonify({"error": "interaction not found"}), 404 |
|
|
| patient_language = interaction["detected_language"] |
| translated_response = translation.translate(response_text, "english", patient_language) |
|
|
| audio_filename = f"preview_{interaction_id}.wav" |
| audio_path = os.path.join(AUDIO_DIR, audio_filename) |
| tts.synthesize_speech(translated_response, patient_language, audio_path) |
|
|
| return jsonify({ |
| "translated_response": translated_response, |
| "audio_url": f"/static/audio/{audio_filename}", |
| }) |
|
|
|
|
| @app.route("/provider-response", methods=["POST"]) |
| def provider_response(): |
| interaction_id = request.form.get("interaction_id") |
| response_text = request.form.get("response_text") |
| |
| |
| |
| |
| |
| |
| |
| cached_translation = request.form.get("translated_response") |
| reuse_audio = request.form.get("reuse_audio") == "true" |
|
|
| if not interaction_id or not response_text: |
| return jsonify({"error": "interaction_id and response_text are required"}), 400 |
|
|
| interaction = database.get_interaction(interaction_id) |
| if not interaction: |
| return jsonify({"error": "interaction not found"}), 404 |
|
|
| patient_language = interaction["detected_language"] |
| audio_filename = f"response_{interaction_id}.wav" |
| audio_path = os.path.join(AUDIO_DIR, audio_filename) |
| preview_path = os.path.join(AUDIO_DIR, f"preview_{interaction_id}.wav") |
|
|
| if cached_translation and reuse_audio and os.path.exists(preview_path): |
| translated_response = cached_translation |
| os.replace(preview_path, audio_path) |
| else: |
| translated_response = translation.translate(response_text, "english", patient_language) |
| tts.synthesize_speech(translated_response, patient_language, audio_path) |
|
|
| database.update_interaction_response(interaction_id, response_text, translated_response) |
|
|
| return jsonify({ |
| "translated_response": translated_response, |
| "audio_url": f"/static/audio/{audio_filename}", |
| }) |
|
|
|
|
| @app.route("/interaction/<interaction_id>") |
| def get_interaction(interaction_id): |
| interaction = database.get_interaction(interaction_id) |
| if not interaction: |
| return jsonify({"error": "interaction not found"}), 404 |
|
|
| audio_filename = f"response_{interaction_id}.wav" |
| audio_path = os.path.join(AUDIO_DIR, audio_filename) |
| interaction["audio_url"] = f"/static/audio/{audio_filename}" if os.path.exists(audio_path) else None |
|
|
| return jsonify(interaction) |
|
|
|
|
| @app.route("/pending-interactions") |
| def pending_interactions(): |
| return jsonify(database.get_pending_interactions()) |
|
|
|
|
| @app.route("/completed-interactions") |
| def completed_interactions(): |
| return jsonify(database.get_completed_interactions()) |
|
|
|
|
| @app.route("/history") |
| def history(): |
| if "user_id" not in session: |
| return jsonify([]) |
| return jsonify(database.get_interactions_for_user(session["user_id"])) |
|
|
|
|
| @app.route("/patient-history/<user_id>") |
| def patient_history(user_id): |
| """ |
| All of one patient's interactions, for the provider workspace's |
| "conversation history" panel. Distinct from /history above: that route |
| always looks up the current browser's own session, which only works |
| for the patient viewing their own messages. The provider needs to look |
| up a specific patient's history by id instead. |
| """ |
| return jsonify(database.get_interactions_for_user(user_id)) |
|
|
|
|
| @app.route("/end-session", methods=["POST"]) |
| def end_session_route(): |
| if "session_id" in session: |
| database.end_session(session["session_id"], datetime.now(timezone.utc).isoformat()) |
| session.clear() |
| return jsonify({"status": "ended"}) |
|
|
|
|
| if __name__ == "__main__": |
| is_production = os.environ.get("ENVIRONMENT") == "production" |
| app.run(debug=not is_production, host="0.0.0.0", port=int(os.environ.get("PORT", 5000))) |
|
|