diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..ea05d98e9a4f01f0d656d175b4581d561253393d --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +# Hugging Face — model downloads + Spaces LLM fallback +HF_TOKEN= + +# Ollama (local LLM) +OLLAMA_MODEL=mindfull + +# Set to 1 when SadTalker checkpoints are not present +DISABLE_VIDEO=0 + +# Override HF model repos (optional; defaults in scripts/download_models.py) +# HF_REPO_F5TTS=IamSamkk/police-bot-f5tts +# HF_REPO_SADTALKER=IamSamkk/sadtalker-checkpoints +# HF_REPO_GFPGAN=IamSamkk/gfpgan-weights diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d7e0abb34841982593abf8d52d3fb0f4b155b21b --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# ── Python ────────────────────────────────────────────────────────────────── +venv/ +.venv/ +env/ +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +htmlcov/ +.coverage + +# ── Secrets & local config ──────────────────────────────────────────────────── +.env +.env.* +!.env.example +*.pem +*.key +credentials.json + +# ── IDE / OS ───────────────────────────────────────────────────────────────── +.idea/ +.vscode/ +*.swp +.DS_Store +Thumbs.db +desktop.ini + +# ── Runtime outputs (regenerated) ───────────────────────────────────────────── +outputs/ +temp/ +*.log + +# ── Large ML weights (host on Hugging Face Hub, not GitHub) ───────────────── +*.pth +*.pt +*.pth.tar +*.safetensors +*.ckpt +*.onnx +*.bin +*.mat +*.dat + +my_finetuned_model/model.pth +my_finetuned_model/dvae.pth +my_finetuned_model/mel_stats.pth + +gfpgan/weights/ +sadtalker+wav2lip/gfpgan/weights/ +sadtalker+wav2lip/sadtalker/gfpgan/weights/ +sadtalker+wav2lip/sadtalker/checkpoints/ +sadtalker+wav2lip/wav2lip/Wav2Lip-SD-NOGAN.pt + +# ── Generated test media in sadtalker folder ────────────────────────────────── +sadtalker+wav2lip/test_*.mp4 +sadtalker+wav2lip/temp/ + +# ── Training data & local media (not for git / HF Space — use HF Hub or local disk) ── +datasets-1/ +segment_43.wav +avatar_assets/ +fm_Police.png +sadtalker+wav2lip/avatar_assets/ + +# Large auto-generated docs (keep locally; optional on GitHub) +CODEBASE_MAP.md +UNUSED_FILES.md diff --git a/INTEGRATION_GUIDE.md b/INTEGRATION_GUIDE.md new file mode 100644 index 0000000000000000000000000000000000000000..7be5ca3930fff5291530b2dc507881c408ebeb61 --- /dev/null +++ b/INTEGRATION_GUIDE.md @@ -0,0 +1,514 @@ +# 🛡️ Police Bot AI Integration Guide + +This guide explains how to integrate the AI wellness assistant with your website and enhance it with additional local LLM capabilities. + +## 🏗️ System Architecture + +``` +Website Frontend (React/HTML/CSS) + ↓ HTTP/WebSocket +Flask Web API (web_api.py) + ↓ +Police Bot Runtime (police_runtime.py) + ↓ +├── LLaMA3 via Ollama (Primary Reasoning) +├── F5-TTS (Voice Synthesis) +└── [Third Local LLM] (Enhanced Capabilities) +``` + +## 🚀 Quick Start + +### 1. Setup the AI Runtime + +```bash +# Run the setup script +python setup.py + +# Start Ollama with the police-bot model +ollama run police-bot + +# Start the web API +python web_api.py +``` + +### 2. Test the System + +```bash +# Test the command-line interface +python police_runtime.py + +# Test the web API +curl -X POST http://localhost:5000/api/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "I am feeling stressed today"}' +``` + +## 🌐 Website Integration + +### Option 1: Simple HTML Integration + +Add this to your website: + +```html + + +
+AI Wellness Companion · talking-head avatar · real TTS
+ """) + + # If running on HF Spaces without SadTalker checkpoints, default to audio-only + _video_disabled = os.environ.get("DISABLE_VIDEO", "0") == "1" + + # ── main layout ─────────────────────────────────────────────────────────── + with gr.Row(): + + # Left column – avatar + controls + with gr.Column(scale=1, min_width=260): + avatar_img = gr.Image( + label="Avatar Image (any portrait)", + type="numpy", + sources=["upload", "webcam"], + height=280, + ) + skip_video_chk = gr.Checkbox( + label="Skip video (faster · audio only)", + value=_video_disabled, + ) + gr.Markdown( + "_Upload **any** face image. " + "Leaving it empty will return text + audio only._", + elem_classes=["status-box"], + ) + + # Right column – chat + with gr.Column(scale=2): + chatbot = gr.Chatbot( + label="Conversation", + height=400, + type="messages", + avatar_images=(None, str(ROOT / "avatar_assets" / "officer.png") + if (ROOT / "avatar_assets" / "officer.png").exists() + else None), + ) + with gr.Row(): + msg_box = gr.Textbox( + placeholder="How are you feeling today?", + label="Your message", + scale=5, + lines=1, + ) + send_btn = gr.Button("Send", variant="primary", scale=1) + + # ── outputs ─────────────────────────────────────────────────────────────── + with gr.Row(): + audio_out = gr.Audio(label="🔊 Voice Response", type="filepath", autoplay=True) + video_out = gr.Video(label="🎬 Avatar Video", autoplay=True) + + status_box = gr.Textbox( + label="Status", interactive=False, lines=1, elem_classes=["status-box"] + ) + + # ── clear button ────────────────────────────────────────────────────────── + with gr.Row(): + clear_btn = gr.Button("🗑 Clear conversation", variant="secondary") + + # ── wiring ──────────────────────────────────────────────────────────────── + def _submit(msg, img, hist, skip): + yield from chat(msg, img, hist, skip) + + send_btn.click( + fn=_submit, + inputs=[msg_box, avatar_img, chatbot, skip_video_chk], + outputs=[chatbot, audio_out, video_out, status_box], + ).then(fn=lambda: "", outputs=msg_box) + + msg_box.submit( + fn=_submit, + inputs=[msg_box, avatar_img, chatbot, skip_video_chk], + outputs=[chatbot, audio_out, video_out, status_box], + ).then(fn=lambda: "", outputs=msg_box) + + clear_btn.click( + fn=lambda: ([], None, None, ""), # empty list = empty messages history + outputs=[chatbot, audio_out, video_out, status_box], + ) + + +if __name__ == "__main__": + demo.queue().launch( + server_name="0.0.0.0", + server_port=7860, + share=False, # set True for a public Gradio link + show_error=True, + ) diff --git a/infer_cli.py b/infer_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..95800fda25ca9b559cdd47652f0de8d9a8e18dfc --- /dev/null +++ b/infer_cli.py @@ -0,0 +1,368 @@ +import argparse +import codecs +import os +import re +from datetime import datetime +from importlib.resources import files +from pathlib import Path + +import numpy as np +import soundfile as sf +import tomli +from cached_path import cached_path +from hydra.utils import get_class +from omegaconf import OmegaConf + +from f5_tts.infer.utils_infer import ( + cfg_strength, + cross_fade_duration, + device, + fix_duration, + infer_process, + load_model, + load_vocoder, + mel_spec_type, + nfe_step, + preprocess_ref_audio_text, + remove_silence_for_generated_wav, + speed, + sway_sampling_coef, + target_rms, +) + + +parser = argparse.ArgumentParser( + prog="python3 infer-cli.py", + description="Commandline interface for E2/F5 TTS with Advanced Batch Processing.", + epilog="Specify options above to override one or more settings from config.", +) +parser.add_argument( + "-c", + "--config", + type=str, + default=os.path.join(files("f5_tts").joinpath("infer/examples/basic"), "basic.toml"), + help="The configuration file, default see infer/examples/basic/basic.toml", +) + + +# Note. Not to provide default value here in order to read default from config file + +parser.add_argument( + "-m", + "--model", + type=str, + help="The model name: F5TTS_v1_Base | F5TTS_Base | E2TTS_Base | etc.", +) +parser.add_argument( + "-mc", + "--model_cfg", + type=str, + help="The path to F5-TTS model config file .yaml", +) +parser.add_argument( + "-p", + "--ckpt_file", + type=str, + help="The path to model checkpoint .pt, leave blank to use default", +) +parser.add_argument( + "-v", + "--vocab_file", + type=str, + help="The path to vocab file .txt, leave blank to use default", +) +parser.add_argument( + "-r", + "--ref_audio", + type=str, + help="The reference audio file.", +) +parser.add_argument( + "-s", + "--ref_text", + type=str, + help="The transcript/subtitle for the reference audio", +) +parser.add_argument( + "-t", + "--gen_text", + type=str, + help="The text to make model synthesize a speech", +) +parser.add_argument( + "-f", + "--gen_file", + type=str, + help="The file with text to generate, will ignore --gen_text", +) +parser.add_argument( + "-o", + "--output_dir", + type=str, + help="The path to output folder", +) +parser.add_argument( + "-w", + "--output_file", + type=str, + help="The name of output file", +) +parser.add_argument( + "--save_chunk", + action="store_true", + help="To save each audio chunks during inference", +) +parser.add_argument( + "--remove_silence", + action="store_true", + help="To remove long silence found in ouput", +) +parser.add_argument( + "--load_vocoder_from_local", + action="store_true", + help="To load vocoder from local dir, default to ../checkpoints/vocos-mel-24khz", +) +parser.add_argument( + "--vocoder_name", + type=str, + choices=["vocos", "bigvgan"], + help=f"Used vocoder name: vocos | bigvgan, default {mel_spec_type}", +) +parser.add_argument( + "--target_rms", + type=float, + help=f"Target output speech loudness normalization value, default {target_rms}", +) +parser.add_argument( + "--cross_fade_duration", + type=float, + help=f"Duration of cross-fade between audio segments in seconds, default {cross_fade_duration}", +) +parser.add_argument( + "--nfe_step", + type=int, + help=f"The number of function evaluation (denoising steps), default {nfe_step}", +) +parser.add_argument( + "--cfg_strength", + type=float, + help=f"Classifier-free guidance strength, default {cfg_strength}", +) +parser.add_argument( + "--sway_sampling_coef", + type=float, + help=f"Sway Sampling coefficient, default {sway_sampling_coef}", +) +parser.add_argument( + "--speed", + type=float, + help=f"The speed of the generated audio, default {speed}", +) +parser.add_argument( + "--fix_duration", + type=float, + help=f"Fix the total duration (ref and gen audios) in seconds, default {fix_duration}", +) +parser.add_argument( + "--device", + type=str, + help="Specify the device to run on", +) +args = parser.parse_args() + + +# config file + +config = tomli.load(open(args.config, "rb")) + + +# command-line interface parameters + +model = args.model or config.get("model", "F5TTS_v1_Base") +ckpt_file = args.ckpt_file or config.get("ckpt_file", "") +vocab_file = args.vocab_file or config.get("vocab_file", "") + +ref_audio = args.ref_audio or config.get("ref_audio", "infer/examples/basic/basic_ref_en.wav") +ref_text = ( + args.ref_text + if args.ref_text is not None + else config.get("ref_text", "Some call me nature, others call me mother nature.") +) +gen_text = args.gen_text or config.get("gen_text", "Here we generate something just for test.") +gen_file = args.gen_file or config.get("gen_file", "") + +output_dir = args.output_dir or config.get("output_dir", "tests") +output_file = args.output_file or config.get( + "output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav" +) + +save_chunk = args.save_chunk or config.get("save_chunk", False) +remove_silence = args.remove_silence or config.get("remove_silence", False) +load_vocoder_from_local = args.load_vocoder_from_local or config.get("load_vocoder_from_local", False) + +vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type) +target_rms = args.target_rms or config.get("target_rms", target_rms) +cross_fade_duration = args.cross_fade_duration or config.get("cross_fade_duration", cross_fade_duration) +nfe_step = args.nfe_step or config.get("nfe_step", nfe_step) +cfg_strength = args.cfg_strength or config.get("cfg_strength", cfg_strength) +sway_sampling_coef = args.sway_sampling_coef or config.get("sway_sampling_coef", sway_sampling_coef) +speed = args.speed or config.get("speed", speed) +fix_duration = args.fix_duration or config.get("fix_duration", fix_duration) +device = args.device or config.get("device", device) + + +# patches for pip pkg user +if "infer/examples/" in ref_audio: + ref_audio = str(files("f5_tts").joinpath(f"{ref_audio}")) +if "infer/examples/" in gen_file: + gen_file = str(files("f5_tts").joinpath(f"{gen_file}")) +if "voices" in config: + for voice in config["voices"]: + voice_ref_audio = config["voices"][voice]["ref_audio"] + if "infer/examples/" in voice_ref_audio: + config["voices"][voice]["ref_audio"] = str(files("f5_tts").joinpath(f"{voice_ref_audio}")) + + +# ignore gen_text if gen_file provided + +if gen_file: + gen_text = codecs.open(gen_file, "r", "utf-8").read() + + +# output path + +wave_path = Path(output_dir) / output_file +# spectrogram_path = Path(output_dir) / "infer_cli_out.png" +if save_chunk: + output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks") + if not os.path.exists(output_chunk_dir): + os.makedirs(output_chunk_dir) + + +# load vocoder + +if vocoder_name == "vocos": + vocoder_local_path = "../checkpoints/vocos-mel-24khz" +elif vocoder_name == "bigvgan": + vocoder_local_path = "../checkpoints/bigvgan_v2_24khz_100band_256x" + +vocoder = load_vocoder( + vocoder_name=vocoder_name, is_local=load_vocoder_from_local, local_path=vocoder_local_path, device=device +) + + +# load TTS model + +model_cfg = OmegaConf.load( + args.model_cfg or config.get("model_cfg", str(files("f5_tts").joinpath(f"configs/{model}.yaml"))) +) +model_cls = get_class(f"f5_tts.model.{model_cfg.model.backbone}") +model_arc = model_cfg.model.arch + +repo_name, ckpt_step, ckpt_type = "F5-TTS", 1250000, "safetensors" + +if model != "F5TTS_Base": + assert vocoder_name == model_cfg.model.mel_spec.mel_spec_type + +# override for previous models +if model == "F5TTS_Base": + if vocoder_name == "vocos": + ckpt_step = 1200000 + elif vocoder_name == "bigvgan": + model = "F5TTS_Base_bigvgan" + ckpt_type = "pt" +elif model == "E2TTS_Base": + repo_name = "E2-TTS" + ckpt_step = 1200000 + +if not ckpt_file: + ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{model}/model_{ckpt_step}.{ckpt_type}")) + +print(f"Using {model}...") +ema_model = load_model( + model_cls, model_arc, ckpt_file, mel_spec_type=vocoder_name, vocab_file=vocab_file, device=device +) + + +# inference process + + +def main(): + main_voice = {"ref_audio": ref_audio, "ref_text": ref_text} + if "voices" not in config: + voices = {"main": main_voice} + else: + voices = config["voices"] + voices["main"] = main_voice + for voice in voices: + print("Voice:", voice) + print("ref_audio ", voices[voice]["ref_audio"]) + voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text( + voices[voice]["ref_audio"], voices[voice]["ref_text"] + ) + print("ref_audio_", voices[voice]["ref_audio"], "\n\n") + + generated_audio_segments = [] + reg1 = r"(?=\[\w+\])" + chunks = re.split(reg1, gen_text) + reg2 = r"\[(\w+)\]" + for text in chunks: + if not text.strip(): + continue + match = re.match(reg2, text) + if match: + voice = match[1] + else: + print("No voice tag found, using main.") + voice = "main" + if voice not in voices: + print(f"Voice {voice} not found, using main.") + voice = "main" + text = re.sub(reg2, "", text) + ref_audio_ = voices[voice]["ref_audio"] + ref_text_ = voices[voice]["ref_text"] + gen_text_ = text.strip() + print(f"Voice: {voice}") + audio_segment, final_sample_rate, spectrogram = infer_process( + ref_audio_, + ref_text_, + gen_text_, + ema_model, + vocoder, + mel_spec_type=vocoder_name, + target_rms=target_rms, + cross_fade_duration=cross_fade_duration, + nfe_step=nfe_step, + cfg_strength=cfg_strength, + sway_sampling_coef=sway_sampling_coef, + speed=speed, + fix_duration=fix_duration, + device=device, + ) + generated_audio_segments.append(audio_segment) + + if save_chunk: + if len(gen_text_) > 200: + gen_text_ = gen_text_[:200] + " ... " + sf.write( + os.path.join(output_chunk_dir, f"{len(generated_audio_segments) - 1}_{gen_text_}.wav"), + audio_segment, + final_sample_rate, + ) + + if generated_audio_segments: + final_wave = np.concatenate(generated_audio_segments) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + with open(wave_path, "wb") as f: + sf.write(f.name, final_wave, final_sample_rate) + # Remove silence + if remove_silence: + remove_silence_for_generated_wav(f.name) + print(f.name) + + +if __name__ == "__main__": + main() diff --git a/mindfull_client.html b/mindfull_client.html new file mode 100644 index 0000000000000000000000000000000000000000..b41147e290bb6e647dd1514896bcfc6f3c344e63 --- /dev/null +++ b/mindfull_client.html @@ -0,0 +1,515 @@ + + + + + +AI Mental Wellness Assistant for Police Officers
+